Update from Google. -- MOE_MIGRATED_REVID=85702957
diff --git a/src/main/java/BUILD b/src/main/java/BUILD new file mode 100644 index 0000000..503426f --- /dev/null +++ b/src/main/java/BUILD
@@ -0,0 +1,87 @@ +java_library( + name = "shell", + srcs = glob(["com/google/devtools/build/lib/shell/*.java"]), + visibility = ["//src:__subpackages__"], + deps = ["//third_party:guava"], +) + +java_library( + name = "bazel-core", + srcs = glob( + ["**/*.java"], + exclude = ["com/google/devtools/build/lib/shell/*.java"], + ), + resources = glob([ + "**/*.txt", + "**/*.html", + "**/*.css", + "**/*.js", + ]), + visibility = ["//src/test/java:__subpackages__"], + runtime_deps = [ + "//third_party:aether", + "//third_party:apache_commons_logging", + "//third_party:apache_httpclient", + "//third_party:apache_httpcore", + "//third_party:maven_model", + "//third_party:plexus_interpolation", + "//third_party:plexus_utils", + ], + deps = [ + ":shell", + "//src/main/protobuf:proto_build", + "//src/main/protobuf:proto_bundlemerge", + "//src/main/protobuf:proto_crosstool_config", + "//src/main/protobuf:proto_extra_actions_base", + "//src/main/protobuf:proto_test_status", + "//src/main/protobuf:proto_xcodegen", + "//src/tools/xcode-common", + "//third_party:aether", + "//third_party:apache_commons_compress", + "//third_party:gson", + "//third_party:guava", + "//third_party:joda-time", + "//third_party:jsr305", + "//third_party:maven_model", + "//third_party:protobuf", + ], +) + +java_binary( + name = "bazel-main", + main_class = "com.google.devtools.build.lib.bazel.BazelMain", + visibility = ["//src:__pkg__"], + runtime_deps = [ + ":bazel-core", + ], +) + +# Build encyclopedia generation. +filegroup( + name = "gen_be_sources", + srcs = glob(["com/google/devtools/build/lib/**/*.java"]), +) + +java_binary( + name = "docgen_bin", + srcs = glob(["com/google/devtools/build/docgen/*.java"]), + data = [":gen_be_sources"], + main_class = "com.google.devtools.build.docgen.BuildEncyclopediaGenerator", + resources = glob( + ["com/google/devtools/build/docgen/templates/*.html"], + ), + deps = [ + ":bazel-core", + "//third_party:guava", + "//third_party:jsr305", + ], +) + +genrule( + name = "gen_buildencyclopedia", + srcs = [":gen_be_sources"], + outs = ["build-encyclopedia.html"], + cmd = " docgen_bin $$PWD/src/main/java/com/google/devtools/build/lib $$PWD;" + + "cp $$PWD/build-encyclopedia.html $@", + tools = [":docgen_bin"], +)
diff --git a/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java b/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java new file mode 100644 index 0000000..12687db --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/BlazeRuleHelpPrinter.java
@@ -0,0 +1,60 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.docgen; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * A helper class to load and store printable build rule documentation. The doc + * printed here doesn't contain attribute and implicit output definitions, just + * the general rule documentation and examples. + */ +public class BlazeRuleHelpPrinter { + + private static Map<String, RuleDocumentation> ruleDocMap = null; + + /** + * Returns the documentation of the given rule to be printed on the console. + */ + public static String getRuleDoc(String ruleName, ConfiguredRuleClassProvider provider) { + if (ruleDocMap == null) { + try { + BuildEncyclopediaProcessor processor = new BuildEncyclopediaProcessor(provider); + Set<RuleDocumentation> ruleDocs = processor.collectAndProcessRuleDocs( + new String[] {"java/com/google/devtools/build/lib/view", + "java/com/google/devtools/build/lib/rules"}, false); + ruleDocMap = new HashMap<>(); + for (RuleDocumentation ruleDoc : ruleDocs) { + ruleDocMap.put(ruleDoc.getRuleName(), ruleDoc); + } + } catch (BuildEncyclopediaDocException e) { + return e.getErrorMsg(); + } catch (IOException e) { + return e.getMessage(); + } + } + // Every rule should be documented and this method should be called only + // for existing rules (a check is performed in HelpCommand). + Preconditions.checkState(ruleDocMap.containsKey(ruleName), String.format( + "ERROR: Documentation of rule %s does not exist.", ruleName)); + return "Rule " + ruleName + ":" + + ruleDocMap.get(ruleName).getCommandLineDocumentation() + "\n"; + } +}
diff --git a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaDocException.java b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaDocException.java new file mode 100644 index 0000000..d3b12c4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaDocException.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.docgen; + +/** + * An exception for Build Encyclopedia generation implementing the common BLAZE + * error formatting, i.e. displaying file name and line number. + */ +public class BuildEncyclopediaDocException extends Exception { + + private String fileName; + private int lineNumber; + private String errorMsg; + + public BuildEncyclopediaDocException(String fileName, int lineNumber, String errorMsg) { + this.fileName = fileName; + this.lineNumber = lineNumber; + this.errorMsg = errorMsg; + } + + public String getFileName() { + return fileName; + } + + public int getLineNumber() { + return lineNumber; + } + + public String getErrorMsg() { + return errorMsg; + } + + @Override + public String getMessage() { + return "Error in " + fileName + ":" + lineNumber + ": " + errorMsg; + } +}
diff --git a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaGenerator.java b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaGenerator.java new file mode 100644 index 0000000..e2bb844 --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaGenerator.java
@@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.docgen; + +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +/** + * The main class for the docgen project. The class checks the input arguments + * and uses the BuildEncyclopediaProcessor for the actual documentation generation. + */ +public class BuildEncyclopediaGenerator { + + private static boolean checkArgs(String[] args) { + if (args.length < 1) { + System.err.println("There has to be one or two input parameters\n" + + " - a comma separated list for input directories\n" + + " - an output directory (optional)."); + return false; + } + return true; + } + + private static void fail(Throwable e, boolean printStackTrace) { + System.err.println("ERROR: " + e.getMessage()); + if (printStackTrace) { + e.printStackTrace(); + } + Runtime.getRuntime().exit(1); + } + + private static ConfiguredRuleClassProvider createRuleClassProvider() + throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, + IllegalAccessException { + Class<?> providerClass = Class.forName(Constants.MAIN_RULE_CLASS_PROVIDER); + Method createMethod = providerClass.getMethod("create"); + return (ConfiguredRuleClassProvider) createMethod.invoke(null); + } + + public static void main(String[] args) { + if (checkArgs(args)) { + // TODO(bazel-team): use flags + try { + BuildEncyclopediaProcessor processor = new BuildEncyclopediaProcessor( + createRuleClassProvider()); + processor.generateDocumentation(args[0].split(","), args.length > 1 ? args[1] : null); + } catch (BuildEncyclopediaDocException e) { + fail(e, false); + } catch (Throwable e) { + fail(e, true); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java new file mode 100644 index 0000000..fb52a4c --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/BuildEncyclopediaProcessor.java
@@ -0,0 +1,403 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.docgen; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.devtools.build.docgen.DocgenConsts.RuleType; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.RuleClass; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +/** + * A class to assemble documentation for the Build Encyclopedia. The + * program parses the documentation fragments of rule-classes and + * generates the html format documentation. + */ +public class BuildEncyclopediaProcessor { + + private ConfiguredRuleClassProvider ruleClassProvider; + + /** + * Creates the BuildEncyclopediaProcessor instance. The ruleClassProvider parameter + * is used for rule class hierarchy and attribute checking. + * + */ + public BuildEncyclopediaProcessor(ConfiguredRuleClassProvider ruleClassProvider) { + this.ruleClassProvider = Preconditions.checkNotNull(ruleClassProvider); + } + + /** + * Collects and processes all the rule and attribute documentation in inputDirs and + * generates the Build Encyclopedia into the outputRootDir. + */ + public void generateDocumentation(String[] inputDirs, String outputRootDir) + throws BuildEncyclopediaDocException, IOException { + BufferedWriter bw = null; + File buildEncyclopediaPath = setupDirectories(outputRootDir); + try { + bw = new BufferedWriter(new FileWriter(buildEncyclopediaPath)); + bw.write(DocgenConsts.HEADER_COMMENT); + + Set<RuleDocumentation> ruleDocEntries = collectAndProcessRuleDocs(inputDirs, false); + writeRuleClassDocs(ruleDocEntries, bw); + + bw.write(SourceFileReader.readTemplateContents(DocgenConsts.FOOTER_TEMPLATE)); + + } finally { + if (bw != null) { + bw.close(); + } + } + } + + /** + * Collects all the rule and attribute documentation present in inputDirs, integrates the + * attribute documentation in the rule documentation and returns the rule documentation. + */ + public Set<RuleDocumentation> collectAndProcessRuleDocs(String[] inputDirs, + boolean printMessages) throws BuildEncyclopediaDocException, IOException { + // RuleDocumentations are generated in order (based on rule type then alphabetically). + // The ordering is also used to determine in which rule doc the common attribute docs are + // generated (they are generated at the first appearance). + Set<RuleDocumentation> ruleDocEntries = new TreeSet<>(); + // RuleDocumentationAttribute objects equal based on attributeName so they have to be + // collected in a List instead of a Set. + ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries = + LinkedListMultimap.create(); + for (String inputDir : inputDirs) { + if (printMessages) { + System.out.println(" Processing input directory: " + inputDir); + } + int ruleNum = ruleDocEntries.size(); + collectDocs(ruleDocEntries, attributeDocEntries, new File(inputDir)); + if (printMessages) { + System.out.println( + " " + (ruleDocEntries.size() - ruleNum) + " rule documentations found."); + } + } + + processAttributeDocs(ruleDocEntries, attributeDocEntries); + return ruleDocEntries; + } + + /** + * Go through all attributes of all documented rules and search the best attribute documentation + * if exists. The best documentation is the closest documentation in the ancestor graph. E.g. if + * java_library.deps documented in $rule and $java_rule then the one in $java_rule is going to + * apply since it's a closer ancestor of java_library. + */ + private void processAttributeDocs(Set<RuleDocumentation> ruleDocEntries, + ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries) + throws BuildEncyclopediaDocException { + for (RuleDocumentation ruleDoc : ruleDocEntries) { + RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleDoc.getRuleName()); + if (ruleClass != null) { + if (ruleClass.isDocumented()) { + Class<? extends RuleDefinition> ruleDefinition = + ruleClassProvider.getRuleClassDefinition(ruleDoc.getRuleName()); + for (Attribute attribute : ruleClass.getAttributes()) { + String attrName = attribute.getName(); + List<RuleDocumentationAttribute> attributeDocList = + attributeDocEntries.get(attrName); + if (attributeDocList != null) { + // There are attribute docs for this attribute. + // Search the closest one in the ancestor graph. + // Note that there can be only one 'closest' attribute since we forbid multiple + // inheritance of the same attribute in RuleClass. + int minLevel = Integer.MAX_VALUE; + RuleDocumentationAttribute bestAttributeDoc = null; + for (RuleDocumentationAttribute attributeDoc : attributeDocList) { + int level = attributeDoc.getDefinitionClassAncestryLevel(ruleDefinition); + if (level >= 0 && level < minLevel) { + bestAttributeDoc = attributeDoc; + minLevel = level; + } + } + if (bestAttributeDoc != null) { + ruleDoc.addAttribute(bestAttributeDoc); + // If there is no matching attribute doc try to add the common. + } else if (ruleDoc.getRuleType().equals(RuleType.BINARY) + && PredefinedAttributes.BINARY_ATTRIBUTES.containsKey(attrName)) { + ruleDoc.addAttribute(PredefinedAttributes.BINARY_ATTRIBUTES.get(attrName)); + } else if (ruleDoc.getRuleType().equals(RuleType.TEST) + && PredefinedAttributes.TEST_ATTRIBUTES.containsKey(attrName)) { + ruleDoc.addAttribute(PredefinedAttributes.TEST_ATTRIBUTES.get(attrName)); + } else if (PredefinedAttributes.COMMON_ATTRIBUTES.containsKey(attrName)) { + ruleDoc.addAttribute(PredefinedAttributes.COMMON_ATTRIBUTES.get(attrName)); + } + } + } + } + } else { + throw ruleDoc.createException("Can't find RuleClass for " + ruleDoc.getRuleName()); + } + } + } + + /** + * Categorizes, checks and prints all the rule-class documentations. + */ + private void writeRuleClassDocs(Set<RuleDocumentation> docEntries, BufferedWriter bw) + throws BuildEncyclopediaDocException, IOException { + Set<RuleDocumentation> binaryDocs = new TreeSet<>(); + Set<RuleDocumentation> libraryDocs = new TreeSet<>(); + Set<RuleDocumentation> testDocs = new TreeSet<>(); + Set<RuleDocumentation> generateDocs = new TreeSet<>(); + Set<RuleDocumentation> otherDocs = new TreeSet<>(); + + for (RuleDocumentation doc : docEntries) { + RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(doc.getRuleName()); + if (ruleClass.isDocumented()) { + if (doc.isLanguageSpecific()) { + switch(doc.getRuleType()) { + case BINARY: + binaryDocs.add(doc); + break; + case LIBRARY: + libraryDocs.add(doc); + break; + case TEST: + testDocs.add(doc); + break; + case OTHER: + otherDocs.add(doc); + break; + } + } else { + otherDocs.add(doc); + } + } + } + + bw.write(SourceFileReader.readTemplateContents(DocgenConsts.HEADER_TEMPLATE, + generateBEHeaderMapping(docEntries))); + + Map<String, String> sectionMapping = ImmutableMap.of( + DocgenConsts.VAR_SECTION_BINARY, getRuleDocs(binaryDocs), + DocgenConsts.VAR_SECTION_LIBRARY, getRuleDocs(libraryDocs), + DocgenConsts.VAR_SECTION_TEST, getRuleDocs(testDocs), + DocgenConsts.VAR_SECTION_GENERATE, getRuleDocs(generateDocs), + DocgenConsts.VAR_SECTION_OTHER, getRuleDocs(otherDocs)); + bw.write(SourceFileReader.readTemplateContents(DocgenConsts.BODY_TEMPLATE, sectionMapping)); + } + + private Map<String, String> generateBEHeaderMapping(Set<RuleDocumentation> docEntries) + throws BuildEncyclopediaDocException { + StringBuilder sb = new StringBuilder(); + + sb.append("<table id=\"rules\" summary=\"Table of rules sorted by language\">\n") + .append("<colgroup span=\"5\" width=\"20%\"></colgroup>\n") + .append("<tr><th>Language</th><th>Binary rules</th><th>Library rules</th>" + + "<th>Test rules</th><th>Other rules</th><th></th></tr>\n"); + + // Separate rule families into language-specific and generic ones. + Set<String> languageSpecificRuleFamilies = new TreeSet<>(); + Set<String> genericRuleFamilies = new TreeSet<>(); + separateRuleFamilies(docEntries, languageSpecificRuleFamilies, genericRuleFamilies); + + // Create a mapping of rules based on rule type and family. + Map<String, ListMultimap<RuleType, RuleDocumentation>> ruleMapping = new HashMap<>(); + createRuleMapping(docEntries, ruleMapping); + + // Generate the table. + for (String ruleFamily : languageSpecificRuleFamilies) { + generateHeaderTableRuleFamily(sb, ruleMapping.get(ruleFamily), ruleFamily); + } + + sb.append("<tr><th> </th></tr>"); + sb.append("<tr><th colspan=\"5\">Rules that do not apply to a " + + "specific programming language</th></tr>"); + for (String ruleFamily : genericRuleFamilies) { + generateHeaderTableRuleFamily(sb, ruleMapping.get(ruleFamily), ruleFamily); + } + sb.append("</table>\n"); + return ImmutableMap.<String, String>of(DocgenConsts.VAR_HEADER_TABLE, sb.toString(), + DocgenConsts.VAR_COMMON_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs( + PredefinedAttributes.COMMON_ATTRIBUTES, DocgenConsts.COMMON_ATTRIBUTES), + DocgenConsts.VAR_TEST_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs( + PredefinedAttributes.TEST_ATTRIBUTES, DocgenConsts.TEST_ATTRIBUTES), + DocgenConsts.VAR_BINARY_ATTRIBUTE_DEFINITION, generateCommonAttributeDocs( + PredefinedAttributes.BINARY_ATTRIBUTES, DocgenConsts.BINARY_ATTRIBUTES), + DocgenConsts.VAR_LEFT_PANEL, generateLeftNavigationPanel(docEntries)); + } + + /** + * Create a mapping of rules based on rule type and family. + */ + private void createRuleMapping(Set<RuleDocumentation> docEntries, + Map<String, ListMultimap<RuleType, RuleDocumentation>> ruleMapping) + throws BuildEncyclopediaDocException { + for (RuleDocumentation ruleDoc : docEntries) { + RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleDoc.getRuleName()); + if (ruleClass != null) { + String ruleFamily = ruleDoc.getRuleFamily(); + if (!ruleMapping.containsKey(ruleFamily)) { + ruleMapping.put(ruleFamily, LinkedListMultimap.<RuleType, RuleDocumentation>create()); + } + if (ruleClass.isDocumented()) { + ruleMapping.get(ruleFamily).put(ruleDoc.getRuleType(), ruleDoc); + } + } else { + throw ruleDoc.createException("Can't find RuleClass for " + ruleDoc.getRuleName()); + } + } + } + + /** + * Separates all rule families in docEntries into language-specific rules and generic rules. + */ + private void separateRuleFamilies(Set<RuleDocumentation> docEntries, + Set<String> languageSpecificRuleFamilies, Set<String> genericRuleFamilies) + throws BuildEncyclopediaDocException { + for (RuleDocumentation ruleDoc : docEntries) { + if (ruleDoc.isLanguageSpecific()) { + if (genericRuleFamilies.contains(ruleDoc.getRuleFamily())) { + throw ruleDoc.createException("The rule is marked as being language-specific, but other " + + "rules of the same family have already been marked as being not."); + } + languageSpecificRuleFamilies.add(ruleDoc.getRuleFamily()); + } else { + if (languageSpecificRuleFamilies.contains(ruleDoc.getRuleFamily())) { + throw ruleDoc.createException("The rule is marked as being generic, but other rules of " + + "the same family have already been marked as being language-specific."); + } + genericRuleFamilies.add(ruleDoc.getRuleFamily()); + } + } + } + + private String generateLeftNavigationPanel(Set<RuleDocumentation> docEntries) { + // Order the rules alphabetically. At this point they are ordered according to + // RuleDocumentation.compareTo() which is not alphabetical. + TreeMap<String, String> ruleNames = new TreeMap<>(); + for (RuleDocumentation ruleDoc : docEntries) { + String ruleName = ruleDoc.getRuleName(); + ruleNames.put(ruleName.toLowerCase(), ruleName); + } + StringBuilder sb = new StringBuilder(); + for (String ruleName : ruleNames.values()) { + RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleName); + Preconditions.checkNotNull(ruleClass); + if (ruleClass.isDocumented()) { + sb.append(String.format("<a href=\"#%s\">%s</a><br/>\n", ruleName, ruleName)); + } + } + return sb.toString(); + } + + private String generateCommonAttributeDocs(Map<String, RuleDocumentationAttribute> attributes, + String attributeGroupName) throws BuildEncyclopediaDocException { + RuleDocumentation ruleDoc = new RuleDocumentation( + attributeGroupName, "OTHER", null, null, 0, null, ImmutableSet.<String>of(), + ruleClassProvider); + for (RuleDocumentationAttribute attribute : attributes.values()) { + ruleDoc.addAttribute(attribute); + } + return ruleDoc.generateAttributeDefinitions(); + } + + private void generateHeaderTableRuleFamily(StringBuilder sb, + ListMultimap<RuleType, RuleDocumentation> ruleTypeMap, String ruleFamily) { + sb.append("<tr>\n") + .append(String.format("<td class=\"lang\">%s</td>\n", ruleFamily)); + boolean otherRulesSplitted = false; + for (RuleType ruleType : DocgenConsts.RuleType.values()) { + sb.append("<td>"); + int i = 0; + List<RuleDocumentation> ruleDocList = ruleTypeMap.get(ruleType); + for (RuleDocumentation ruleDoc : ruleDocList) { + if (i > 0) { + if (ruleType.equals(RuleType.OTHER) + && ruleDocList.size() >= 4 && i == (ruleDocList.size() + 1) / 2) { + // Split 'other rules' into two columns if there are too many of them. + sb.append("</td>\n<td>"); + otherRulesSplitted = true; + } else { + sb.append("<br/>"); + } + } + String ruleName = ruleDoc.getRuleName(); + String deprecatedString = ruleDoc.hasFlag(DocgenConsts.FLAG_DEPRECATED) + ? " class=\"deprecated\"" : ""; + sb.append(String.format("<a href=\"#%s\"%s>%s</a>", ruleName, deprecatedString, ruleName)); + i++; + } + sb.append("</td>\n"); + } + // There should be 6 columns. + if (!otherRulesSplitted) { + sb.append("<td></td>\n"); + } + sb.append("</tr>\n"); + } + + private String getRuleDocs(Iterable<RuleDocumentation> docEntries) { + StringBuilder sb = new StringBuilder(); + for (RuleDocumentation doc : docEntries) { + sb.append(doc.getHtmlDocumentation()); + } + return sb.toString(); + } + + /** + * Goes through all the html files and subdirs under inputPath and collects the rule + * and attribute documentations using the ruleDocEntries and attributeDocEntries variable. + */ + public void collectDocs(Set<RuleDocumentation> ruleDocEntries, + ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries, + File inputPath) throws BuildEncyclopediaDocException, IOException { + if (inputPath.isFile()) { + if (DocgenConsts.JAVA_SOURCE_FILE_SUFFIX.apply(inputPath.getName())) { + SourceFileReader sfr = new SourceFileReader( + ruleClassProvider, inputPath.getAbsolutePath()); + sfr.readDocsFromComments(); + ruleDocEntries.addAll(sfr.getRuleDocEntries()); + if (attributeDocEntries != null) { + // Collect all attribute documentations from this file. + attributeDocEntries.putAll(sfr.getAttributeDocEntries()); + } + } + } else if (inputPath.isDirectory()) { + for (File childPath : inputPath.listFiles()) { + collectDocs(ruleDocEntries, attributeDocEntries, childPath); + } + } + } + + private File setupDirectories(String outputRootDir) { + if (outputRootDir != null) { + File outputRootPath = new File(outputRootDir); + outputRootPath.mkdirs(); + return new File(outputRootDir + File.separator + DocgenConsts.BUILD_ENCYCLOPEDIA_NAME); + } else { + return new File(DocgenConsts.BUILD_ENCYCLOPEDIA_NAME); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/docgen/DocCheckerUtils.java b/src/main/java/com/google/devtools/build/docgen/DocCheckerUtils.java new file mode 100644 index 0000000..3b64362 --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/DocCheckerUtils.java
@@ -0,0 +1,95 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.docgen; + +import com.google.common.collect.ImmutableSet; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A utility class to check the generated documentations. + */ +public class DocCheckerUtils { + + // TODO(bazel-team): remove elements from this list and clean up the tested documentations. + private static final ImmutableSet<String> UNCHECKED_HTML_TAGS = ImmutableSet.<String>of( + "br", "li", "ul", "p"); + + private static final Pattern TAG_PATTERN = Pattern.compile( + "<([/]?[a-z0-9_]+)" + + "([^>]*)" + + ">", + Pattern.CASE_INSENSITIVE); + + private static final Pattern COMMENT_PATTERN = Pattern.compile( + "<!--.*?-->", + Pattern.CASE_INSENSITIVE); + + /** + * Returns the first unmatched html tag of srcs or null if no such tag exists. + * Note that this check is not performed on br, ul, li and p tags. The method also + * prints some help in case an unmatched tag is found. The check is performed + * inside comments too. + */ + public static String getFirstUnclosedTagAndPrintHelp(String src) { + return getFirstUnclosedTag(src, true); + } + + static String getFirstUnclosedTag(String src) { + return getFirstUnclosedTag(src, false); + } + + // TODO(bazel-team): run this on the Skylark docs too. + private static String getFirstUnclosedTag(String src, boolean printHelp) { + Matcher commentMatcher = COMMENT_PATTERN.matcher(src); + src = commentMatcher.replaceAll(""); + Matcher tagMatcher = TAG_PATTERN.matcher(src); + Deque<String> tagStack = new ArrayDeque<>(); + while (tagMatcher.find()) { + String tag = tagMatcher.group(1); + String rest = tagMatcher.group(2); + String strippedTag = tag.substring(1); + + // Ignoring self closing tags. + if (!rest.endsWith("/") + // Ignoring unchecked tags. + && !UNCHECKED_HTML_TAGS.contains(tag) && !UNCHECKED_HTML_TAGS.contains(strippedTag)) { + if (tag.startsWith("/")) { + // Closing tag. Removing '/' from the beginning. + tag = strippedTag; + String lastTag = tagStack.removeLast(); + if (!lastTag.equals(tag)) { + if (printHelp) { + System.err.println( + "Unclosed tag: " + lastTag + "\n" + + "Trying to close with: " + tag + "\n" + + "Stack of open tags: " + tagStack + "\n" + + "Last 200 characters:\n" + + src.substring(Math.max(tagMatcher.start() - 200, 0), tagMatcher.start())); + } + return lastTag; + } + } else { + // Starting tag. + tagStack.addLast(tag); + } + } + } + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/docgen/DocgenConsts.java b/src/main/java/com/google/devtools/build/docgen/DocgenConsts.java new file mode 100644 index 0000000..a2e7583 --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/DocgenConsts.java
@@ -0,0 +1,169 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.docgen; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; + +import java.util.Map; +import java.util.regex.Pattern; + +/** + * All the constants for the Docgen project. + */ +public class DocgenConsts { + + public static final String LS = "\n"; + + public static final String HEADER_TEMPLATE = "templates/be-header.html"; + public static final String FOOTER_TEMPLATE = "templates/be-footer.html"; + public static final String BODY_TEMPLATE = "templates/be-body.html"; + public static final String SKYLARK_BODY_TEMPLATE = "templates/skylark-body.html"; + + public static final String VAR_LEFT_PANEL = "LEFT_PANEL"; + + public static final String VAR_SECTION_BINARY = "SECTION_BINARY"; + public static final String VAR_SECTION_LIBRARY = "SECTION_LIBRARY"; + public static final String VAR_SECTION_TEST = "SECTION_TEST"; + public static final String VAR_SECTION_GENERATE = "SECTION_GENERATE"; + public static final String VAR_SECTION_OTHER = "SECTION_OTHER"; + + public static final String VAR_IMPLICIT_OUTPUTS = "IMPLICIT_OUTPUTS"; + public static final String VAR_ATTRIBUTE_SIGNATURE = "ATTRIBUTE_SIGNATURE"; + public static final String VAR_ATTRIBUTE_DEFINITION = "ATTRIBUTE_DEFINITION"; + public static final String VAR_NAME = "NAME"; + public static final String VAR_HEADER_TABLE = "HEADER_TABLE"; + public static final String VAR_COMMON_ATTRIBUTE_DEFINITION = "COMMON_ATTRIBUTE_DEFINITION"; + public static final String VAR_TEST_ATTRIBUTE_DEFINITION = "TEST_ATTRIBUTE_DEFINITION"; + public static final String VAR_BINARY_ATTRIBUTE_DEFINITION = "BINARY_ATTRIBUTE_DEFINITION"; + public static final String VAR_SYNOPSIS = "SYNOPSIS"; + + public static final String VAR_SECTION_SKYLARK_BUILTIN = "SECTION_BUILTIN"; + + public static final String COMMON_ATTRIBUTES = "common"; + public static final String TEST_ATTRIBUTES = "test"; + public static final String BINARY_ATTRIBUTES = "binary"; + + /** + * Mark the attribute as deprecated in the Build Encyclopedia. + */ + public static final String FLAG_DEPRECATED = "DEPRECATED"; + public static final String FLAG_GENERIC_RULE = "GENERIC_RULE"; + + public static final String HEADER_COMMENT = + "<!DOCTYPE html>\n" + + "<!--\n" + + " This document is synchronized with Blaze releases.\n" + + " To edit, submit changes to the Blaze source code.\n" + + " Generated by: blaze build java/com/google/devtools/build/docgen:build-encyclopedia.html\n" + + "-->\n"; + + public static final String BUILD_ENCYCLOPEDIA_NAME = "build-encyclopedia.html"; + + public static final FileTypeSet JAVA_SOURCE_FILE_SUFFIX = FileTypeSet.of(FileType.of(".java")); + + public static final String META_KEY_NAME = "NAME"; + public static final String META_KEY_TYPE = "TYPE"; + public static final String META_KEY_FAMILY = "FAMILY"; + + /** + * Types a rule can have (Binary, Library, Test or Other). + */ + public static enum RuleType { + BINARY, LIBRARY, TEST, OTHER + } + + /** + * i.e. <!-- #BLAZE_RULE(NAME = RULE_NAME, TYPE = RULE_TYPE, FAMILY = RULE_FAMILY) --> + * i.e. <!-- #BLAZE_RULE(...)[DEPRECATED] --> + */ + public static final Pattern BLAZE_RULE_START = Pattern.compile( + "^[\\s]*/\\*[\\s]*\\<!\\-\\-[\\s]*#BLAZE_RULE[\\s]*\\(([\\w\\s=,+/()-]+)\\)" + + "(\\[[\\w,]+\\])?[\\s]*\\-\\-\\>"); + /** + * i.e. <!-- #END_BLAZE_RULE --> + */ + public static final Pattern BLAZE_RULE_END = Pattern.compile( + "^[\\s]*\\<!\\-\\-[\\s]*#END_BLAZE_RULE[\\s]*\\-\\-\\>[\\s]*\\*/"); + /** + * i.e. <!-- #BLAZE_RULE.EXAMPLE --> + */ + public static final Pattern BLAZE_RULE_EXAMPLE_START = Pattern.compile( + "[\\s]*\\<!--[\\s]*#BLAZE_RULE.EXAMPLE[\\s]*--\\>[\\s]*"); + /** + * i.e. <!-- #BLAZE_RULE.END_EXAMPLE --> + */ + public static final Pattern BLAZE_RULE_EXAMPLE_END = Pattern.compile( + "[\\s]*\\<!--[\\s]*#BLAZE_RULE.END_EXAMPLE[\\s]*--\\>[\\s]*"); + /** + * i.e. <!-- #BLAZE_RULE(RULE_NAME).VARIABLE_NAME --> + */ + public static final Pattern BLAZE_RULE_VAR_START = Pattern.compile( + "^[\\s]*/\\*[\\s]*\\<!\\-\\-[\\s]*#BLAZE_RULE\\(([\\w\\$]+)\\)\\.([\\w]+)[\\s]*\\-\\-\\>"); + /** + * i.e. <!-- #END_BLAZE_RULE.VARIABLE_NAME --> + */ + public static final Pattern BLAZE_RULE_VAR_END = Pattern.compile( + "^[\\s]*\\<!\\-\\-[\\s]*#END_BLAZE_RULE\\.([\\w]+)[\\s]*\\-\\-\\>[\\s]*\\*/"); + /** + * i.e. <!-- #BLAZE_RULE(RULE_NAME).ATTRIBUTE(ATTR_NAME) --> + * i.e. <!-- #BLAZE_RULE(RULE_NAME).ATTRIBUTE(ATTR_NAME)[DEPRECATED] --> + */ + public static final Pattern BLAZE_RULE_ATTR_START = Pattern.compile( + "^[\\s]*/\\*[\\s]*\\<!\\-\\-[\\s]*#BLAZE_RULE\\(([\\w\\$]+)\\)\\." + + "ATTRIBUTE\\(([\\w]+)\\)(\\[[\\w,]+\\])?[\\s]*\\-\\-\\>"); + /** + * i.e. <!-- #END_BLAZE_RULE.ATTRIBUTE --> + */ + public static final Pattern BLAZE_RULE_ATTR_END = Pattern.compile( + "^[\\s]*\\<!\\-\\-[\\s]*#END_BLAZE_RULE\\.ATTRIBUTE[\\s]*\\-\\-\\>[\\s]*\\*/"); + + public static final Pattern BLAZE_RULE_FLAGS = Pattern.compile("^.*\\[(.*)\\].*$"); + + public static final Map<String, Integer> ATTRIBUTE_ORDERING = ImmutableMap + .<String, Integer>builder() + .put("name", -99) + .put("deps", -98) + .put("src", -97) + .put("srcs", -96) + .put("data", -95) + .put("resource", -94) + .put("resources", -93) + .put("out", -92) + .put("outs", -91) + .put("hdrs", -90) + .build(); + + static String toCommandLineFormat(String cmdDoc) { + // Replace html <br> tags with line breaks + cmdDoc = cmdDoc.replaceAll("(<br>|<br[\\s]*/>)", "\n") + "\n"; + // Replace other links <a href=".*">s with human readable links + cmdDoc = cmdDoc.replaceAll("\\<a href=\"([^\"]+)\">[^\\<]*\\</a\\>", "$1"); + // Delete other html tags + cmdDoc = cmdDoc.replaceAll("\\<[/]?[^\\>]+\\>", ""); + // Delete docgen variables + cmdDoc = cmdDoc.replaceAll("\\$\\{[\\w_]+\\}", ""); + // Substitute more than 2 line breaks in a row with 2 line breaks + cmdDoc = cmdDoc.replaceAll("[\\n]{2,}", "\n\n"); + // Ensure that the doc starts and ends with exactly two line breaks + cmdDoc = cmdDoc.replaceAll("^[\\n]+", "\n\n"); + cmdDoc = cmdDoc.replaceAll("[\\n]+$", "\n\n"); + return cmdDoc; + } + + static String removeDuplicatedNewLines(String doc) { + return doc.replaceAll("[\\n][\\s]*[\\n]", "\n"); + } +}
diff --git a/src/main/java/com/google/devtools/build/docgen/PredefinedAttributes.java b/src/main/java/com/google/devtools/build/docgen/PredefinedAttributes.java new file mode 100644 index 0000000..c885cc1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/PredefinedAttributes.java
@@ -0,0 +1,347 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.docgen; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +/** + * A class to contain the base definition of common BUILD rule attributes. + */ +public class PredefinedAttributes { + + public static final Map<String, RuleDocumentationAttribute> COMMON_ATTRIBUTES = ImmutableMap + .<String, RuleDocumentationAttribute>builder() + .put("deps", RuleDocumentationAttribute.create("deps", DocgenConsts.COMMON_ATTRIBUTES, + "A list of dependencies of this rule.\n" + + "<i>(List of <a href=\"build-ref.html#labels\">labels</a>; optional)</i><br/>\n" + + "The precise semantics of what it means for this rule to depend on\n" + + "another using <code>deps</code> are specific to the kind of this rule,\n" + + "and the rule-specific documentation below goes into more detail.\n" + + "At a minimum, though, the targets named via <code>deps</code> will\n" + + "appear in the <code>*.runfiles</code> area of this rule, if it has\n" + + "one.\n" + + "<p>Most often, a <code>deps</code> dependency is used to allow one\n" + + "module to use symbols defined in another module written in the\n" + + "same programming language and separately compiled. Cross-language\n" + + "dependencies are also permitted in many cases: for example,\n" + + "a <code>java_library</code> rule may depend on C++ code in\n" + + "a <code>cc_library</code> rule, by declaring the latter in\n" + + "the <code>deps</code> attribute. See the definition\n" + + "of <a href=\"build-ref.html#deps\">dependencies</a> for more\n" + + "information.</p>\n" + + "<p>Almost all rules permit a <code>deps</code> attribute, but where\n" + + "this attribute is not allowed, this fact is documented under the\n" + + "specific rule.</p>")) + .put("data", RuleDocumentationAttribute.create("data", DocgenConsts.COMMON_ATTRIBUTES, + "The list of files needed by this rule at runtime.\n" + + "<i>(List of <a href=\"build-ref.html#labels\">labels</a>; optional)</i><br/>\n" + + "Targets named in the <code>data</code> attribute will appear in\n" + + "the <code>*.runfiles</code> area of this rule, if it has one. This\n" + + "may include data files needed by a binary or library, or other\n" + + "programs needed by it. See the\n" + + "<a href=\"build-ref.html#data\">data dependencies</a> section for more\n" + + "information about how to depend on and use data files.\n" + + "<p>Almost all rules permit a <code>data</code> attribute, but where\n" + + "this attribute is not allowed, this fact is documented under the\n" + + "specific rule.</p>")) + .put("licenses", RuleDocumentationAttribute.create("licenses", + DocgenConsts.COMMON_ATTRIBUTES, + "<i>(List of strings; optional)</i><br/>\n" + + "A list of license-type strings to be used for this particular build rule.\n" + + "Overrides the <code>BUILD</code>-file scope defaults defined by the\n" + + "<a href=\"#licenses\"><code>licenses()</code></a> directive.")) + .put("distribs", RuleDocumentationAttribute.create("distribs", + DocgenConsts.COMMON_ATTRIBUTES, + "<i>(List of strings; optional)</i><br/>\n" + + "A list of distribution-method strings to be used for this particular build rule.\n" + + "Overrides the <code>BUILD</code>-file scope defaults defined by the\n" + + "<a href=\"#distribs\"><code>distribs()</code></a> directive.")) + .put("deprecation", RuleDocumentationAttribute.create("deprecation", + DocgenConsts.COMMON_ATTRIBUTES, + "<i>(String; optional)</i><br/>\n" + + "An explanatory warning message associated with this rule.\n" + + "Typically this is used to notify users that a rule has become obsolete,\n" + + "or has become superseded by another rule, is private to a package, or is\n" + + "perhaps \"considered harmful\" for some reason. It is a good idea to include\n" + + "some reference (like a webpage, a bug number or example migration CLs) so\n" + + "that one can easily find out what changes are required to avoid the message.\n" + + "If there is a new target that can be used as a drop in replacement, it is a good idea\n" + + "to just migrate all users of the old target.\n" + + "<p>\n" + + "This attribute has no effect on the way things are built, but it\n" + + "may affect a build tool's diagnostic output. The build tool issues a\n" + + "warning when a rule with a <code>deprecation</code> attribute is\n" + + "depended upon by another rule.</p>\n" + + "<p>\n" + + "Intra-package dependencies are exempt from this warning, so that,\n" + + "for example, building the tests of a deprecated rule does not\n" + + "encounter a warning.</p>\n" + + "<p>\n" + + "If a deprecated rule depends on another deprecated rule, no warning\n" + + "message is issued.</p>\n" + + "<p>\n" + + "Once people have stopped using it, the package can be removed or marked as\n" + + "<a href=\"#common.obsolete\"><code>obsolete</code></a>.</p>")) + .put("obsolete", RuleDocumentationAttribute.create("obsolete", + DocgenConsts.COMMON_ATTRIBUTES, + "<i>(Boolean; optional; default 0)</i><br/>\n" + + "If 1, only obsolete targets can depend on this target. It is an error when\n" + + "a non-obsolete target depends on an obsolete target.\n" + + "<p>\n" + + "As a transition, one can first mark a package as in\n" + + "<a href=\"#common.deprecation\"><code>deprecation</code></a>.</p>\n" + + "<p>\n" + + "This attribute is useful when you want to prevent a target from\n" + + "being used but are yet not ready to delete the sources.</p>")) + .put("testonly", RuleDocumentationAttribute.create("testonly", + DocgenConsts.COMMON_ATTRIBUTES, + "<i>(Boolean; optional; default 0 except as noted)</i><br />\n" + + "If 1, only testonly targets (such as tests) can depend on this target.\n" + + "<p>Equivalently, a rule that is not <code>testonly</code> is not allowed to\n" + + "depend on any rule that is <code>testonly</code>.</p>\n" + + "<p>Tests (<code>*_test</code> rules)\n" + + "and test suites (<a href=\"#test_suite\">test_suite</a> rules)\n" + + "are <code>testonly</code> by default.</p>\n" + + "<p>By virtue of\n" + + "<a href=\"#package.default_testonly\"><code>default_testonly</code></a>,\n" + + "targets under <code>javatests</code> are <code>testonly</code> by default.</p>\n" + + "<p>This attribute is intended to mean that the target should not be\n" + + "contained in binaries that are released to production.</p>\n" + + "<p>Because testonly is enforced at build time, not run time, and propagates\n" + + "virally through the dependency tree, it should be applied judiciously. For\n" + + "example, stubs and fakes that\n" + + "are useful for unit tests may also be useful for integration tests\n" + + "involving the same binaries that will be released to production, and\n" + + "therefore should probably not be marked testonly. Conversely, rules that\n" + + "are dangerous to even link in, perhaps because they unconditionally\n" + + "override normal behavior, should definitely be marked testonly.</p>")) + .put("tags", RuleDocumentationAttribute.create("tags", DocgenConsts.COMMON_ATTRIBUTES, + "List of arbitrary text tags. Tags may be any valid string; default is the\n" + + "empty list.<br/>\n" + + "<i>Tags</i> can be used on any rule; but <i>tags</i> are most useful\n" + + "on test and <code>test_suite</code> rules. Tags on non-test rules\n" + + "are only useful to humans and/or external programs.\n" + + "<i>Tags</i> are generally used to annotate a test's role in your debug\n" + + "and release process. Typically, tags are most useful for C++ and\n" + + "Python tests, which\n" + + "lack any runtime annotation ability. The use of tags and size elements\n" + + "gives flexibility in assembling suites of tests based around codebase\n" + + "check-in policy.\n" + + "<p>\n" + + "A few tags have special meaning to the build tool, such as\n" + + "indicating that a particular test cannot be run remotely, for\n" + + "example. Consult\n" + + "the <a href='blaze-user-manual.html#tags_keywords'>Blaze\n" + + "documentation</a> for details.\n" + + "</p>")) + .put("visibility", RuleDocumentationAttribute.create("visibility", + DocgenConsts.COMMON_ATTRIBUTES, + "<i>(List of <a href=\"build-ref.html#labels\">" + + "labels</a>; optional; default private)</i><br/>\n" + + "<p>The <code>visibility</code> attribute on a rule controls whether\n" + + "the rule can be used by other packages. Rules are always visible to\n" + + "other rules declared in the same package.</p>\n" + + "<p>There are five forms (and one temporary form) a visibility label can take:\n" + + "<ul>\n" + + "<li><code>[\"//visibility:public\"]</code>: Anyone can use this rule.</li>\n" + + "<li><code>[\"//visibility:private\"]</code>: Only rules in this package\n" + + "can use this rule. Rules in <code>javatests/foo/bar</code>\n" + + "can always use rules in <code>java/foo/bar</code>.\n" + + "</li>\n" + + "<li><code>[\"//some/package:__pkg__\", \"//other/package:__pkg__\"]</code>:\n" + + "Only rules in <code>some/package</code> and <code>other/package</code>\n" + + "(defined in <code>some/package/BUILD</code> and\n" + + "<code>other/package/BUILD</code>) have access to this rule. Note that\n" + + "sub-packages do not have access to the rule; for example,\n" + + "<code>//some/package/foo:bar</code> or\n" + + "<code>//other/package/testing:bla</code> wouldn't have access.\n" + + "<code>__pkg__</code> is a special target and must be used verbatim.\n" + + "It represents all of the rules in the package.\n" + + "</li>\n" + + "<li><code>[\"//project:__subpackages__\", \"//other:__subpackages__\"]</code>:\n" + + "Only rules in packages <code>project</code> or <code>other</code> or\n" + + "in one of their sub-packages have access to this rule. For example,\n" + + "<code>//project:rule</code>, <code>//project/library:lib</code> or\n" + + "<code>//other/testing/internal:munge</code> are allowed to depend on\n" + + "this rule (but not <code>//independent:evil</code>)\n" + + "</li>\n" + + "<li><code>[\"//some/package:my_package_group\"]</code>:\n" + + "A <a href=\"#package_group\">package group</a> is\n" + + "a named set of package names. Package groups can also grant access rights\n" + + "to entire subtrees, e.g.<code>//myproj/...</code>.\n" + + "</li>\n" + + "<li><code>[\"//visibility:legacy_public\"]</code>: Anyone can use this\n" + + "rule (for now). <i>Developer action is needed</i>.\n" + + "<p>This value has been used during the transition to the new\n" + + "<code>[\"//visibility:private\"]</code> default, on June 6, 2011.\n" + + "<i>We will eventually deprecate and then disallow this value.</i>\n" + + "</li>\n" + + "</ul>\n" + + "<p>The visibility specifications of <code>//visibility:public</code>,\n" + + "<code>//visibility:private</code> and\n" + + "<code>//visibility:legacy_public</code>\n" + + "can not be combined with any other visibility specifications.\n" + + "A visibility specification may contain a combination of package labels\n" + + "(i.e. //foo:__pkg__) and package_groups.</p>\n" + + "<p>If a rule does not specify the visibility attribute,\n" + + "the <code><a href=\"#package\">default_visibility</a></code>\n" + + "attribute of the <code><a href=\"#package\">package</a></code>\n" + + "statement in the BUILD file containing the rule is used\n" + + "(except <a href=\"#exports_files\">exports_files</a> and\n" + + "<a href=\"#cc_public_library\">cc_public_library</a>, which always default to\n" + + "public).</p>\n" + + "<p>If the default visibility for the package is not specified,\n" + + "the rule is private: on June 6, 2011, in order to prevent teams\n" + + "from reaching into private code, the default has been changed\n" + + "to <code>[\"//visibility:private\"]</code>.</p>\n" + + "<p><b>Example</b>:</p>\n" + + "<p>\n" + + "File <code>//frobber/bin/BUILD</code>:\n" + + "</p>\n" + + "<pre class=\"code\">\n" + + "# This rule is visible to everyone\n" + + "py_binary(\n" + + " name = \"executable\",\n" + + " visibility = [\"//visibility:public\"],\n" + + " deps = [\":library\"],\n" + + ")\n" + + "\n" + + "# This rule is visible only to rules declared in the same package\n" + + "py_library(\n" + + " name = \"library\",\n" + + " visibility = [\"//visibility:private\"],\n" + + ")\n" + + "\n" + + "# This rule is visible to rules in package //object and //noun\n" + + "py_library(\n" + + " name = \"subject\",\n" + + " visibility = [\n" + + " \"//noun:__pkg__\",\n" + + " \"//object:__pkg__\",\n" + + " ],\n" + + ")\n" + + "\n" + + "# See package group //frobber:friends (below) for who can access this rule.\n" + + "py_library(\n" + + " name = \"thingy\",\n" + + " visibility = [\"//frobber:friends\"],\n" + + ")\n" + + "</pre>\n" + + "<p>\n" + + "File <code>//frobber/BUILD</code>:\n" + + "</p>\n" + + "<pre class=\"code\">\n" + + "# This is the package group declaration to which rule //frobber/bin:thingy refers.\n" + + "#\n" + + "# Our friends are packages //frobber, //fribber and any subpackage of //fribber.\n" + + "package_group(\n" + + " name = \"friends\",\n" + + " packages = [\n" + + " \"//fribber/...\",\n" + + " \"//frobber\",\n" + + " ],\n" + + ")\n" + + "</pre>")) + .build(); + + public static final Map<String, RuleDocumentationAttribute> BINARY_ATTRIBUTES = ImmutableMap.of( + "args", RuleDocumentationAttribute.create("args", DocgenConsts.BINARY_ATTRIBUTES, + "Add these arguments to the target when executed by\n" + + "<code>blaze run</code>.\n" + + "<i>(List of strings; optional; subject to\n" + + "<a href=\"#make_variables\">\"Make variable\"</a> substitution and\n" + + "<a href=\"#sh-tokenization\">Bourne shell tokenization</a>)</i><br/>\n" + + "These arguments are passed to the target before the target options\n" + + "specified on the <code>blaze run</code> command line.\n" + + "<p>Most binary rules permit an <code>args</code> attribute, but where\n" + + "this attribute is not allowed, this fact is documented under the\n" + + "specific rule.</p>"), + "output_licenses", RuleDocumentationAttribute.create("output_licenses", + DocgenConsts.BINARY_ATTRIBUTES, + "The licenses of the output files that this binary generates.\n" + + "<i>(List of strings; optional)</i><br/>\n" + + "Describes the licenses of the output of the binary generated by\n" + + "the rule. When a binary is referenced in a host attribute (for\n" + + "example, the <code>tools</code> attribute of\n" + + "a <code>genrule</code>), this license declaration is used rather\n" + + "than the union of the licenses of its transitive closure. This\n" + + "argument is useful when a binary is used as a tool during the\n" + + "build of a rule, and it is not desirable for its license to leak\n" + + "into the license of that rule. If this attribute is missing, the\n" + + "license computation proceeds as if the host dependency was a\n" + + "regular dependency.\n" + + "<p>(For more about the distinction between host and target\n" + + "configurations,\n" + + "see <a href=\"blaze-user-manual.html#configurations\">" + + "Build configurations</a> in the Blaze manual.)\n" + + "<p><em class=\"harmful\">WARNING: in some cases (specifically, in\n" + + "genrules) the build tool cannot guarantee that the binary\n" + + "referenced by this attribute is actually used as a tool, and is\n" + + "not, for example, copied to the output. In these cases, it is the\n" + + "responsibility of the user to make sure that this is\n" + + "true.</em></p>")); + + public static final Map<String, RuleDocumentationAttribute> TEST_ATTRIBUTES = ImmutableMap + .<String, RuleDocumentationAttribute>builder() + .put("args", RuleDocumentationAttribute.create("args", DocgenConsts.TEST_ATTRIBUTES, + "Add these arguments to the <code>--test_arg</code>\n" + + "when executed by <code>blaze test</code>.\n" + + "<i>(List of strings; optional; subject to\n" + + "<a href=\"#make_variables\">\"Make variable\"</a> substitution and\n" + + "<a href=\"#sh-tokenization\">Bourne shell tokenization</a>)</i><br/>\n" + + "These arguments are passed before the <code>--test_arg</code> values\n" + + "specified on the <code>blaze test</code> command line.")) + .put("size", RuleDocumentationAttribute.create("size", DocgenConsts.TEST_ATTRIBUTES, + "How \"heavy\" the test is\n" + + "<i>(String \"enormous\", \"large\" \"medium\" or \"small\",\n" + + "default is \"medium\")</i><br/>\n" + + "A classification of the test's \"heaviness\": how much time/resources\n" + + "it needs to run." + + "Unittests are considered \"small\", integration tests \"medium\", " + + "and end-to-end tests \"large\" or \"enormous\". " + + "Blaze uses the size only to determine a default timeout.")) + .put("timeout", RuleDocumentationAttribute.create("timeout", DocgenConsts.TEST_ATTRIBUTES, + "How long the test is\n" + + "normally expected to run before returning.\n" + + "<i>(String \"eternal\", \"long\", \"moderate\", or \"short\"\n" + + "with the default derived from a test's size attribute)</i><br/>\n" + + "While a test's size attribute controls resource estimation, a test's\n" + + "timeout may be set independently. If not explicitly specified, the\n" + + "timeout is based on the test's size (with \"small\" ⇒ \"short\",\n" + + "\"medium\" ⇒ \"moderate\", etc...). " + + "\"short\" means 1 minute, \"moderate\" 5 minutes, and \"long\" 15 minutes.")) + .put("flaky", RuleDocumentationAttribute.create("flaky", DocgenConsts.TEST_ATTRIBUTES, + "Marks test as flaky. <i>(Boolean; optional)</i><br/>\n" + + "If set, executes the test up to 3 times before being declared as failed.\n" + + "By default this attribute is set to 0 and test is considered to be stable.\n" + + "Note, that use of this attribute is generally discouraged - we do prefer\n" + + "all tests to be stable.")) + .put("shard_count", RuleDocumentationAttribute.create("shard_count", + DocgenConsts.TEST_ATTRIBUTES, + "Specifies the number of parallel shards\n" + + "to use to run the test. <i>(Non-negative integer less than or equal to 50;\n" + + "optional)</i><br/>\n" + + "This value will override any heuristics used to determine the number of\n" + + "parallel shards with which to run the test. Note that for some test\n" + + "rules, this parameter may be required to enable sharding\n" + + "in the first place. Also see --test_sharding_strategy.")) + .put("local", RuleDocumentationAttribute.create("local", DocgenConsts.TEST_ATTRIBUTES, + "Forces the test to be run locally. <i>(Boolean; optional)</i><br/>\n" + + "By default this attribute is set to 0 and the default testing strategy is\n" + + "used. This is equivalent to providing \"local\" as a tag\n" + + "(<code>tags=[\"local\"]</code>).")) + .build(); +}
diff --git a/src/main/java/com/google/devtools/build/docgen/RuleDocumentation.java b/src/main/java/com/google/devtools/build/docgen/RuleDocumentation.java new file mode 100644 index 0000000..e8835f4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/RuleDocumentation.java
@@ -0,0 +1,353 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.docgen; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.docgen.DocgenConsts.RuleType; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.RuleClass; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; + +/** + * A class representing the documentation of a rule along with some meta-data. The sole ruleName + * field is used as a key for comparison, equals and hashcode. + * + * <p> The class contains meta information about the rule: + * <ul> + * <li> Rule type: categorizes the rule based on it's general (language independent) purpose, + * see {@link RuleType}. + * <li> Rule family: categorizes the rule based on language. + * </ul> + * + * <p> The class also contains physical information about the documentation, + * such as declaring file name and the first line of the raw documentation. This can be useful for + * proper error signaling during documentation processing. + */ +class RuleDocumentation implements Comparable<RuleDocumentation> { + + private final String ruleName; + private final RuleType ruleType; + private final String ruleFamily; + private final String htmlDocumentation; + // Store these information for error messages + private final int startLineCount; + private final String fileName; + private final ImmutableSet<String> flags; + + private final Map<String, String> docVariables = new HashMap<>(); + // Only one attribute per attributeName is allowed + private final Set<RuleDocumentationAttribute> attributes = new TreeSet<>(); + private final ConfiguredRuleClassProvider ruleClassProvider; + + /** + * Creates a RuleDocumentation from the rule's name, type, family and raw html documentation + * (meaning without expanding the variables in the doc). + */ + RuleDocumentation(String ruleName, String ruleType, String ruleFamily, + String htmlDocumentation, int startLineCount, String fileName, ImmutableSet<String> flags, + ConfiguredRuleClassProvider ruleClassProvider) + throws BuildEncyclopediaDocException { + Preconditions.checkNotNull(ruleName); + this.ruleName = ruleName; + try { + this.ruleType = RuleType.valueOf(ruleType); + } catch (IllegalArgumentException e) { + throw new BuildEncyclopediaDocException( + fileName, startLineCount, "Invalid rule type " + ruleType); + } + this.ruleFamily = ruleFamily; + this.htmlDocumentation = htmlDocumentation; + this.startLineCount = startLineCount; + this.fileName = fileName; + this.flags = flags; + this.ruleClassProvider = ruleClassProvider; + } + + /** + * Returns the name of the rule. + */ + String getRuleName() { + return ruleName; + } + + /** + * Returns the type of the rule + */ + RuleType getRuleType() { + return ruleType; + } + + /** + * Returns the family of the rule. The family is usually the corresponding programming language, + * except for rules independent of language, such as genrule. E.g. the family of the java_library + * rule is 'JAVA', the family of genrule is 'GENERAL'. + */ + String getRuleFamily() { + return ruleFamily; + } + + /** + * Returns the number of first line of the rule documentation in its declaration file. + */ + int getStartLineCount() { + return startLineCount; + } + + /** + * Returns true if this rule documentation has the parameter flag. + */ + boolean hasFlag(String flag) { + return flags.contains(flag); + } + + /** + * Returns true if this rule applies to a specific programming language (e.g. java_library), + * returns false if it is a generic action (e.g. genrule, filegroup). + * + * A rule is considered to be specific to a programming language by default. Generic rules have + * to be marked with the flag GENERIC_RULE in their #BLAZE_RULE definition. + */ + boolean isLanguageSpecific() { + return !flags.contains(DocgenConsts.FLAG_GENERIC_RULE); + } + + /** + * Adds a variable name - value pair to the documentation to be substituted. + */ + void addDocVariable(String varName, String value) { + docVariables.put(varName, value); + } + + /** + * Adds a rule documentation attribute to this rule documentation. + */ + void addAttribute(RuleDocumentationAttribute attribute) { + attributes.add(attribute); + } + + /** + * Returns the html documentation in the exact format it should be written into the Build + * Encyclopedia (expanding variables). + */ + String getHtmlDocumentation() { + String expandedDoc = htmlDocumentation; + // Substituting variables + for (Entry<String, String> docVariable : docVariables.entrySet()) { + expandedDoc = expandedDoc.replace("${" + docVariable.getKey() + "}", + expandBuiltInVariables(docVariable.getKey(), docVariable.getValue())); + } + expandedDoc = expandedDoc.replace("${" + DocgenConsts.VAR_ATTRIBUTE_SIGNATURE + "}", + generateAttributeSignatures()); + expandedDoc = expandedDoc.replace("${" + DocgenConsts.VAR_ATTRIBUTE_DEFINITION + "}", + generateAttributeDefinitions(true)); + return String.format("<h3 id=\"%s\"%s>%s</h3>\n\n%s", ruleName, + getDeprecatedString(hasFlag(DocgenConsts.FLAG_DEPRECATED)), ruleName, expandedDoc); + } + + /** + * Returns the documentation of the rule in a form which is printable on the command line. + */ + String getCommandLineDocumentation() { + return "\n" + DocgenConsts.toCommandLineFormat(htmlDocumentation); + } + + /** + * Returns the html code of the attribute definitions without the header and name + * attribute of the rule. + */ + String generateAttributeDefinitions() { + return generateAttributeDefinitions(false); + } + + private String generateAttributeDefinitions(boolean generateNameAndHeader) { + StringBuilder sb = new StringBuilder(); + if (generateNameAndHeader){ + String nameExtraHtmlDoc = docVariables.containsKey(DocgenConsts.VAR_NAME) + ? docVariables.get(DocgenConsts.VAR_NAME) : ""; + sb.append(String.format(Joiner.on('\n').join(new String[] { + "<h4 id=\"%s_args\">Arguments</h4>", + "<ul>", + "<li id=\"%s.name\"><code>name</code>: A unique name for this rule.", + "<i>(<a href=\"build-ref.html#name\">Name</a>; required)</i>%s</li>\n"}), + ruleName, ruleName, nameExtraHtmlDoc)); + } else { + sb.append("<ul>\n"); + } + for (RuleDocumentationAttribute attributeDoc : attributes) { + // Only generate attribute documentation here if the rule and the attribute is + // either both user defined or built in (of common type). + if (isCommonType() == attributeDoc.isCommonType()) { + String attrName = attributeDoc.getAttributeName(); + Attribute attribute = isCommonType() ? null + : ruleClassProvider.getRuleClassMap().get(ruleName).getAttributeByName(attrName); + sb.append(String.format("<li id=\"%s.%s\"%s><code>%s</code>:\n%s</li>\n", + ruleName.toLowerCase(), attrName, getDeprecatedString( + attributeDoc.hasFlag(DocgenConsts.FLAG_DEPRECATED)), + attrName, attributeDoc.getHtmlDocumentation(attribute))); + } + } + sb.append("</ul>\n"); + RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(ruleName); + if (ruleClass != null && ruleClass.isPublicByDefault()) { + sb.append( + "The default visibility is public: <code>visibility = [\"//visibility:public\"]</code>."); + } + return sb.toString(); + } + + private String getDeprecatedString(boolean deprecated) { + return deprecated ? " class=\"deprecated\"" : ""; + } + + private String generateAttributeSignatures() { + StringBuilder sb = new StringBuilder(); + sb.append(String.format( + "<p class=\"rule-signature\">\n%s(<a href=\"#%s.name\">name</a>,\n", + ruleName, ruleName)); + int i = 0; + for (RuleDocumentationAttribute attributeDoc : attributes) { + String attrName = attributeDoc.getAttributeName(); + // Generate the link for the attribute documentation + sb.append(String.format("<a href=\"#%s.%s\">%s</a>", + attributeDoc.getGeneratedInRule(ruleName).toLowerCase(), attrName, attrName)); + if (i < attributes.size() - 1) { + sb.append(","); + } else { + sb.append(")"); + } + sb.append("\n"); + i++; + } + sb.append("</p>\n"); + return sb.toString(); + } + + private String expandBuiltInVariables(String key, String value) { + // Some built in BLAZE variables need special handling, e.g. adding headers + switch (key) { + case DocgenConsts.VAR_IMPLICIT_OUTPUTS: + return String.format("<h4 id=\"%s_implicit_outputs\">Implicit output targets</h4>\n%s", + ruleName.toLowerCase(), value); + default: + return value; + } + } + + /** + * Returns a set of examples based on markups which can be used as BUILD file + * contents for testing. + */ + Set<String> extractExamples() throws BuildEncyclopediaDocException { + String[] lines = htmlDocumentation.split(DocgenConsts.LS); + Set<String> examples = new HashSet<>(); + StringBuilder sb = null; + boolean inExampleCode = false; + int lineCount = 0; + for (String line : lines) { + if (!inExampleCode) { + if (DocgenConsts.BLAZE_RULE_EXAMPLE_START.matcher(line).matches()) { + inExampleCode = true; + sb = new StringBuilder(); + } else if (DocgenConsts.BLAZE_RULE_EXAMPLE_END.matcher(line).matches()) { + throw new BuildEncyclopediaDocException(fileName, startLineCount + lineCount, + "No matching start example tag (#BLAZE_RULE.EXAMPLE) for end example tag."); + } + } else { + if (DocgenConsts.BLAZE_RULE_EXAMPLE_END.matcher(line).matches()) { + inExampleCode = false; + examples.add(sb.toString()); + sb = null; + } else if (DocgenConsts.BLAZE_RULE_EXAMPLE_START.matcher(line).matches()) { + throw new BuildEncyclopediaDocException(fileName, startLineCount + lineCount, + "No start example tags (#BLAZE_RULE.EXAMPLE) in a row."); + } else { + sb.append(line + DocgenConsts.LS); + } + } + lineCount++; + } + return examples; + } + + /** + * Return true if the rule doesn't belong to a specific rule family. + */ + private boolean isCommonType() { + return ruleFamily == null; + } + + /** + * Creates a BuildEncyclopediaDocException with the file containing this rule doc and + * the number of the first line (where the rule doc is defined). Can be used to create + * general BuildEncyclopediaDocExceptions about this rule. + */ + BuildEncyclopediaDocException createException(String msg) { + return new BuildEncyclopediaDocException(fileName, startLineCount, msg); + } + + @Override + public int hashCode() { + return ruleName.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof RuleDocumentation)) { + return false; + } + return ruleName.equals(((RuleDocumentation) obj).ruleName); + } + + private int getTypePriority() { + switch (ruleType) { + case BINARY: + return 1; + case LIBRARY: + return 2; + case TEST: + return 3; + case OTHER: + return 4; + } + throw new IllegalArgumentException("Illegal value of ruleType: " + ruleType); + } + + @Override + public int compareTo(RuleDocumentation o) { + if (this.getTypePriority() < o.getTypePriority()) { + return -1; + } else if (this.getTypePriority() > o.getTypePriority()) { + return 1; + } else { + return this.ruleName.compareTo(o.ruleName); + } + } + + @Override + public String toString() { + return String.format("%s (TYPE = %s, FAMILY = %s)", ruleName, ruleType, ruleFamily); + } +}
diff --git a/src/main/java/com/google/devtools/build/docgen/RuleDocumentationAttribute.java b/src/main/java/com/google/devtools/build/docgen/RuleDocumentationAttribute.java new file mode 100644 index 0000000..0bdc1f1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/RuleDocumentationAttribute.java
@@ -0,0 +1,248 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.docgen; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.TriState; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.Set; + +/** + * A class storing a rule attribute documentation along with some meta information. + * The class provides functionality to compute the ancestry level of this attribute's + * generator rule definition class compared to other rule definition classes. + * + * <p>Warning, two RuleDocumentationAttribute objects are equal based on only the attributeName. + */ +class RuleDocumentationAttribute implements Comparable<RuleDocumentationAttribute> { + + private static final Map<Type<?>, String> TYPE_DESC = ImmutableMap.<Type<?>, String>builder() + .put(Type.BOOLEAN, "Boolean") + .put(Type.INTEGER, "Integer") + .put(Type.INTEGER_LIST, "List of Integer") + .put(Type.STRING, "String") + .put(Type.STRING_LIST, "List of String") + .put(Type.TRISTATE, "Integer") + .put(Type.LABEL, "<a href=\"build-ref.html#labels\">Label</a>") + .put(Type.LABEL_LIST, "List of <a href=\"build-ref.html#labels\">labels</a>") + .put(Type.NODEP_LABEL, "<a href=\"build-ref.html#name\">Name</a>") + .put(Type.NODEP_LABEL_LIST, "List of <a href=\"build-ref.html#name\">names</a>") + .put(Type.OUTPUT, "<a href=\"build-ref.html#filename\">Filename</a>") + .put(Type.OUTPUT_LIST, "List of <a href=\"build-ref.html#filename\">filenames</a>") + .build(); + + private final Class<? extends RuleDefinition> definitionClass; + private final String attributeName; + private final String htmlDocumentation; + private final String commonType; + private int startLineCnt; + private Set<String> flags; + + /** + * Creates common RuleDocumentationAttribute such as deps or data. + * These attribute docs have no definitionClass or htmlDocumentation (it's in the BE header). + */ + static RuleDocumentationAttribute create( + String attributeName, String commonType, String htmlDocumentation) { + RuleDocumentationAttribute docAttribute = new RuleDocumentationAttribute( + null, attributeName, htmlDocumentation, 0, ImmutableSet.<String>of(), commonType); + return docAttribute; + } + + /** + * Creates a RuleDocumentationAttribute with all the necessary fields for explicitly + * defined rule attributes. + */ + static RuleDocumentationAttribute create(Class<? extends RuleDefinition> definitionClass, + String attributeName, String htmlDocumentation, int startLineCnt, Set<String> flags) { + return new RuleDocumentationAttribute(definitionClass, attributeName, htmlDocumentation, + startLineCnt, flags, null); + } + + private RuleDocumentationAttribute(Class<? extends RuleDefinition> definitionClass, + String attributeName, String htmlDocumentation, int startLineCnt, Set<String> flags, + String commonType) { + Preconditions.checkNotNull(attributeName, "AttributeName must not be null."); + this.definitionClass = definitionClass; + this.attributeName = attributeName; + this.htmlDocumentation = htmlDocumentation; + this.startLineCnt = startLineCnt; + this.flags = flags; + this.commonType = commonType; + } + + /** + * Returns the name of the rule attribute. + */ + String getAttributeName() { + return attributeName; + } + + /** + * Returns the raw html documentation of the rule attribute. + */ + String getHtmlDocumentation(Attribute attribute) { + // TODO(bazel-team): this is needed for common type attributes. Fix those and remove this. + if (attribute == null) { + return htmlDocumentation; + } + StringBuilder sb = new StringBuilder() + .append("<i>(") + .append(TYPE_DESC.get(attribute.getType())) + .append("; " + (attribute.isMandatory() ? "required" : "optional")) + .append(getDefaultValue(attribute)) + .append(")</i><br/>\n"); + return htmlDocumentation.replace("${" + DocgenConsts.VAR_SYNOPSIS + "}", sb.toString()); + } + + private String getDefaultValue(Attribute attribute) { + String prefix = "; default is "; + Object value = attribute.getDefaultValueForTesting(); + if (value instanceof Boolean) { + return prefix + ((Boolean) value ? "1" : "0"); + } else if (value instanceof Integer) { + return prefix + String.valueOf(value); + } else if (value instanceof String && !(((String) value).isEmpty())) { + return prefix + "\"" + value + "\""; + } else if (value instanceof TriState) { + switch((TriState) value) { + case AUTO: + return prefix + "-1"; + case NO: + return prefix + "0"; + case YES: + return prefix + "1"; + } + } else if (value instanceof Label) { + return prefix + "<code>" + value + "</code>"; + } + return ""; + } + + /** + * Returns the number of first line of the attribute documentation in its declaration file. + */ + int getStartLineCnt() { + return startLineCnt; + } + + /** + * Returns true if the attribute doc is of a common attribute type. + */ + boolean isCommonType() { + return commonType != null; + } + + /** + * Returns the common attribute type if this attribute doc is of a common type + * otherwise actualRule. + */ + String getGeneratedInRule(String actualRule) { + return isCommonType() ? commonType : actualRule; + } + + /** + * Returns true if this attribute documentation has the parameter flag. + */ + boolean hasFlag(String flag) { + return flags.contains(flag); + } + + /** + * Returns the length of a shortest path from usingClass to the definitionClass of this + * RuleDocumentationAttribute in the rule definition ancestry graph. Returns -1 + * if definitionClass is not the ancestor (transitively) of usingClass. + */ + int getDefinitionClassAncestryLevel(Class<? extends RuleDefinition> usingClass) { + if (usingClass.equals(definitionClass)) { + return 0; + } + // Storing nodes (rule class definitions) with the length of the shortest path from usingClass + Map<Class<? extends RuleDefinition>, Integer> visited = new HashMap<>(); + LinkedList<Class<? extends RuleDefinition>> toVisit = new LinkedList<>(); + visited.put(usingClass, 0); + toVisit.add(usingClass); + // Searching the shortest path from usingClass to this.definitionClass using BFS + do { + Class<? extends RuleDefinition> ancestor = toVisit.removeFirst(); + visitAncestor(ancestor, visited, toVisit); + if (ancestor.equals(definitionClass)) { + return visited.get(ancestor); + } + } while (!toVisit.isEmpty()); + return -1; + } + + private void visitAncestor( + Class<? extends RuleDefinition> usingClass, + Map<Class<? extends RuleDefinition>, Integer> visited, + LinkedList<Class<? extends RuleDefinition>> toVisit) { + BlazeRule ann = usingClass.getAnnotation(BlazeRule.class); + if (ann != null) { + for (Class<? extends RuleDefinition> ancestor : ann.ancestors()) { + if (!visited.containsKey(ancestor)) { + toVisit.addLast(ancestor); + visited.put(ancestor, visited.get(usingClass) + 1); + } + } + } + } + + private int getAttributeOrderingPriority(RuleDocumentationAttribute attribute) { + if (DocgenConsts.ATTRIBUTE_ORDERING.containsKey(attribute.attributeName)) { + return DocgenConsts.ATTRIBUTE_ORDERING.get(attribute.attributeName); + } else { + return 0; + } + } + + @Override + public int compareTo(RuleDocumentationAttribute o) { + int thisPriority = getAttributeOrderingPriority(this); + int otherPriority = getAttributeOrderingPriority(o); + if (thisPriority > otherPriority) { + return 1; + } else if (thisPriority < otherPriority) { + return -1; + } else { + return this.attributeName.compareTo(o.attributeName); + } + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof RuleDocumentationAttribute)) { + return false; + } + return attributeName.equals(((RuleDocumentationAttribute) obj).attributeName); + } + + @Override + public int hashCode() { + return attributeName.hashCode(); + } +}
diff --git a/src/main/java/com/google/devtools/build/docgen/RuleDocumentationVariable.java b/src/main/java/com/google/devtools/build/docgen/RuleDocumentationVariable.java new file mode 100644 index 0000000..21cf9ec --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/RuleDocumentationVariable.java
@@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.docgen; + +/** + * Rule documentation variables for modular rule documentation, e.g. + * separate section for Implicit outputs. + */ +public class RuleDocumentationVariable { + + private String ruleName; + private String variableName; + private String value; + private int startLineCnt; + + public RuleDocumentationVariable( + String ruleName, String variableName, String value, int startLineCnt) { + this.ruleName = ruleName; + this.variableName = variableName; + this.value = value; + this.startLineCnt = startLineCnt; + } + + public String getRuleName() { + return ruleName; + } + + public String getVariableName() { + return variableName; + } + + public String getValue() { + return value; + } + + public int getStartLineCnt() { + return startLineCnt; + } +}
diff --git a/src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationGenerator.java b/src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationGenerator.java new file mode 100644 index 0000000..c4f8cf3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationGenerator.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.docgen; + + +/** + * The main class for the skylark documentation generator. + */ +public class SkylarkDocumentationGenerator { + + private static boolean checkArgs(String[] args) { + if (args.length < 1) { + System.err.println("There has to be one input parameter\n" + + " - an output file."); + return false; + } + return true; + } + + private static void fail(Throwable e, boolean printStackTrace) { + System.err.println("ERROR: " + e.getMessage()); + if (printStackTrace) { + e.printStackTrace(); + } + Runtime.getRuntime().exit(1); + } + + public static void main(String[] args) { + if (checkArgs(args)) { + System.out.println("Generating Skylark documentation..."); + SkylarkDocumentationProcessor processor = new SkylarkDocumentationProcessor(); + try { + processor.generateDocumentation(args[0]); + } catch (Throwable e) { + fail(e, true); + } + System.out.println("Finished."); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationProcessor.java b/src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationProcessor.java new file mode 100644 index 0000000..d2208ea --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/SkylarkDocumentationProcessor.java
@@ -0,0 +1,437 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.docgen; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.docgen.SkylarkJavaInterfaceExplorer.SkylarkBuiltinMethod; +import com.google.devtools.build.docgen.SkylarkJavaInterfaceExplorer.SkylarkJavaMethod; +import com.google.devtools.build.docgen.SkylarkJavaInterfaceExplorer.SkylarkModuleDoc; +import com.google.devtools.build.lib.packages.MethodLibrary; +import com.google.devtools.build.lib.rules.SkylarkModules; +import com.google.devtools.build.lib.rules.SkylarkRuleContext; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.Environment.NoneType; +import com.google.devtools.build.lib.syntax.EvalUtils; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + +/** + * A class to assemble documentation for Skylark. + */ +public class SkylarkDocumentationProcessor { + + private static final String TOP_LEVEL_ID = "_top_level"; + + private static final boolean USE_TEMPLATE = false; + + @SkylarkModule(name = "Global objects, functions and modules", + doc = "Objects, functions and modules registered in the global environment.") + private static final class TopLevelModule {} + + static SkylarkModule getTopLevelModule() { + return TopLevelModule.class.getAnnotation(SkylarkModule.class); + } + + /** + * Generates the Skylark documentation to the given output directory. + */ + public void generateDocumentation(String outputPath) throws IOException, + BuildEncyclopediaDocException { + BufferedWriter bw = null; + File skylarkDocPath = new File(outputPath); + try { + bw = new BufferedWriter(new FileWriter(skylarkDocPath)); + if (USE_TEMPLATE) { + bw.write(SourceFileReader.readTemplateContents(DocgenConsts.SKYLARK_BODY_TEMPLATE, + ImmutableMap.<String, String>of( + DocgenConsts.VAR_SECTION_SKYLARK_BUILTIN, generateAllBuiltinDoc()))); + } else { + bw.write(generateAllBuiltinDoc()); + } + System.out.println("Skylark documentation generated: " + skylarkDocPath.getAbsolutePath()); + } finally { + if (bw != null) { + bw.close(); + } + } + } + + @VisibleForTesting + Map<String, SkylarkModuleDoc> collectModules() { + Map<String, SkylarkModuleDoc> modules = new TreeMap<>(); + Map<String, SkylarkModuleDoc> builtinModules = collectBuiltinModules(); + Map<SkylarkModule, Class<?>> builtinJavaObjects = collectBuiltinJavaObjects(); + + modules.putAll(builtinModules); + SkylarkJavaInterfaceExplorer explorer = new SkylarkJavaInterfaceExplorer(); + for (SkylarkModuleDoc builtinObject : builtinModules.values()) { + // Check the return type for built-in functions, it can be a module previously not added. + for (SkylarkBuiltinMethod builtinMethod : builtinObject.getBuiltinMethods().values()) { + Class<?> type = builtinMethod.annotation.returnType(); + if (type.isAnnotationPresent(SkylarkModule.class)) { + explorer.collect(type.getAnnotation(SkylarkModule.class), type, modules); + } + } + explorer.collect(builtinObject.getAnnotation(), builtinObject.getClassObject(), modules); + } + for (Entry<SkylarkModule, Class<?>> builtinModule : builtinJavaObjects.entrySet()) { + explorer.collect(builtinModule.getKey(), builtinModule.getValue(), modules); + } + return modules; + } + + private String generateAllBuiltinDoc() { + Map<String, SkylarkModuleDoc> modules = collectModules(); + + StringBuilder sb = new StringBuilder(); + // Generate the top level module first in the doc + SkylarkModuleDoc topLevelModule = modules.remove(getTopLevelModule().name()); + generateModuleDoc(topLevelModule, sb); + for (SkylarkModuleDoc module : modules.values()) { + if (!module.getAnnotation().hidden()) { + sb.append("<hr>"); + generateModuleDoc(module, sb); + } + } + return sb.toString(); + } + + private void generateModuleDoc(SkylarkModuleDoc module, StringBuilder sb) { + SkylarkModule annotation = module.getAnnotation(); + sb.append(String.format("<h2 id=\"modules.%s\">%s</h2>\n", + getModuleId(annotation), + annotation.name())) + .append(annotation.doc()) + .append("\n"); + sb.append("<ul>"); + // Sort Java and SkylarkBuiltin methods together. The map key is only used for sorting. + TreeMap<String, Object> methodMap = new TreeMap<>(); + for (SkylarkJavaMethod method : module.getJavaMethods()) { + methodMap.put(method.name + method.method.getParameterTypes().length, method); + } + for (SkylarkBuiltinMethod builtin : module.getBuiltinMethods().values()) { + methodMap.put(builtin.annotation.name(), builtin); + } + for (Object object : methodMap.values()) { + if (object instanceof SkylarkJavaMethod) { + SkylarkJavaMethod method = (SkylarkJavaMethod) object; + generateDirectJavaMethodDoc(annotation.name(), method.name, method.method, + method.callable, sb); + } + if (object instanceof SkylarkBuiltinMethod) { + generateBuiltinItemDoc(getModuleId(annotation), (SkylarkBuiltinMethod) object, sb); + } + } + sb.append("</ul>"); + } + + private String getModuleId(SkylarkModule annotation) { + if (annotation == getTopLevelModule()) { + return TOP_LEVEL_ID; + } else { + return annotation.name(); + } + } + + private void generateBuiltinItemDoc( + String moduleId, SkylarkBuiltinMethod method, StringBuilder sb) { + SkylarkBuiltin annotation = method.annotation; + if (annotation.hidden()) { + return; + } + sb.append(String.format("<li><h3 id=\"modules.%s.%s\">%s</h3>\n", + moduleId, + annotation.name(), + annotation.name())); + + if (com.google.devtools.build.lib.syntax.Function.class.isAssignableFrom(method.fieldClass)) { + sb.append(getSignature(moduleId, annotation)); + } else { + if (!annotation.returnType().equals(Object.class)) { + sb.append("<code>" + getTypeAnchor(annotation.returnType()) + "</code><br>"); + } + } + + sb.append(annotation.doc() + "\n"); + printParams(moduleId, annotation, sb); + } + + private void printParams(String moduleId, SkylarkBuiltin annotation, StringBuilder sb) { + if (annotation.mandatoryParams().length + annotation.optionalParams().length > 0) { + sb.append("<h4>Parameters</h4>\n"); + printParams(moduleId, annotation.name(), annotation.mandatoryParams(), sb); + printParams(moduleId, annotation.name(), annotation.optionalParams(), sb); + } else { + sb.append("<br/>\n"); + } + } + + private void generateDirectJavaMethodDoc(String objectName, String methodName, + Method method, SkylarkCallable annotation, StringBuilder sb) { + if (annotation.hidden()) { + return; + } + + sb.append(String.format("<li><h3 id=\"modules.%s.%s\">%s</h3>\n%s\n", + objectName, + methodName, + methodName, + getSignature(objectName, methodName, method))) + .append(annotation.doc()) + .append(getReturnTypeExtraMessage(annotation)) + .append("\n"); + } + + private String getReturnTypeExtraMessage(SkylarkCallable annotation) { + if (annotation.allowReturnNones()) { + return " May return <code>None</code>.\n"; + } + return ""; + } + + private String getSignature(String objectName, String methodName, Method method) { + String args = method.getAnnotation(SkylarkCallable.class).structField() + ? "" : "(" + getParameterString(method) + ")"; + + return String.format("<code>%s %s.%s%s</code><br>", + getTypeAnchor(method.getReturnType()), objectName, methodName, args); + } + + private String getSignature(String objectName, SkylarkBuiltin method) { + List<String> argList = new ArrayList<>(); + for (Param param : method.mandatoryParams()) { + argList.add(param.name()); + } + for (Param param : method.optionalParams()) { + argList.add(param.name() + "?"); + } + String args = "(" + Joiner.on(", ").join(argList) + ")"; + if (!objectName.equals(TOP_LEVEL_ID)) { + return String.format("<code>%s %s.%s%s</code><br>\n", + getTypeAnchor(method.returnType()), objectName, method.name(), args); + } else { + return String.format("<code>%s %s%s</code><br>\n", + getTypeAnchor(method.returnType()), method.name(), args); + } + } + + private String getTypeAnchor(Class<?> returnType, Class<?> generic1) { + return getTypeAnchor(returnType) + " of " + getTypeAnchor(generic1) + "s"; + } + + private String getTypeAnchor(Class<?> returnType) { + if (returnType.equals(String.class)) { + return "<a class=\"anchor\" href=\"#modules.string\">string</a>"; + } else if (Map.class.isAssignableFrom(returnType)) { + return "<a class=\"anchor\" href=\"#modules.dict\">dict</a>"; + } else if (returnType.equals(Void.TYPE) || returnType.equals(NoneType.class)) { + return "<a class=\"anchor\" href=\"#modules." + TOP_LEVEL_ID + ".None\">None</a>"; + } else if (returnType.isAnnotationPresent(SkylarkModule.class)) { + // TODO(bazel-team): this can produce dead links for types don't show up in the doc. + // The correct fix is to generate those types (e.g. SkylarkFileType) too. + String module = returnType.getAnnotation(SkylarkModule.class).name(); + return "<a class=\"anchor\" href=\"#modules." + module + "\">" + module + "</a>"; + } else { + return EvalUtils.getDataTypeNameFromClass(returnType); + } + } + + private String getParameterString(Method method) { + return Joiner.on(", ").join(Iterables.transform( + ImmutableList.copyOf(method.getParameterTypes()), new Function<Class<?>, String>() { + @Override + public String apply(Class<?> input) { + return getTypeAnchor(input); + } + })); + } + + private void printParams(String moduleId, String methodName, + Param[] params, StringBuilder sb) { + if (params.length > 0) { + sb.append("<ul>\n"); + for (Param param : params) { + String paramType = param.type().equals(Object.class) ? "" + : (param.generic1().equals(Object.class) + ? " (" + getTypeAnchor(param.type()) + ")" + : " (" + getTypeAnchor(param.type(), param.generic1()) + ")"); + sb.append(String.format("\t<li id=\"modules.%s.%s.%s\"><code>%s%s</code>: ", + moduleId, + methodName, + param.name(), + param.name(), + paramType)) + .append(param.doc()) + .append("\n\t</li>\n"); + } + sb.append("</ul>\n"); + } + } + + private Map<String, SkylarkModuleDoc> collectBuiltinModules() { + Map<String, SkylarkModuleDoc> modules = new HashMap<>(); + collectBuiltinDoc(modules, Environment.class.getDeclaredFields()); + collectBuiltinDoc(modules, MethodLibrary.class.getDeclaredFields()); + for (Class<?> moduleClass : SkylarkModules.MODULES) { + collectBuiltinDoc(modules, moduleClass.getDeclaredFields()); + } + return modules; + } + + private Map<SkylarkModule, Class<?>> collectBuiltinJavaObjects() { + Map<SkylarkModule, Class<?>> modules = new HashMap<>(); + collectBuiltinModule(modules, SkylarkRuleContext.class); + return modules; + } + + /** + * Returns the top level modules and functions with their documentation in a command-line + * printable format. + */ + public Map<String, String> collectTopLevelModules() { + Map<String, String> modules = new TreeMap<>(); + for (SkylarkModuleDoc doc : collectBuiltinModules().values()) { + if (doc.getAnnotation() == getTopLevelModule()) { + for (Map.Entry<String, SkylarkBuiltinMethod> entry : doc.getBuiltinMethods().entrySet()) { + if (!entry.getValue().annotation.hidden()) { + modules.put(entry.getKey(), + DocgenConsts.toCommandLineFormat(entry.getValue().annotation.doc())); + } + } + } else { + modules.put(doc.getAnnotation().name(), + DocgenConsts.toCommandLineFormat(doc.getAnnotation().doc())); + } + } + return modules; + } + + /** + * Returns the API doc for the specified Skylark object in a command line printable format, + * params[0] identifies either a module or a top-level object, the optional params[1] identifies a + * method in the module.<br> + * Returns null if no Skylark object is found. + */ + public String getCommandLineAPIDoc(String[] params) { + Map<String, SkylarkModuleDoc> modules = collectModules(); + SkylarkModuleDoc toplevelModuleDoc = modules.get(getTopLevelModule().name()); + if (modules.containsKey(params[0])) { + // Top level module + SkylarkModuleDoc module = modules.get(params[0]); + if (params.length == 1) { + String moduleName = module.getAnnotation().name(); + StringBuilder sb = new StringBuilder(); + sb.append(moduleName).append("\n\t").append(module.getAnnotation().doc()).append("\n"); + // Print the signature of all built-in methods + for (SkylarkBuiltinMethod method : module.getBuiltinMethods().values()) { + printBuiltinFunctionDoc(moduleName, method.annotation, sb); + } + // Print all Java methods + for (SkylarkJavaMethod method : module.getJavaMethods()) { + printJavaFunctionDoc(moduleName, method, sb); + } + return DocgenConsts.toCommandLineFormat(sb.toString()); + } else { + return getFunctionDoc(module.getAnnotation().name(), params[1], module); + } + } else if (toplevelModuleDoc.getBuiltinMethods().containsKey(params[0])){ + // Top level object / function + return getFunctionDoc(null, params[0], toplevelModuleDoc); + } + return null; + } + + private String getFunctionDoc(String moduleName, String methodName, SkylarkModuleDoc module) { + if (module.getBuiltinMethods().containsKey(methodName)) { + // Create the doc for the built-in function + SkylarkBuiltinMethod method = module.getBuiltinMethods().get(methodName); + StringBuilder sb = new StringBuilder(); + printBuiltinFunctionDoc(moduleName, method.annotation, sb); + printParams(moduleName, method.annotation, sb); + return DocgenConsts.removeDuplicatedNewLines(DocgenConsts.toCommandLineFormat(sb.toString())); + } else { + // Search if there are matching Java functions + StringBuilder sb = new StringBuilder(); + boolean foundMatchingMethod = false; + for (SkylarkJavaMethod method : module.getJavaMethods()) { + if (method.name.equals(methodName)) { + printJavaFunctionDoc(moduleName, method, sb); + foundMatchingMethod = true; + } + } + if (foundMatchingMethod) { + return DocgenConsts.toCommandLineFormat(sb.toString()); + } + } + return null; + } + + private void printBuiltinFunctionDoc( + String moduleName, SkylarkBuiltin annotation, StringBuilder sb) { + if (moduleName != null) { + sb.append(moduleName).append("."); + } + sb.append(annotation.name()).append("\n\t").append(annotation.doc()).append("\n"); + } + + private void printJavaFunctionDoc(String moduleName, SkylarkJavaMethod method, StringBuilder sb) { + sb.append(getSignature(moduleName, method.name, method.method)) + .append("\t").append(method.callable.doc()).append("\n"); + } + + private void collectBuiltinModule( + Map<SkylarkModule, Class<?>> modules, Class<?> moduleClass) { + if (moduleClass.isAnnotationPresent(SkylarkModule.class)) { + SkylarkModule skylarkModule = moduleClass.getAnnotation(SkylarkModule.class); + modules.put(skylarkModule, moduleClass); + } + } + + private void collectBuiltinDoc(Map<String, SkylarkModuleDoc> modules, Field[] fields) { + for (Field field : fields) { + if (field.isAnnotationPresent(SkylarkBuiltin.class)) { + SkylarkBuiltin skylarkBuiltin = field.getAnnotation(SkylarkBuiltin.class); + Class<?> moduleClass = skylarkBuiltin.objectType(); + SkylarkModule skylarkModule = moduleClass.equals(Object.class) + ? getTopLevelModule() + : moduleClass.getAnnotation(SkylarkModule.class); + if (!modules.containsKey(skylarkModule.name())) { + modules.put(skylarkModule.name(), new SkylarkModuleDoc(skylarkModule, moduleClass)); + } + modules.get(skylarkModule.name()).getBuiltinMethods() + .put(skylarkBuiltin.name(), new SkylarkBuiltinMethod(skylarkBuiltin, field.getType())); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/docgen/SkylarkJavaInterfaceExplorer.java b/src/main/java/com/google/devtools/build/docgen/SkylarkJavaInterfaceExplorer.java new file mode 100644 index 0000000..8f58f71 --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/SkylarkJavaInterfaceExplorer.java
@@ -0,0 +1,160 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.docgen; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.util.StringUtilities; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** + * A helper class to collect all the Java objects / methods reachable from Skylark. + */ +public class SkylarkJavaInterfaceExplorer { + /** + * A class representing a Java method callable from Skylark with annotation. + */ + static final class SkylarkJavaMethod { + public final String name; + public final Method method; + public final SkylarkCallable callable; + + private String getName(Method method, SkylarkCallable callable) { + return callable.name().isEmpty() + ? StringUtilities.toPythonStyleFunctionName(method.getName()) + : callable.name(); + } + + SkylarkJavaMethod(Method method, SkylarkCallable callable) { + this.name = getName(method, callable); + this.method = method; + this.callable = callable; + } + } + + /** + * A class representing a Skylark built-in object or method. + */ + static final class SkylarkBuiltinMethod { + public final SkylarkBuiltin annotation; + public final Class<?> fieldClass; + + public SkylarkBuiltinMethod(SkylarkBuiltin annotation, Class<?> fieldClass) { + this.annotation = annotation; + this.fieldClass = fieldClass; + } + } + + /** + * A class representing a Skylark built-in object with its {@link SkylarkBuiltin} annotation + * and the {@link SkylarkCallable} methods it might have. + */ + static final class SkylarkModuleDoc { + + private final SkylarkModule module; + private final Class<?> classObject; + private final Map<String, SkylarkBuiltinMethod> builtin; + private ArrayList<SkylarkJavaMethod> methods = null; + + SkylarkModuleDoc(SkylarkModule module, Class<?> classObject) { + this.module = Preconditions.checkNotNull(module, + "Class has to be annotated with SkylarkModule: " + classObject); + this.classObject = classObject; + this.builtin = new TreeMap<>(); + } + + SkylarkModule getAnnotation() { + return module; + } + + Class<?> getClassObject() { + return classObject; + } + + private boolean javaMethodsNotCollected() { + return methods == null; + } + + private void setJavaMethods(ArrayList<SkylarkJavaMethod> methods) { + this.methods = methods; + } + + Map<String, SkylarkBuiltinMethod> getBuiltinMethods() { + return builtin; + } + + ArrayList<SkylarkJavaMethod> getJavaMethods() { + return methods; + } + } + + /** + * Collects and returns all the Java objects reachable in Skylark from (and including) + * firstClassObject with the corresponding SkylarkBuiltin annotations. + * + * <p>Note that the {@link SkylarkBuiltin} annotation for firstClassObject - firstAnnotation - + * is also an input parameter, because some top level Skylark built-in objects and methods + * are not annotated on the class, but on a field referencing them. + */ + void collect(SkylarkModule firstModule, Class<?> firstClass, + Map<String, SkylarkModuleDoc> modules) { + Set<Class<?>> processedClasses = new HashSet<>(); + LinkedList<Class<?>> classesToProcess = new LinkedList<>(); + Map<Class<?>, SkylarkModule> annotations = new HashMap<>(); + + classesToProcess.addLast(firstClass); + annotations.put(firstClass, firstModule); + + while (!classesToProcess.isEmpty()) { + Class<?> classObject = classesToProcess.removeFirst(); + SkylarkModule annotation = annotations.get(classObject); + processedClasses.add(classObject); + if (!modules.containsKey(annotation.name())) { + modules.put(annotation.name(), new SkylarkModuleDoc(annotation, classObject)); + } + SkylarkModuleDoc module = modules.get(annotation.name()); + + if (module.javaMethodsNotCollected()) { + ImmutableMap<Method, SkylarkCallable> methods = + FuncallExpression.collectSkylarkMethodsWithAnnotation(classObject); + ArrayList<SkylarkJavaMethod> methodList = new ArrayList<>(); + for (Map.Entry<Method, SkylarkCallable> entry : methods.entrySet()) { + methodList.add(new SkylarkJavaMethod(entry.getKey(), entry.getValue())); + } + module.setJavaMethods(methodList); + + for (Map.Entry<Method, SkylarkCallable> method : methods.entrySet()) { + Class<?> returnClass = method.getKey().getReturnType(); + if (returnClass.isAnnotationPresent(SkylarkModule.class) + && !processedClasses.contains(returnClass)) { + classesToProcess.addLast(returnClass); + annotations.put(returnClass, returnClass.getAnnotation(SkylarkModule.class)); + } + } + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/docgen/SourceFileReader.java b/src/main/java/com/google/devtools/build/docgen/SourceFileReader.java new file mode 100644 index 0000000..c17b3f6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/SourceFileReader.java
@@ -0,0 +1,322 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.docgen; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.regex.Matcher; + +/** + * A helper class to read and process documentations for rule classes and attributes + * from exactly one java source file. + */ +public class SourceFileReader { + + private Collection<RuleDocumentation> ruleDocEntries; + private ListMultimap<String, RuleDocumentationAttribute> attributeDocEntries; + private final ConfiguredRuleClassProvider ruleClassProvider; + private final String javaSourceFilePath; + + public SourceFileReader( + ConfiguredRuleClassProvider ruleClassProvider, String javaSourceFilePath) { + this.ruleClassProvider = ruleClassProvider; + this.javaSourceFilePath = javaSourceFilePath; + } + + /** + * The handler class of the line read from the text file. + */ + public abstract static class ReadAction { + + // Text file line indexing starts from 1 + private int lineCnt = 1; + + protected abstract void readLineImpl(String line) + throws BuildEncyclopediaDocException, IOException; + + protected int getLineCnt() { + return lineCnt; + } + + public void readLine(String line) + throws BuildEncyclopediaDocException, IOException { + readLineImpl(line); + lineCnt++; + } + } + + private static final String LS = DocgenConsts.LS; + + /** + * Reads the attribute and rule documentation present in the file represented by + * SourceFileReader.javaSourceFilePath. The rule doc variables are added to the rule + * documentation (which therefore must be defined in the same file). The attribute docs are + * stored in a different class member, so they need to be handled outside this method. + */ + public void readDocsFromComments() throws BuildEncyclopediaDocException, IOException { + final Map<String, RuleDocumentation> docMap = new HashMap<>(); + final List<RuleDocumentationVariable> docVariables = new LinkedList<>(); + final ListMultimap<String, RuleDocumentationAttribute> docAttributes = + LinkedListMultimap.create(); + readTextFile(javaSourceFilePath, new ReadAction() { + + private boolean inBlazeRuleDocs = false; + private boolean inBlazeRuleVarDocs = false; + private boolean inBlazeAttributeDocs = false; + private StringBuilder sb = new StringBuilder(); + private String ruleName; + private String ruleType; + private String ruleFamily; + private String variableName; + private String attributeName; + private ImmutableSet<String> flags; + private int startLineCnt; + + @Override + public void readLineImpl(String line) throws BuildEncyclopediaDocException { + // TODO(bazel-team): check if copy paste code can be reduced using inner classes + if (inBlazeRuleDocs) { + if (DocgenConsts.BLAZE_RULE_END.matcher(line).matches()) { + endBlazeRuleDoc(docMap); + } else { + appendLine(line); + } + } else if (inBlazeRuleVarDocs) { + if (DocgenConsts.BLAZE_RULE_VAR_END.matcher(line).matches()) { + endBlazeRuleVarDoc(docVariables); + } else { + appendLine(line); + } + } else if (inBlazeAttributeDocs) { + if (DocgenConsts.BLAZE_RULE_ATTR_END.matcher(line).matches()) { + endBlazeAttributeDoc(docAttributes); + } else { + appendLine(line); + } + } + Matcher ruleStartMatcher = DocgenConsts.BLAZE_RULE_START.matcher(line); + Matcher ruleVarStartMatcher = DocgenConsts.BLAZE_RULE_VAR_START.matcher(line); + Matcher ruleAttrStartMatcher = DocgenConsts.BLAZE_RULE_ATTR_START.matcher(line); + if (ruleStartMatcher.find()) { + startBlazeRuleDoc(line, ruleStartMatcher); + } else if (ruleVarStartMatcher.find()) { + startBlazeRuleVarDoc(ruleVarStartMatcher); + } else if (ruleAttrStartMatcher.find()) { + startBlazeAttributeDoc(line, ruleAttrStartMatcher); + } + } + + private void appendLine(String line) { + // Add another line of html code to the building rule documentation + // Removing whitespace and java comment asterisk from the beginning of the line + sb.append(line.replaceAll("^[\\s]*\\*", "") + LS); + } + + private void startBlazeRuleDoc(String line, Matcher matcher) + throws BuildEncyclopediaDocException { + checkDocValidity(); + // Start of a new rule + String[] metaData = matcher.group(1).split(","); + + ruleName = readMetaData(metaData, DocgenConsts.META_KEY_NAME); + ruleType = readMetaData(metaData, DocgenConsts.META_KEY_TYPE); + ruleFamily = readMetaData(metaData, DocgenConsts.META_KEY_FAMILY); + startLineCnt = getLineCnt(); + addFlags(line); + inBlazeRuleDocs = true; + } + + private void endBlazeRuleDoc(final Map<String, RuleDocumentation> documentations) + throws BuildEncyclopediaDocException { + // End of a rule, create RuleDocumentation object + documentations.put(ruleName, new RuleDocumentation(ruleName, ruleType, + ruleFamily, sb.toString(), getLineCnt(), javaSourceFilePath, flags, + ruleClassProvider)); + sb = new StringBuilder(); + inBlazeRuleDocs = false; + } + + private void startBlazeRuleVarDoc(Matcher matcher) throws BuildEncyclopediaDocException { + checkDocValidity(); + // Start of a new rule variable + ruleName = matcher.group(1).replaceAll("[\\s]", ""); + variableName = matcher.group(2).replaceAll("[\\s]", ""); + startLineCnt = getLineCnt(); + inBlazeRuleVarDocs = true; + } + + private void endBlazeRuleVarDoc(final List<RuleDocumentationVariable> docVariables) { + // End of a rule, create RuleDocumentationVariable object + docVariables.add( + new RuleDocumentationVariable(ruleName, variableName, sb.toString(), startLineCnt)); + sb = new StringBuilder(); + inBlazeRuleVarDocs = false; + } + + private void startBlazeAttributeDoc(String line, Matcher matcher) + throws BuildEncyclopediaDocException { + checkDocValidity(); + // Start of a new attribute + ruleName = matcher.group(1).replaceAll("[\\s]", ""); + attributeName = matcher.group(2).replaceAll("[\\s]", ""); + startLineCnt = getLineCnt(); + addFlags(line); + inBlazeAttributeDocs = true; + } + + private void endBlazeAttributeDoc( + final ListMultimap<String, RuleDocumentationAttribute> docAttributes) { + // End of a attribute, create RuleDocumentationAttribute object + docAttributes.put(attributeName, RuleDocumentationAttribute.create( + ruleClassProvider.getRuleClassDefinition(ruleName), + attributeName, sb.toString(), startLineCnt, flags)); + sb = new StringBuilder(); + inBlazeAttributeDocs = false; + } + + private void addFlags(String line) { + // Add flags if there's any + Matcher matcher = DocgenConsts.BLAZE_RULE_FLAGS.matcher(line); + if (matcher.find()) { + flags = ImmutableSet.<String>copyOf(matcher.group(1).split(",")); + } else { + flags = ImmutableSet.<String>of(); + } + } + + private void checkDocValidity() throws BuildEncyclopediaDocException { + if (inBlazeRuleDocs || inBlazeRuleVarDocs || inBlazeAttributeDocs) { + throw new BuildEncyclopediaDocException(javaSourceFilePath, getLineCnt(), + "Malformed documentation, #BLAZE_RULE started after another #BLAZE_RULE."); + } + } + }); + + // Adding rule doc variables to the corresponding rules + for (RuleDocumentationVariable docVariable : docVariables) { + if (docMap.containsKey(docVariable.getRuleName())) { + docMap.get(docVariable.getRuleName()).addDocVariable( + docVariable.getVariableName(), docVariable.getValue()); + } else { + throw new BuildEncyclopediaDocException(javaSourceFilePath, + docVariable.getStartLineCnt(), String.format( + "Malformed rule variable #BLAZE_RULE(%s).%s, " + + "rule %s not found in file.", docVariable.getRuleName(), + docVariable.getVariableName(), docVariable.getRuleName())); + } + } + ruleDocEntries = docMap.values(); + attributeDocEntries = docAttributes; + } + + public Collection<RuleDocumentation> getRuleDocEntries() { + return ruleDocEntries; + } + + public ListMultimap<String, RuleDocumentationAttribute> getAttributeDocEntries() { + return attributeDocEntries; + } + + private String readMetaData(String[] metaData, String metaKey) { + for (String metaDataItem : metaData) { + String[] metaDataItemParts = metaDataItem.split("=", 2); + if (metaDataItemParts.length != 2) { + return null; + } + + if (metaDataItemParts[0].trim().equals(metaKey)) { + return metaDataItemParts[1].trim(); + } + } + return null; + } + + /** + * Reads the template file without variable substitution. + */ + public static String readTemplateContents(String templateFilePath) + throws BuildEncyclopediaDocException, IOException { + return readTemplateContents(templateFilePath, null); + } + + /** + * Reads the template file and expands the variables. The variables has to have + * the following format in the template file: ${VARIABLE_NAME}. In the Map + * input parameter the key has to be VARIABLE_NAME. Variables can be null. + */ + public static String readTemplateContents( + String templateFilePath, final Map<String, String> variables) + throws BuildEncyclopediaDocException, IOException { + final StringBuilder sb = new StringBuilder(); + readTextFile(templateFilePath, new ReadAction() { + @Override + public void readLineImpl(String line) { + sb.append(expandVariables(line, variables) + LS); + } + }); + return sb.toString(); + } + + private static String expandVariables(String line, Map<String, String> variables) { + if (variables != null) { + for (Entry<String, String> variable : variables.entrySet()) { + line = line.replace("${" + variable.getKey() + "}", variable.getValue()); + } + } + return line; + } + + public static void readTextFile(String filePath, ReadAction action) + throws BuildEncyclopediaDocException, IOException { + BufferedReader br = null; + try { + File file = new File(filePath); + if (file.exists()) { + br = new BufferedReader(new FileReader(file)); + } else { + InputStream is = SourceFileReader.class.getResourceAsStream(filePath); + if (is != null) { + br = new BufferedReader(new InputStreamReader(is)); + } + } + if (br != null) { + String line = null; + while ((line = br.readLine()) != null) { + action.readLine(line); + } + } else { + System.out.println("Couldn't find file or resource: " + filePath); + } + } finally { + if (br != null) { + br.close(); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/docgen/templates/be-body.html b/src/main/java/com/google/devtools/build/docgen/templates/be-body.html new file mode 100644 index 0000000..d1fc7ce --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/templates/be-body.html
@@ -0,0 +1,313 @@ +<!-- ============================================ + binary + ============================================ +--> +<h2 id="binary">*_binary</h2> + +<p>A <code>*_binary</code> rule compiles an application. This might be + an executable, a <code>.jar</code> file, and/or a collection of scripts.</p> + +${SECTION_BINARY} + +<!-- ============================================ + library + ============================================ +--> +<h2 id="library">*_library</h2> + +<p>A <code>*_library()</code> rule compiles some sources into a library. + In general, a <code><var>language</var>_library</code> rule works like + the corresponding <code><var>language</var>_binary</code> rule, but + doesn't generate something executable.</p> + +${SECTION_LIBRARY} + +<!-- ============================================ + test + ============================================ +--> +<h2 id="test">*_test</h2> + +<p>A <code>*_test</code> rule compiles a +test. See <a href="#common-attributes-tests">Common attributes for +tests</a> for an explanation of the common attributes. + +${SECTION_TEST} + +<!-- ============================================ + generate code and data + ============================================ +--> +<h2>Rules to Generate Code and Data</h2> + +${SECTION_GENERATE} + +<!-- ============================================ + variables + ============================================ +--> +<h2 id="make_variables">"Make" Variables</h2> + +<p> + This section describes how to use or define a special class of string + variables that are called the "Make" environment. Bazel defines a set of + standard "Make" variables, and you can also define your own. +</p> + +<p>(The reason for the term "Make" is historical: the syntax and semantics of + these variables are somewhat similar to those of GNU Make, and in the + original implementation, were implemented by GNU Make. The + scare-quotes are present because newer build tools support + "Make" variables without being implemented using GNU Make; therefore + it is important to read the specification below carefully to + understand the differences.) +</p> + +<p>To see the list of all common "Make" variables and their values, + run <code>bazel info --show_make_env</code>. +</p> + +<p>Build rules can introduce additional rule specific variables. One example is + the <a href="#genrule.cmd"><code>cmd</code> attribute of a genrule</a>. +</p> + +<h3 id='make-var-substitution'>"Make" variable substitution</h3> + +<p>Variables can be referenced in attributes and other variables using either + <code>$(FOO)</code> or <code>varref('FOO')</code>, where <code>FOO</code> is + the variable name. In the attribute documentation of rules, it is mentioned + when an attribute is subject to "Make" variable substitution. For those + attributes this means that any substrings of the form <code>$(X)</code> + within those attributes will be interpreted as references to the "Make" + variable <var>X</var>, and will be replaced by the appropriate value of that + variable for the applicable build configuration. The parens may be omitted + for variables whose name is a single character. +</p> +<p> + It is an error if such attributes contain embedded strings of the + form <code>$(X)</code> where <var>X</var> is not the name of a + "Make" variable, or unclosed references such as <code>$(</code> not + matched by a corresponding <code>)</code>. +</p> +<p> + Within such attributes, literal dollar signs must be escaped + as <code>$$</code> to prevent this expansion. +</p> +<p> + Those attributes that are subject to this substitution are + explicitly indicated as such in their definitions in this document. +</p> + +<h3 id="predefined_variables">Predefined "Make" Variables</h3> + +<p>Bazel defines a set of "Make" variables for you.</p> + +<p>The build system also provides a consistent PATH for genrules and tests which + need to execute shell commands. For genrules, you can indirect your commands + using the Make variables below. For basic Unix utilities, prefer relying on + the PATH variable to guarantee correct results. For genrules involving + compiler and platform invocation, you must use the Make variable syntax. + The same basic command set is also available during tests. Simply rely on the + PATH.</p> + +<p>Bazel uses a tiny Unix distribution to guarantee consistent behavior of + build and test steps which execute shell code across all build execution + hosting environments but it does not enforce a pure chroot. As such, do + <b>not</b> use hard coded paths, such as + <code>/usr/bin/foo</code>. Binaries referenced in hardcoded paths are not + hermetic and can lead to unexpected and non-reproducible build behavior.</p> + +<p><strong>Command Variables for genrules</strong></p> + +<p>Note that in general, you should simply refer to many common utilities as +bare commands that the $PATH variable will correctly resolve to hermetic +versions for you.</p> + +<p><strong>Path Variables</strong></p> + +<ul><!-- keep alphabetically sorted --> + <li><code>BINDIR</code>: The base of the generated binary tree for the target + architecture. (Note that a different tree may be used for + programs that run during the build on the host architecture, + to support cross-compiling. If you want to run a tool from + within a genrule, the recommended way of specifying the path to + the tool is to use <code>$(location <i>toolname</i>)</code>, + where <i>toolname</i> must be listed in the <code>tools</code> + attribute for the genrule.</li> + <li><code>GENDIR</code>: The base of the generated code + tree for the target architecture.</li> + <li><code>JAVABASE</code>: + The base directory containing the Java utilities. + It will have a "bin" subdirectory.</li> +</ul> + +<p><strong>Architecture Variables</strong></p> + +<ul><!-- keep alphabetically sorted --> + <li><code>ABI</code>: The C++ ABI version. </li> + <li><code>ANDROID_CPU</code>: The Android target architecture's cpu. </li> + <li><code>JAVA_CPU</code>: The Java target architecture's cpu. </li> + <li> <code>TARGET_CPU</code>: The target architecture's cpu. </li> +</ul> + +<p id="predefined_variables.genrule.cmd"> + <strong> + Other Variables available to <a href="#genrule.cmd">the cmd attribute of a genrule</a> + </strong> +</p> +<ul><!-- keep alphabetically sorted --> + <li><code>OUTS</code>: The <code>outs</code> list. If you have only one output + file, you can also use <code>$@</code>.</li> + <li><code>SRCS</code>: The <code>srcs</code> list (or more + precisely, the pathnames of the files corresponding to + labels in the <code>srcs</code> list). If you have only one + source file, you can also use <code>$<</code>.</li> + <li><code><</code>: <code>srcs</code>, if it is a single file.</li> + <li><code>@</code>: <code>outs</code>, if it is a single file.</li> + <li><code>@D</code>: The output directory. If there is only + one filename in <code>outs</code>, this expands to the + directory containing that file. If there are multiple + filenames, this variable instead expands to the package's root + directory in the <code>genfiles</code> tree, <i>even if all + the generated files belong to the same subdirectory</i>! + <!-- (as a consequence of the "middleman" implementation) --> + If the genrule needs to generate temporary intermediate files + (perhaps as a result of using some other tool like a compiler) + then it should attempt to write the temporary files to + <code>@D</code> (although <code>/tmp</code> will also be + writable), and to remove any such generated temporary files. + Especially, avoid writing to directories containing inputs - + they may be on read-only filesystems. </li> +</ul> + +</ul> + +<h3 id="define_your_own_make_vars">Defining Your Own Variables</h3> + +<p> +You may want to use Python-style variable assignments rather than "Make" +variables, because they work in more use cases and are less surprising. "Make" +variables will work in the <a href="#genrule.cmd">cmd</a> attribute of genrules +and in the key of the <a href="#cc_library.abi_deps">abi_deps</a> attribute of +a limited number of rules, but only in very few other places. + +</p> +<p>To define your "Make" own variables, first call <a + href="#vardef">vardef()</a> to define your variable, then call <a + href="#varref">varref(name)</a> to retrieve it. varref can be embedded as part + of a larger string. Custom "Make" variables differ from ordinary "Python" + variables in the BUILD language in two important ways: +</p> +<ul> + <li>Only string values are supported,</li> + <li>The "Make" environment is parameterized over the build + platform, so that variables can be conditionally defined based on + the target architecture, ABI or compiler, and</li> + <li>The values of custom "Make" variables are <i>not available</i> during + BUILD-file evaluation. To work around this, you must call <a + href="#varref">varref()</a> to retrieve the value of a variable (unlike + predefined values, which can be retrieved using <code>$(FOO)</code>. + varref defers evaluation until after BUILD file evaluation.</li> +</ul> + +<h4 id="vardef">vardef()</h4> + +<p><code>vardef(name, value, platform)</code></p> + + <p> + Define a variable for use within this <code>BUILD</code> file only. + This variable can then be used by <a href="#varref">varref()</a>. + The value of the variable can be overridden on the command line by using the + <code class='flag'><a href='bazel-user-manual.html#flag--define'>--define</a></code> + flag. + </p> + + <p id="vardef_args"><strong>Arguments</strong></p> +<ul> + <li><code>name</code>: The name of the variable. + <i>(String; required)</i><br/> + Convention is to use names consisting of ALL CAPS. This name must + be a unique identifier in this package. + </li> + <li><code>value</code>: The value to assign to this variable. + <i>(String; required)</i><br/> + The value may make use of variables you know are defined in the "Make" + environment. + </li> + <li><code>platform</code>: Conditionally define this variable for a given + platform. + <i>(String; optional)</i><br/> + + <code>vardef</code> binds the <code>name</code> to <code>value</code> if we're + compiling for <code>platform</code>. + </li> +</ul> + +<p id="vardef_notes"><strong>Notes</strong></p> +<p> + Because references to "Make" variables are expanded <i>after</i> + BUILD file evaluation, the relative order of <code>vardef</code> + statements and rule declarations is unimportant; it is order of + <code>vardef</code> statements relative to each other, and hence the + state of the "Make" environment at the end of evaluation that + matters. +</p> +<p> + If there are multiple matching <code>vardef</code> definitions for + the same variable, the definition that wins is + the <strong>last</strong> matching definition + <strong>that specifies a platform</strong>, unless there are no matching + definitions that specify a platform, in which case the definition + that wins is the <strong>last</strong> definition <strong>without + a platform</strong>. +</p> + +<!-- ================================================================= + varref() + ================================================================= + --> + +<h4 id="varref">varref</h4> + +<p><code>varref(name)</code></p> + +<p> +<code>varref("FOO")</code> is equivalent of writing "$(FOO)". It is used to +dereference variables defined with <a href="#vardef"><code>vardef</code></a> +as well as <a href="#predefined_variables">predefined variables</a>. +</p> + +<p> + In rule attributes that are subject to "Make" variable + substitution, the string produced by <code>varref(<i>name</i>)</code> + will expand to the value of variable <i>name</i>. +</p> + +<p><code>varref(name)</code> may not be used in rule attributes that are +not subject to "Make" variable substitution.</p> + +<p id="varref_args"><strong>Arguments</strong></p> +<ul> + <li><code>name</code>: The name of the variable to dereference. + </li> +</ul> + +<p id="varref_notes"><strong>Notes</strong></p> +<ul> + <li><code>varref</code> can access either local or global variables. + It prefers the local variable, if both a local and a global exist with + the same name. + </li> +</ul> + +<p id="varref_examples"><strong>Examples</strong></p> +<p>See <a href="#vardef_examples">vardef()</a> examples.</p> + + +<!-- ============================================ + other + ============================================ +--> +<h2 id="misc">Other Stuff</h2> + +${SECTION_OTHER}
diff --git a/src/main/java/com/google/devtools/build/docgen/templates/be-footer.html b/src/main/java/com/google/devtools/build/docgen/templates/be-footer.html new file mode 100644 index 0000000..fb4cc67 --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/templates/be-footer.html
@@ -0,0 +1,407 @@ +<!-- ================================================================= + package() + ================================================================= +--> + +<h3 id="package">package</h3> + +<p>This function declares metadata that applies to every subsequent rule in the +package.</p> + +<p>The <a href="build-ref.html#package">package</a> + function is used at most once within a build package (BUILD file). + It is recommended that the package function is called at the top of the + file, before any rule.</p> + +<h4 id="package_args">Arguments</h4> + +<ul> + + <li id="package.default_visibility"><code>default_visibility</code>: + The default visibility of the rules in this package. + <i>(List of <a href="build-ref.html#labels">labels</a>; optional)</i><br/> + + <p>Every rule in this package has the visibility specified in this + attribute, unless otherwise specified in the <code>visibility</code> + attribute of the rule. For detailed information about the syntax of this + attribute, see the documentation of the <a href="#common.visibility">visibility</a> + attribute. + </li> + + <li id="package.default_obsolete"><code>default_obsolete</code>: + The default value of <a href="#common.obsolete"><code>obsolete</code></a> property + for all rules in this package. <i>(Boolean; optional; default is 0)</i> + </li> + + <li id="package.default_deprecation"><code>default_deprecation</code>: + Sets the default <a href="#common.deprecation"><code>deprecation</code></a> message + for all rules in this package. <i>(String; optional)</i> + </li> + + <li id="package.default_testonly"><code>default_testonly</code>: + Sets the default <a href="#common.testonly"><code>testonly</code></a> property + for all rules in this package. <i>(Boolean; optional; default is 0 except as noted)</i> + + <p>In packages under <code>javatests</code> the default value is 1.</p> + </li> + +</ul> + +<h4 id="package_example">Examples</h4> + +The declaration below declares that the rules in this package are +visible only to members of package +group <code>//foo:target</code>. Individual visibility declarations +on a rule, if present, override this specification. + +<pre class="code"> +package(default_visibility = ["//foo:target"]) +</pre> + +<!-- ================================================================= + package_group() + ================================================================= +--> + +<h3 id="package_group">package_group</h3> + +<p><code>package_group(name, packages, includes)</code></p> + +<p>This function defines a set of build packages. + +Package groups are used for visibility control. You can grant access to a rule +to one or more package groups, every rule, or only to rules declared +in the same package. For more detailed description of the visibility system, see +the <a href="#common.visibility">visibility</a> attribute. + +<h4 id="package_group_args">Arguments</h4> + +<ul> + <li id="package_group.name"><code>name</code>: + A unique name for this rule. + <i>(<a href="build-ref.html#name">Name</a>; required)</i> + </li> + + <li id="package_group.packages"><code>packages</code>: + A complete enumeration of packages in this group. + <i>(List of <a href="build-ref.html#s4">Package</a>; optional)</i><br/> + + <p>Packages should be referred to using their full names, + starting with a double slash. For + example, <code>//foo/bar/main</code> is a valid element + of this list.</p> + + <p>You can also specify wildcards: the specification + <code>//foo/...</code> specifies every package under + <code>//foo</code>, including <code>//foo</code> itself. + + <p>If this attribute is missing, the package group itself will contain + no packages (but it can still include other package groups).</p> + </li> + + <li id="package_group.includes"><code>includes</code>: + Other package groups that are included in this one. + <i>(List of <a href="build-ref.html#labels">labels</a>; optional)</i><br/> + + <p>The labels in this attribute must refer to other package + groups. Packages in referenced package groups are taken to be part + of this package group. This is transitive, that is, if package + group <code>a</code> contains package group <code>b</code>, + and <code>b</code> contains package group <code>c</code>, every + package in <code>c</code> will also be a member of <code>a</code>.</p> + </li> +</ul> + + +<h4 id="package_group_example">Examples</h4> + +The following <code>package_group</code> declaration specifies a +package group called "tropical" that contains tropical fruits. + +<pre class="code"> +package_group( + name = "tropical", + packages = [ + "//fruits/mango", + "//fruits/orange", + "//fruits/papaya/...", + ], +) +</pre> + +The following declarations specify the package groups of a fictional +application: + +<pre class="code"> +package_group( + name = "fooapp", + includes = [ + ":controller", + ":model", + ":view", + ], +) + +package_group( + name = "model", + packages = ["//fooapp/database"], +) + +package_group( + name = "view", + packages = [ + "//fooapp/swingui", + "//fooapp/webui", + ], +) + +package_group( + name = "controller", + packages = ["//fooapp/algorithm"], +) +</pre> + + +<!-- ================================================================= + DESCRIPTION + ================================================================= + --> + +<h3 id="description">Description</h3> + +<p><code># Description: <var>...</var></code></p> + + <p> + Each BUILD file should contain a <code>Description</code> comment. + </p> + + <p> + Description comments may contain references to other + documentation. A string that starts with "http" will become a + link. HTML markup is + allowed in description comments, but please keep the BUILD files readable. + We encourage you to list the URLs of relevant design docs and howtos + in these description comments. + </p> + +<h3 id="distribs">distribs</h3> + +<p><code>distribs(distrib_methods)</code></p> + +<p><code>distribs()</code> specifies the default distribution method (or + methods) of the build rules in a <code>BUILD</code> file. The <code>distribs()</code> + directive + should appear close to the beginning of the <code>BUILD</code> file, + before any build rules, as it sets the <code>BUILD</code>-file scope + default for build rule distribution methods. +</p> + +<h4 id="distribs_args">Arguments</h4> + +<p>The argument, <code id="distribs.distrib_methods">distrib_methods</code>, + is a list of distribution-method strings. +</p> + +<!-- ================================================================= + exports_files([label, ...]) + ================================================================= + --> + +<h3 id="exports_files">exports_files</h3> + +<p><code>exports_files([<i>label</i>, ...], visibility, licenses)</code></p> + +<p> + <code>exports_files()</code> specifies a list of files belonging to + this package that are exported to other packages but not otherwise + mentioned in the BUILD file. +</p> + +<p> + The BUILD file for a package may only refer to files belonging to another + package if they are mentioned somewhere in the other packages's BUILD file, + whether as an input to a rule or an explicit or implicit output from a rule. + The remaining files are not associated with a specific rule but are just "data", + and for these, <code>exports_files</code> ensures that they may be referenced by + other packages. (One kind of data for which this is particularly useful are + shell and Perl scripts.) +</p> + +<p> + Note: A BUILD file only consisting of <code>exports_files()</code> statements + is needless though, as there are no BUILD rules that could own any files. + The files listed can already be accessed through the containing package and + exported from there if needed. +</p> + +<h4 id="exports_files_args">Arguments</h4> + +<p> + The argument is a list of names of files within the current package. A + visibility declaration can also be specified; in this case, the files will be + visible to the targets specified. If no visibility is specified, the files + will be visible to every package, even if a package default visibility was + specified in the <code><a href="#package">package</a></code> function. The + <a href="#common.licenses">licenses</a> can also be specified. +</p> + +<!-- ================================================================= + glob() + ================================================================= + --> + +<h3 id="glob">glob</h3> + +<p><code>glob(include, exclude=[], exclude_directories=1)</code> +</p> + +<p> +Glob is a helper function that can be used anywhere a list of filenames +is expected. It takes one or two lists of filename patterns containing +the <code>*</code> wildcard: as per the Unix shell, this wildcard +matches any string excluding the directory separator <code>/</code>. +In addition filename patterns can contain the recursive <code>**</code> +wildcard. This wildcard will match zero or more complete +path segments separated by the directory separator <code>/</code>. +This wildcard can only be used as a complete path segment. For example, +<code>"x/**/*.java"</code> is legal, but <code>"test**/testdata.xml"</code> +and <code>"**.java"</code> are both illegal. No other wildcards are supported. +</p> +<p> +Glob returns a list of every file in the current build package that: +</p> +<ul> + <li style="margin-bottom: 0"> + Matches at least one pattern in <code>include</code>. </li> + <li> + Does not match any of the patterns in <code>exclude</code> (default []).</li> +</ul> +<p> +If the <code>exclude_directories</code> argument is enabled (1), files of +type directory will be omitted from the results (default 1). +</p> +<p> +There are several important limitations and caveats: +</p> + +<ol> + <li> + Globs only match files in your source tree, never + generated files. If you are building a target that requires both + source and generated files, create an explicit list of generated + files, and use <code>+</code> to add it to the result of the + <code>glob()</code> call. + </li> + + <li> + Globs may match files in subdirectories. And subdirectory names + may be wildcarded. However... + </li> + + <li> + Labels are not allowed to cross the package boundary and glob does + not match files in subpackages. + + For example, the glob expression <code>**/*.cc</code> in package + <code>x</code> does not include <code>x/y/z.cc</code> if + <code>x/y</code> exists as a package (either as + <code>x/y/BUILD</code>, or somewhere else on the package-path). This + means that the result of the glob expression actually depends on the + existence of BUILD files — that is, the same glob expression would + include <code>x/y/z.cc</code> if there was no package called + <code>x/y</code>. + </li> + + <li> + The restriction above applies to all glob expressions, + no matter which wildcards they use. + </li> +</ol> + +<p> +In general, you should <b>try to provide an appropriate extension (e.g. *.html) +instead of using a bare '*'</b> for a glob pattern. The more explicit name +is both self documenting and ensures that you don't accidentally match backup +files, or emacs/vi/... auto-save files. +</p> + +<h4 id="glob_example">Glob Examples</h4> + +<p>Include all txt files in directory testdata except experimental.txt. +Note that files in subdirectories of testdata will not be included. If +you want those files to be included, use a recursive glob (**).</p> +<pre class="code"> +java_test( + name = "myprog", + srcs = ["myprog.java"], + data = glob( + ["testdata/*.txt"], + exclude = ["testdata/experimental.txt"], + ), +) +</pre> + +<h4 id="recursive_glob_example">Recursive Glob Examples</h4> + +<p>Create a library built from all java files in this directory and all +subdirectories except those whose path includes a directory named testing. +Subdirectories containing a BUILD file are ignored. +<b>This should be a very common pattern.</b> +</p> +<pre class="code"> +java_library( + name = "mylib", + srcs = glob( + ["**/*.java"], + exclude = ["**/testing/**"], + ), +) +</pre> + +<p>Make the test depend on all txt files in the testdata directory, + its subdirectories</p> +<pre class="code"> +java_test( + name = "mytest", + srcs = ["mytest.java"], + data = glob(["testdata/**/*.txt"]), +) +</pre> + +<!-- ================================================================= + licenses() + ================================================================= +--> + +<h3 id="licenses">licenses</h3> + +<p><code>licenses(license_types)</code></p> + +<p><code>licenses()</code> specifies the default license type (or types) + of the build rules in a <code>BUILD</code> file. The <code>licenses()</code> + directive should appear close to the + beginning of the <code>BUILD</code> file, before any build rules, as it + sets the <code>BUILD</code>-file scope default for build rule license + types.</p> + +<h4 id="licenses_args">Arguments</h4> + +<p>The argument, <code id="licenses.licence_types">license_types</code>, + is a list of license-type strings. +</p> + +<!-- ================================================================= + include() + ================================================================= +--> + +<h3 id="include">include</h3> + +<p><code>include(name)</code></p> + +<p><code>include()</code> incorporates build + language definitions from an external file into the evaluation of the + current <code>BUILD</code> file.</p> + +</body> +</html>
diff --git a/src/main/java/com/google/devtools/build/docgen/templates/be-header.html b/src/main/java/com/google/devtools/build/docgen/templates/be-header.html new file mode 100644 index 0000000..cdebf9b --- /dev/null +++ b/src/main/java/com/google/devtools/build/docgen/templates/be-header.html
@@ -0,0 +1,612 @@ +<!DOCTYPE html> +<html> +<head> + <title>Bazel BUILD Encyclopedia of Functions</title> + + <style type="text/css" id="internalStyle"> + body { + background-color: #ffffff; + color: black; + margin-right: 10%; + margin-left: 10%; + } + + h1, h2, h3, h4, h5, h6 { + color: #dd7755; + font-family: sans-serif; + } + @media print { + /* Darker version for printing */ + h1, h2, h3, h4, h5, h6 { + color: #008000; + font-family: helvetica, sans-serif; + } + } + + h1 { + text-align: center; + } + h2 { + margin-left: -0.5in; + } + h3 { + margin-left: -0.25in; + } + h4 { + margin-left: -0.125in; + } + hr { + margin-left: -1in; + } + address { + text-align: right; + } + + /* A compact unordered list */ + ul.tight > li { + margin-bottom: 0; + } + + /* Use the <code> tag for bits of code and <var> for variable and object names. */ + code,pre,samp,var { + color: #006000; + } + /* Use the <file> tag for file and directory paths and names. */ + file { + color: #905050; + font-family: monospace; + } + /* Use the <kbd> tag for stuff the user should type. */ + kbd { + color: #600000; + } + div.note p { + float: right; + width: 3in; + margin-right: 0%; + padding: 1px; + border: 2px solid #60a060; + background-color: #fffff0; + } + + table.grid { + background-color: #ffffee; + border: 1px solid black; + border-collapse: collapse; + margin-left: 2mm; + margin-right: 2mm; + } + + table.grid th, + table.grid td { + border: 1px solid black; + padding: 0 2mm 0 2mm; + } + + /* Use pre.code for code listings. + Use pre.interaction for "Here's what you see when you run a.out.". + (Within pre.interaction, use <kbd> things the user types) + */ + pre.code { + background-color: #FFFFEE; + border: 1px solid black; + color: #004000; + font-size: 10pt; + margin-left: 2mm; + margin-right: 2mm; + padding: 2mm; + -moz-border-radius: 12px 0px 0px 0px; + } + + pre.interaction { + background-color: #EEFFEE; + color: #004000; + padding: 2mm; + } + + pre.interaction kbd { + font-weight: bold; + color: #000000; + } + + /* legacy style */ + pre.interaction b.astyped { + color: #000000; + } + + h1 { margin-bottom: 5px; } + .toc { margin: 0px; } + ul li { margin-bottom: 1em; } + ul.toc li { margin-bottom: 0px; } + em.harmful { color: red; } + + .deprecated { text-decoration: line-through; } + .discouraged { text-decoration: line-through; } + + #rules { width: 980px; border-collapse: collapse; } + #rules td { border-top: 1px solid gray; padding: 4px; vertical-align: top; } + #rules th { text-align: left; padding: 4px; } + + table.layout { width: 980px; } + table.layout td { vertical-align: top; } + + #maintainer { text-align: right; } + + dt { + font-weight: bold; + margin-top: 0.5em; + margin-bottom: 0.5em; + } + dd dt { + font-weight: normal; + text-decoration: underline; + color: gray; + } + </style> + + <style type="text/css"> + .rule-signature { + color: #006000; + font-family: monospace; + } + </style> +</head> + +<body> + +<h1>Bazel BUILD Encyclopedia of Functions</h1> + +<h2>Contents</h2> + + <h3>Concepts and terminology</h3> + <table class="layout"><tr><td> + <ul class="toc"> + <li><a href="#common-definitions">Common definitions</a>: + <ul> + <li><a href="#sh-tokenization">Bourne shell tokenization</a></li> + <li><a href="#label-expansion">Label expansion</a></li> + <li><a href="#common-attributes">Common attributes</a></li> + <li><a href="#common-attributes-tests">Common attributes for tests</a></li> + <li><a href="#common-attributes-binaries">Common attributes for binaries</a></li> + <li><a href="#implicit-outputs">Implicit output targets</a></li> + </ul> + </li> + </ul> + </td><td> + <ul class="toc"> + <li><a href="#make_variables">"Make" variables</a> + <ul class="toc"> + <li><a href="#make-var-substitution">"Make" variable substitution</a></li> + <li><a href="#predefined_variables">Predefined variables</a></li> + <li><a href="#define_your_own_make_vars">Defining your own variables</a> + <ul> + <li><a href="#vardef">vardef</a></li> + <li><a href="#varref">varref</a></li> + </ul> + </li> + </ul> + <li><a href="#predefined-python-variables">Predefined Python Variables</a></li> + </ul> + </td><td> + <ul class="toc"> + <li><a href="#include">include</a></li> + <li><a href="#package">package</a></li> + <li><a href="#package_group">package_group</a></li> + <li><a href="#description">Description</a></li> + <li><a href="#distribs">distribs</a></li> + <li><a href="#licenses">licenses</a></li> + <li><a href="#exports_files">exports_files</a></li> + <li><a href="#glob">glob</a></li> + </ul> + </td></tr></table> + +${HEADER_TABLE} + +<h2 id="common-definitions">Common definitions</h2> + +<p>This section defines various terms and concepts that are common to +many functions or build rules below. +</p> + +<!-- we haven't defined 'rules' or 'attributes' yet. --> + +<h3 id='sh-tokenization'>Bourne shell tokenization</h3> +<p> + Certain string attributes of some rules are split into multiple + words according to the tokenization rules of the Bourne shell: + unquoted spaces delimit separate words, and single- and + double-quotes characters and backslashes are used to prevent + tokenization. +</p> +<p> + Those attributes that are subject to this tokenization are + explicitly indicated as such in their definitions in this document. +</p> +<p> + Attributes subject to "Make" variable expansion and Bourne shell + tokenization are typically used for passing arbitrary options to + compilers and other tools, such as the <code>copts</code> attribute + of <code>cc_library</code>, or the <code>javacopts</code> attribute of + <code>java_library</code>. Together these substitutions allow a + single string variable to expand into a configuration-specific list + of option words. +</p> + +<h3 id='label-expansion'>Label expansion</h3> +<p> + Some string attributes of a very few rules are subject to label + expansion: if those strings contain a valid build label as a + substring, such as <code>//mypkg:target</code>, and that label is a + declared prerequisite of the current rule, it is expanded into the + pathname of the file represented by the target <code>//mypkg:target</code>. +</p> +<p> + Example attributes include the <code>cmd</code> attribute of + genrule, and the <code>linkopts</code> attribute + of <code>cc_library</code>. The details may vary significantly in + each case, over such issues as: whether relative labels are + expanded; how labels that expand to multiple files are + treated, etc. Consult the rule attribute documentation for + specifics. +</p> + +<h3 id="common-attributes">Attributes common to all build rules</h3> + +<p>This section describes attributes that are common to all build rules.<br/> +Please note that it is an error to list the same label twice in a list of +labels attribute. +</p> + +<ul> +<li id="common.deps"><code>deps</code>: +A list of dependencies of this rule. +<i>(List of <a href="build-ref.html#labels">labels</a>; optional)</i><br/> +The precise semantics of what it means for this rule to depend on +another using <code>deps</code> are specific to the kind of this rule, +and the rule-specific documentation below goes into more detail. +At a minimum, though, the targets named via <code>deps</code> will +appear in the <code>*.runfiles</code> area of this rule, if it has +one. +<p>Most often, a <code>deps</code> dependency is used to allow one +module to use symbols defined in another module written in the +same programming language and separately compiled. Cross-language +dependencies are also permitted in many cases: for example, +a <code>java_library</code> rule may depend on C++ code in +a <code>cc_library</code> rule, by declaring the latter in +the <code>deps</code> attribute. See the definition +of <a href="build-ref.html#deps">dependencies</a> for more +information.</p> +<p>Almost all rules permit a <code>deps</code> attribute, but where +this attribute is not allowed, this fact is documented under the +specific rule.</p></li> +<li id="common.data"><code>data</code>: +The list of files needed by this rule at runtime. +<i>(List of <a href="build-ref.html#labels">labels</a>; optional)</i><br/> +Targets named in the <code>data</code> attribute will appear in +the <code>*.runfiles</code> area of this rule, if it has one. This +may include data files needed by a binary or library, or other +programs needed by it. See the +<a href="build-ref.html#data">data dependencies</a> section for more +information about how to depend on and use data files. +<p>Almost all rules permit a <code>data</code> attribute, but where +this attribute is not allowed, this fact is documented under the +specific rule.</p></li> +<li id="common.deprecation"><code>deprecation</code>: +<i>(String; optional)</i><br/> +An explanatory warning message associated with this rule. +Typically this is used to notify users that a rule has become obsolete, +or has become superseded by another rule, is private to a package, or is +perhaps "considered harmful" for some reason. It is a good idea to include +some reference (like a webpage, a bug number or example migration CLs) so +that one can easily find out what changes are required to avoid the message. +If there is a new target that can be used as a drop in replacement, it is a good idea +to just migrate all users of the old target. +<p> +This attribute has no effect on the way things are built, but it +may affect a build tool's diagnostic output. The build tool issues a +warning when a rule with a <code>deprecation</code> attribute is +depended upon by another rule.</p> +<p> +Intra-package dependencies are exempt from this warning, so that, +for example, building the tests of a deprecated rule does not +encounter a warning.</p> +<p> +If a deprecated rule depends on another deprecated rule, no warning +message is issued.</p> +<p> +Once people have stopped using it, the package can be removed or marked as +<a href="#common.obsolete"><code>obsolete</code></a>.</p></li> +<li id="common.distribs"><code>distribs</code>: +<i>(List of strings; optional)</i><br/> +A list of distribution-method strings to be used for this particular build rule. +Overrides the <code>BUILD</code>-file scope defaults defined by the +<a href="#distribs"><code>distribs()</code></a> directive.</li> +<li id="common.licenses"><code>licenses</code>: +<i>(List of strings; optional)</i><br/> +A list of license-type strings to be used for this particular build rule. +Overrides the <code>BUILD</code>-file scope defaults defined by the +<a href="#licenses"><code>licenses()</code></a> directive.</li> +<li id="common.obsolete"><code>obsolete</code>: +<i>(Boolean; optional; default 0)</i><br/> +If 1, only obsolete targets can depend on this target. It is an error when +a non-obsolete target depends on an obsolete target. +<p> +As a transition, one can first mark a package as in +<a href="#common.deprecation"><code>deprecation</code></a>.</p> +<p> +This attribute is useful when you want to prevent a target from +being used but are yet not ready to delete the sources.</p></li> +<li id="common.tags"><code>tags</code>: +List of arbitrary text tags. Tags may be any valid string; default is the +empty list.<br/> +<i>Tags</i> can be used on any rule; but <i>tags</i> are most useful +on test and <code>test_suite</code> rules. Tags on non-test rules +are only useful to humans and/or external programs. +<i>Tags</i> are generally used to annotate a test's role in your debug +and release process. The use of tags and size elements +gives flexibility in assembling suites of tests based around codebase +check-in policy. +<p> +A few tags have special meaning to the build tool; consult +the <a href='bazel-user-manual.html#tags_keywords'>Bazel +documentation</a> for details. +</p></li> +<li id="common.testonly"><code>testonly</code>: +<i>(Boolean; optional; default 0 except as noted)</i><br /> +If 1, only testonly targets (such as tests) can depend on this target. +<p>Equivalently, a rule that is not <code>testonly</code> is not allowed to +depend on any rule that is <code>testonly</code>.</p> +<p>Tests (<code>*_test</code> rules) +and test suites (<a href="#test_suite">test_suite</a> rules) +are <code>testonly</code> by default.</p> +<p>By virtue of +<a href="#package.default_testonly"><code>default_testonly</code></a>, +targets under <code>javatests</code> are <code>testonly</code> by default.</p> +<p>This attribute is intended to mean that the target should not be +contained in binaries that are released to production.</p> +<p>Because testonly is enforced at build time, not run time, and propagates +virally through the dependency tree, it should be applied judiciously. For +example, stubs and fakes that +are useful for unit tests may also be useful for integration tests +involving the same binaries that will be released to production, and +therefore should probably not be marked testonly. Conversely, rules that +are dangerous to even link in, perhaps because they unconditionally +override normal behavior, should definitely be marked testonly.</p></li> +<li id="common.visibility"><code>visibility</code>: +<i>(List of <a href="build-ref.html#labels">labels</a>; optional; default private)</i><br/> +<p>The <code>visibility</code> attribute on a rule controls whether +the rule can be used by other packages. Rules are always visible to +other rules declared in the same package.</p> +<p>There are five forms (and one temporary form) a visibility label can take: +<ul> +<li><code>['//visibility:public']</code>: Anyone can use this rule.</li> +<li><code>['//visibility:private']</code>: Only rules in this package +can use this rule. Rules in <code>javatests/foo/bar</code> +can always use rules in <code>java/foo/bar</code>. +</li> +<li><code>['//some/package:__pkg__', '//other/package:__pkg__']</code>: +Only rules in <code>some/package</code> and <code>other/package</code> +(defined in <code>some/package/BUILD</code> and +<code>other/package/BUILD</code>) have access to this rule. Note that +sub-packages do not have access to the rule; for example, +<code>//some/package/foo:bar</code> or +<code>//other/package/testing:bla</code> wouldn't have access. +<code>__pkg__</code> is a special target and must be used verbatim. +It represents all of the rules in the package. +</li> +<li><code>['//project:__subpackages__', '//other:__subpackages__']</code>: +Only rules in packages <code>project</code> or <code>other</code> or +in one of their sub-packages have access to this rule. For example, +<code>//project:rule</code>, <code>//project/library:lib</code> or +<code>//other/testing/internal:munge</code> are allowed to depend on +this rule (but not <code>//independent:evil</code>) +</li> +<li><code>['//some/package:my_package_group']</code>: +A <a href="#package_group">package group</a> is +a named set of package names. Package groups can also grant access rights +to entire subtrees, e.g.<code>//myproj/...</code>. +</li> +</ul> +<p>The visibility specifications of <code>//visibility:public</code>, +<code>//visibility:private</code> and +<code>//visibility:legacy_public</code> +can not be combined with any other visibility specifications. +A visibility specification may contain a combination of package labels +(i.e. //foo:__pkg__) and package_groups.</p> +<p>If a rule does not specify the visibility attribute, +the <code><a href="#package">default_visibility</a></code> +attribute of the <code><a href="#package">package</a></code> +statement in the BUILD file containing the rule is used +(except <a href="#exports_files">exports_files</a> and +<a href="#cc_public_library">cc_public_library</a>, which always default to +public).</p> +<p><b>Example</b>:</p> +<p> +File <code>//frobber/bin/BUILD</code>: +</p> +<pre class="code"> +# This rule is visible to everyone +java_binary( + name = "executable", + visibility = ["//visibility:public"], + deps = [":library"], +) + +# This rule is visible only to rules declared in the same package +java_library( + name = "library", + visibility = ["//visibility:private"], +) + +# This rule is visible to rules in package //object and //noun +java_library( + name = "subject", + visibility = [ + "//noun:__pkg__", + "//object:__pkg__", + ], +) + +# See package group //frobber:friends (below) for who can access this rule. +java_library( + name = "thingy", + visibility = ["//frobber:friends"], +) +</pre> +<p> +File <code>//frobber/BUILD</code>: +</p> +<pre class="code"> +# This is the package group declaration to which rule //frobber/bin:thingy refers. +# +# Our friends are packages //frobber, //fribber and any subpackage of //fribber. +package_group( + name = "friends", + packages = [ + "//fribber/...", + "//frobber", + ], +) +</pre></li> +</ul> + + +<h3 id="common-attributes-tests">Attributes common to all test rules (*_test)</h3> + +<p>This section describes attributes that are common to all test rules.</p> + +<ul> +<li id="test.args"><code>args</code>: +Add these arguments to the <code>--test_arg</code> +when executed by <code>bazel test</code>. +<i>(List of strings; optional; subject to +<a href="#make_variables">"Make variable"</a> substitution and +<a href="#sh-tokenization">Bourne shell tokenization</a>)</i><br/> +These arguments are passed before the <code>--test_arg</code> values +specified on the <code>bazel test</code> command line.</li> +<li id="test.flaky"><code>flaky</code>: +Marks test as flaky. <i>(Boolean; optional)</i><br/> +If set, executes the test up to 3 times before being declared as failed. +By default this attribute is set to 0 and test is considered to be stable. +Note, that use of this attribute is generally discouraged - we do prefer +all tests to be stable.</li> +<li id="test.local"><code>local</code>: +Forces the test to be run locally. <i>(Boolean; optional)</i><br/> +By default this attribute is set to 0 and the default testing strategy is +used. This is equivalent to providing 'local' as a tag +(<code>tags=["local"]</code>).</li> +<li id="test.shard_count"><code>shard_count</code>: +Specifies the number of parallel shards +to use to run the test. <i>(Non-negative integer less than or equal to 50; +optional)</i><br/> +This value will override any heuristics used to determine the number of +parallel shards with which to run the test.</li> +<li id="test.size"><code>size</code>: +How "heavy" the test is +<i>(String "enormous", "large" "medium" or "small", +default is "medium")</i><br/> +A classification of the test's "heaviness": how much time/resources +it needs to run. This is useful when deciding which tests to run. +Before checking in a change, you might run the small tests. +Before a big release, you might run the large tests. +</li> +<li id="test.timeout"><code>timeout</code>: +How long the test is +normally expected to run before returning. +<i>(String "eternal", "long", "moderate", or "short" +with the default derived from a test's size attribute)</i><br/> +While a test's size attribute controls resource estimation, a test's +timeout may be set independently. If not explicitly specified, the +timeout is based on the test's size (with "small" ⇒ "short", +"medium" ⇒ "moderate", etc...). While size and runtime are generally +heavily correlated, they are not strictly causal, hence the ability to set +them independently.</li> +</ul> + + +<h3 id="common-attributes-binaries">Attributes common to all binary rules (*_binary)</h3> + +<p>This section describes attributes that are common to all binary rules.</p> + +<ul> +<li id="binary.args"><code>args</code>: +Add these arguments to the target when executed by +<code>bazel run</code>. +<i>(List of strings; optional; subject to +<a href="#make_variables">"Make variable"</a> substitution and +<a href="#sh-tokenization">Bourne shell tokenization</a>)</i><br/> +These arguments are passed to the target before the target options +specified on the <code>bazel run</code> command line. +<p>Most binary rules permit an <code>args</code> attribute, but where +this attribute is not allowed, this fact is documented under the +specific rule.</p></li> +<li id="binary.output_licenses"><code>output_licenses</code>: +The licenses of the output files that this binary generates. +<i>(List of strings; optional)</i><br/> +Describes the licenses of the output of the binary generated by +the rule. When a binary is referenced in a host attribute (for +example, the <code>tools</code> attribute of +a <code>genrule</code>), this license declaration is used rather +than the union of the licenses of its transitive closure. This +argument is useful when a binary is used as a tool during the +build of a rule, and it is not desirable for its license to leak +into the license of that rule. If this attribute is missing, the +license computation proceeds as if the host dependency was a +regular dependency. +<p><em class="harmful">WARNING: in some cases (specifically, in +genrules) the build tool cannot guarantee that the binary +referenced by this attribute is actually used as a tool, and is +not, for example, copied to the output. In these cases, it is the +responsibility of the user to make sure that this is +true.</em></p></li> +</ul> + + +<h3 id="implicit-outputs">Implicit output targets</h3> + +<p>When you define a build rule in a BUILD file, you are explicitly + declaring a new, named rule target in a package. Many build rule + functions also <i>implicitly</i> entail one or more output file + targets, whose contents and meaning are rule-specific. + + For example, when you explicitly declare a + <code>java_binary(name='foo', ...)</code> rule, you are also + <i>implicitly</i> declaring an output file + target <code>foo_deploy.jar</code> as a member of the same package. + (This particular target is a self-contained Java archive suitable + for deployment.) +</p> + +<p> + Implicit output targets are first-class members of the build + target graph. Just like other targets, they are built on demand, + either when specified in the top-level built command, or when they + are necessary prerequisites for other build targets. They can be + referenced as dependencies in BUILD files, and can be observed in + the output of analysis tools such as <code>bazel query</code>. +</p> + +<p> + For each kind of build rule, the rule's documentation contains a + special section detailing the names and contents of any implicit + outputs entailed by a declaration of that kind of rule. +</p> + +<p> + Please note an important but somewhat subtle distinction between the + two namespaces used by the build system. Build + <a href="build-ref.html#labels">labels</a> identify <em>targets</em>, + which may be rules or files, and file targets may be divided into + either source (or input) file targets and derived (or output) file + targets. These are the things you can mention in BUILD files, + build from the command-line, or examine using <code>bazel query</code>; + this is the <em>target namespace</em>. Each file target corresponds + to one actual file on disk (the "file system namespace"); each rule + target may correspond to zero, one or more actual files on disk. + There may be files on disk that have no corresponding target; for + example, <code>.o</code> object files produced during C++ compilation + cannot be referenced from within BUILD files or from the command line. + In this way, the build tool may hide certain implementation details of + how it does its job. This is explained more fully in + the <a href="build-ref.html">BUILD Concept Reference</a>. +</p>
diff --git a/src/main/java/com/google/devtools/build/lib/Constants.java b/src/main/java/com/google/devtools/build/lib/Constants.java new file mode 100644 index 0000000..052b090 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/Constants.java
@@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib; + +import com.google.common.collect.ImmutableList; + +/** + * Various constants required by Bazel. + * + * <p>The extra {@code .toString()} calls are there so that javac doesn't inline these constants + * so that we can replace this class file in the .jar after Bazel was built. + */ +public class Constants { + private Constants() { + } + + public static final String PRODUCT_NAME = "bazel".toString(); + public static final ImmutableList<String> DEFAULT_PACKAGE_PATH = ImmutableList.of("%workspace%"); + public static final String MAIN_RULE_CLASS_PROVIDER = + "com.google.devtools.build.lib.bazel.rules.BazelRuleClassProvider".toString(); + public static final ImmutableList<String> IGNORED_TEST_WARNING_PREFIXES = ImmutableList.of(); + public static final String RUNFILES_PREFIX = "".toString(); + + public static final ImmutableList<String> WATCHFS_BLACKLIST = ImmutableList.of(); + + public static final String PRELUDE_FILE_DEPOT_RELATIVE_PATH = "tools/build_rules/prelude_bazel"; +}
diff --git a/src/main/java/com/google/devtools/build/lib/UnixJniLoader.java b/src/main/java/com/google/devtools/build/lib/UnixJniLoader.java new file mode 100644 index 0000000..ca3ca3b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/UnixJniLoader.java
@@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib; + +import java.io.File; + +/** + * A class to load JNI dependencies for Bazel. + */ +public class UnixJniLoader { + public static void loadJni() { + try { + System.loadLibrary("unix"); + } catch (UnsatisfiedLinkError ex) { + // We are probably in tests, let's try to find the library relative to where we are. + File cwd = new File(System.getProperty("user.dir")); + String libunix = "src" + File.separator + "main" + File.separator + "native" + File.separator + + System.mapLibraryName("unix"); + File toTest = new File(cwd, libunix); + if (toTest.exists()) { + System.load(toTest.toString()); + } else { + throw ex; + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/AbstractAction.java b/src/main/java/com/google/devtools/build/lib/actions/AbstractAction.java new file mode 100644 index 0000000..87105d5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/AbstractAction.java
@@ -0,0 +1,420 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Symlinks; + +import java.io.IOException; +import java.util.Collection; + +/** + * Abstract implementation of Action which implements basic functionality: the + * inputs, outputs, and toString method. Both input and output sets are + * immutable. + */ +@Immutable @ThreadSafe +public abstract class AbstractAction implements Action { + + /** + * An arbitrary default resource set. Currently 250MB of memory, 50% CPU and 0% of total I/O. + */ + public static final ResourceSet DEFAULT_RESOURCE_SET = new ResourceSet(250, 0.5, 0); + + // owner/inputs/outputs attributes below should never be directly accessed even + // within AbstractAction itself. The appropriate getter methods should be used + // instead. This has to be done due to the fact that the getter methods can be + // overridden in subclasses. + private final ActionOwner owner; + // The variable inputs is non-final only so that actions that discover their inputs can modify it. + private Iterable<Artifact> inputs; + private final ImmutableSet<Artifact> outputs; + + private int cachedInputCount = -1; + private String cachedKey; + + /** + * Construct an abstract action with the specified inputs and outputs; + */ + protected AbstractAction(ActionOwner owner, + Iterable<Artifact> inputs, + Iterable<Artifact> outputs) { + Preconditions.checkNotNull(owner); + // TODO(bazel-team): Use RuleContext.actionOwner here instead + this.owner = new ActionOwnerDescription(owner); + this.inputs = CollectionUtils.makeImmutable(inputs); + this.outputs = ImmutableSet.copyOf(outputs); + Preconditions.checkArgument(!this.outputs.isEmpty(), owner); + } + + @Override + public final ActionOwner getOwner() { + return owner; + } + + @Override + public boolean inputsKnown() { + return true; + } + + @Override + public boolean discoversInputs() { + return false; + } + + @Override + public void discoverInputs(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + throw new IllegalStateException("discoverInputs cannot be called for " + this.prettyPrint() + + " since it does not discover inputs"); + } + + @Override + public void updateInputsFromCache( + ArtifactResolver artifactResolver, Collection<PathFragment> inputPaths) { + throw new IllegalStateException( + "Method must be overridden for actions that may have unknown inputs."); + } + + /** + * Should only be overridden by actions that need to optionally insert inputs. Actions that + * discover their inputs should use {@link #setInputs} to set the new iterable of inputs when they + * know it. + */ + @Override + public Iterable<Artifact> getInputs() { + return inputs; + } + + /** + * Set the inputs of the action. May only be used by an action that {@link #discoversInputs()}. + * The iterable passed in is automatically made immutable. + */ + public void setInputs(Iterable<Artifact> inputs) { + Preconditions.checkState(discoversInputs()); + this.inputs = CollectionUtils.makeImmutable(inputs); + cachedInputCount = -1; + } + + /* + * Get count of inputs. + * + * <p>Computes the count on first invocation, returns cached value for further invocations. + */ + @Override + @ThreadSafe + public synchronized int getInputCount() { + if (cachedInputCount == -1) { + cachedInputCount = Iterables.size(getInputs()); + } + return cachedInputCount; + } + + @Override + public ImmutableSet<Artifact> getOutputs() { + return outputs; + } + + @Override + public Artifact getPrimaryInput() { + // The default behavior is to return the first input artifact. + // Call through the method, not the field, because it may be overridden. + return Iterables.getFirst(getInputs(), null); + } + + @Override + public Artifact getPrimaryOutput() { + // Default behavior is to return the first output artifact. + // Use the method rather than field in case of overriding in subclasses. + return Iterables.getFirst(getOutputs(), null); + } + + @Override + public Iterable<Artifact> getMandatoryInputs() { + return getInputs(); + } + + @Override + public String toString() { + return prettyPrint() + " (" + getMnemonic() + "[" + ImmutableList.copyOf(getInputs()) + + (inputsKnown() ? " -> " : ", unknown inputs -> ") + + getOutputs() + "]" + ")"; + } + + @Override + public abstract String getMnemonic(); + protected abstract String computeKey(); + + @Override + public synchronized final String getKey() { + if (cachedKey == null) { + cachedKey = computeKey(); + } + return cachedKey; + } + + @Override + public String describeKey() { + return null; + } + + @Override + public boolean executeUnconditionally() { + return false; + } + + @Override + public boolean isVolatile() { + return false; + } + + @Override + public boolean showsOutputUnconditionally() { + return false; + } + + @Override + public final String getProgressMessage() { + String message = getRawProgressMessage(); + if (message == null) { + return null; + } + String additionalInfo = getOwner().getAdditionalProgressInfo(); + return additionalInfo == null ? message : message + " [" + additionalInfo + "]"; + } + + /** + * Returns a progress message string that is specific for this action. This is + * then annotated with additional information, currently the string '[for host]' + * for actions in the host configurations. + * + * <p>A return value of null indicates no message should be reported. + */ + protected String getRawProgressMessage() { + // A cheesy default implementation. Subclasses are invited to do something + // more meaningful. + return defaultProgressMessage(); + } + + private String defaultProgressMessage() { + return getMnemonic() + " " + getPrimaryOutput().prettyPrint(); + } + + @Override + public String prettyPrint() { + return "action '" + describe() + "'"; + } + + /** + * Deletes all of the action's output files, if they exist. If any of the + * Artifacts refers to a directory recursively removes the contents of the + * directory. + * + * @param execRoot the exec root in which this action is executed + */ + protected void deleteOutputs(Path execRoot) throws IOException { + for (Artifact output : getOutputs()) { + deleteOutput(output); + } + } + + /** + * Helper method to remove an Artifact. If the Artifact refers to a directory + * recursively removes the contents of the directory. + */ + protected void deleteOutput(Artifact output) throws IOException { + Path path = output.getPath(); + try { + // Optimize for the common case: output artifacts are files. + path.delete(); + } catch (IOException e) { + // Only try to recursively delete a directory if the output root is known. This is just a + // sanity check so that we do not start deleting random files on disk. + // TODO(bazel-team): Strengthen this test by making sure that the output is part of the + // output tree. + if (path.isDirectory(Symlinks.NOFOLLOW) && output.getRoot() != null) { + FileSystemUtils.deleteTree(path); + } else { + throw e; + } + } + } + + /** + * If the action might read directories as inputs in a way that is unsound wrt dependency + * checking, this method must be called. + */ + protected void checkInputsForDirectories(EventHandler eventHandler, + MetadataHandler metadataHandler) { + // Report "directory dependency checking" warning only for non-generated directories (generated + // ones will be reported earlier). + for (Artifact input : getMandatoryInputs()) { + // Assume that if the file did not exist, we would not have gotten here. + if (input.isSourceArtifact() && !metadataHandler.isRegularFile(input)) { + eventHandler.handle(Event.warn(getOwner().getLocation(), "input '" + + input.prettyPrint() + "' to " + getOwner().getLabel() + + " is a directory; dependency checking of directories is unsound")); + } + } + } + + @Override + public MiddlemanType getActionType() { + return MiddlemanType.NORMAL; + } + + /** + * If the action might create directories as outputs this method must be called. + */ + protected void checkOutputsForDirectories(EventHandler eventHandler) { + for (Artifact output : getOutputs()) { + Path path = output.getPath(); + String ownerString = Label.print(getOwner().getLabel()); + if (path.isDirectory()) { + eventHandler.handle(new Event(EventKind.WARNING, getOwner().getLocation(), + "output '" + output.prettyPrint() + "' of " + ownerString + + " is a directory; dependency checking of directories is unsound", + ownerString)); + } + } + } + + @Override + public void prepare(Path execRoot) throws IOException { + deleteOutputs(execRoot); + } + + @Override + public String describe() { + String progressMessage = getProgressMessage(); + return progressMessage != null ? progressMessage : defaultProgressMessage(); + } + + @Override + public abstract ResourceSet estimateResourceConsumption(Executor executor); + + @Override + public boolean shouldReportPathPrefixConflict(Action action) { + return this != action; + } + + @Override + public ExtraActionInfo.Builder getExtraActionInfo() { + return ExtraActionInfo.newBuilder() + .setOwner(getOwner().getLabel().toString()) + .setId(getKey()) + .setMnemonic(getMnemonic()); + } + + /** + * Returns input files that need to be present to allow extra_action rules to shadow this action + * correctly when run remotely. This is at least the normal inputs of the action, but may include + * other files as well. For example C(++) compilation may perform include file header scanning. + * This needs to be mirrored by the extra_action rule. Called by + * {@link com.google.devtools.build.lib.rules.extra.ExtraAction} at execution time. + * + * <p>As this method is called from the ExtraAction, make sure it is ok to call + * this method from a different thread than the one this action is executed on. + * + * @param actionExecutionContext Services in the scope of the action, like the Out/Err streams. + * @throws ActionExecutionException only when code called from this method + * throws that exception. + * @throws InterruptedException if interrupted + */ + public Iterable<Artifact> getInputFilesForExtraAction( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + return getInputs(); + } + + /** + * A copying implementation of {@link ActionOwner}. + * + * <p>ConfiguredTargets implement ActionOwner themselves, but we do not want actions + * to keep direct references to configured targets just for a label and a few strings. + */ + @Immutable + private static class ActionOwnerDescription implements ActionOwner { + + private final Location location; + private final Label label; + private final String configurationName; + private final String configurationMnemonic; + private final String configurationKey; + private final String targetKind; + private final String additionalProgressInfo; + + private ActionOwnerDescription(ActionOwner originalOwner) { + this.location = originalOwner.getLocation(); + this.label = originalOwner.getLabel(); + this.configurationName = originalOwner.getConfigurationName(); + this.configurationMnemonic = originalOwner.getConfigurationMnemonic(); + this.configurationKey = originalOwner.getConfigurationShortCacheKey(); + this.targetKind = originalOwner.getTargetKind(); + this.additionalProgressInfo = originalOwner.getAdditionalProgressInfo(); + } + + @Override + public Location getLocation() { + return location; + } + + @Override + public Label getLabel() { + return label; + } + + @Override + public String getConfigurationName() { + return configurationName; + } + + @Override + public String getConfigurationMnemonic() { + return configurationMnemonic; + } + + @Override + public String getConfigurationShortCacheKey() { + return configurationKey; + } + + @Override + public String getTargetKind() { + return targetKind; + } + + @Override + public String getAdditionalProgressInfo() { + return additionalProgressInfo; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/AbstractActionOwner.java b/src/main/java/com/google/devtools/build/lib/actions/AbstractActionOwner.java new file mode 100644 index 0000000..0272160 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/AbstractActionOwner.java
@@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Label; + +/** + * An action owner base class that provides default implementations for some of + * the {@link ActionOwner} methods. + */ +public abstract class AbstractActionOwner implements ActionOwner { + + @Override + public String getAdditionalProgressInfo() { + return null; + } + + @Override + public Location getLocation() { + return null; + } + + @Override + public Label getLabel() { + return null; + } + + @Override + public String getTargetKind() { + return "empty target kind"; + } + + @Override + public String getConfigurationName() { + return "empty configuration"; + } + + /** + * An action owner for special cases. Usage is strongly discouraged. + */ + public static final ActionOwner SYSTEM_ACTION_OWNER = new AbstractActionOwner() { + @Override + public final String getConfigurationName() { + return "system"; + } + + @Override + public String getConfigurationMnemonic() { + return "system"; + } + + @Override + public final String getConfigurationShortCacheKey() { + return "system"; + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/Action.java b/src/main/java/com/google/devtools/build/lib/actions/Action.java new file mode 100644 index 0000000..4970203 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/Action.java
@@ -0,0 +1,188 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadCompatible; +import com.google.devtools.build.lib.profiler.Describable; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.Collection; + +import javax.annotation.Nullable; + +/** + * An Action represents a function from Artifacts to Artifacts executed as an + * atomic build step. Examples include compilation of a single C++ source + * file, or linking a single library. + */ +public interface Action extends ActionMetadata, Describable { + + /** + * Prepares for executing this action; called by the Builder prior to + * executing the Action itself. This method should prepare the file system, so + * that the execution of the Action can write the output files. At a minimum + * any pre-existing and write protected output files should be removed or the + * permissions should be changed, so that they can be safely overwritten by + * the action. + * + * @throws IOException if there is an error deleting the outputs. + */ + void prepare(Path execRoot) throws IOException; + + /** + * Executes this action; called by the Builder when all of this Action's + * inputs have been successfully created. (Behaviour is undefined if the + * prerequisites are not up to date.) This method <i>actually does the work + * of the Action, unconditionally</i>; in other words, it is invoked by the + * Builder only when dependency analysis has deemed it necessary.</p> + * + * <p>The framework guarantees that the output directory for each file in + * <code>getOutputs()</code> has already been created, and will check to + * ensure that each of those files is indeed created.</p> + * + * <p>Implementations of this method should try to honour the {@link + * java.lang.Thread#interrupted} contract: if an interrupt is delivered to + * the thread in which execution occurs, the action should detect this on a + * best-effort basis and terminate as quickly as possible by throwing an + * ActionExecutionException. + * + * <p>Action execution must be ThreadCompatible in order to be safely used + * with a concurrent Builder implementation such as ParallelBuilder. + * + * @param actionExecutionContext Services in the scope of the action, like the output and error + * streams to use for messages arising during action execution. + * @throws ActionExecutionException if execution fails for any reason. + * @throws InterruptedException + */ + @ConditionallyThreadCompatible + void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException; + + /** + * Returns true iff action must be executed regardless of its current state. + * Default implementation can be overridden by some actions that might be + * executed unconditionally under certain circumstances - e.g., if caching of + * test results is not requested, this method could be used to force test + * execution even if all dependencies are up-to-date. + * + * <p>Note, it is <b>very</b> important not to abuse this method, since it + * completely overrides dependency checking. Any use of this method must + * be carefully reviewed and proved to be necessary. + * + * <p>Note that the definition of {@link #isVolatile} depends on the + * definition of this method, so be sure to consider both methods together + * when making changes. + */ + boolean executeUnconditionally(); + + /** + * Returns true if it's ever possible that {@link #executeUnconditionally} + * could evaluate to true during the lifetime of this instance, false + * otherwise. + */ + boolean isVolatile(); + + /** + * Method used to find inputs before execution for an action that + * {@link ActionMetadata#discoversInputs}. + */ + public void discoverInputs(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException; + + /** + * Method used to update action inputs based on the information contained in + * the action cache. It will be called iff inputsKnown() is false for the + * given action instance and there is a related cache entry in the action + * cache. + * + * Method must be redefined for any action that may return + * inputsKnown() == false. It also expects that implementation will ensure + * that inputsKnown() returns true after call to this method. + * + * @param artifactResolver the artifact factory that can be used to manufacture artifacts + * @param inputPaths List of relative (to the execution root) input paths + */ + public void updateInputsFromCache( + ArtifactResolver artifactResolver, Collection<PathFragment> inputPaths); + + /** + * Return a best-guess estimate of the operation's resource consumption on the + * local host itself for use in scheduling. + * + * @param executor the application-specific value passed to the + * executor parameter of the top-level call to + * Builder.buildArtifacts(). + */ + @Nullable ResourceSet estimateResourceConsumption(Executor executor); + + /** + * @return true iff path prefix conflict (conflict where two actions generate + * two output artifacts with one of the artifact's path being the + * prefix for another) between this action and another action should + * be reported. + */ + boolean shouldReportPathPrefixConflict(Action action); + + /** + * Returns true if the output should bypass output filtering. This is used for test actions. + */ + boolean showsOutputUnconditionally(); + + /** + * Called by {@link com.google.devtools.build.lib.rules.extra.ExtraAction} at execution time to + * extract information from this action into a protocol buffer to be used by extra_action rules. + * + * <p>As this method is called from the ExtraAction, make sure it is ok to call this method from + * a different thread than the one this action is executed on. + */ + ExtraActionInfo.Builder getExtraActionInfo(); + + /** + * Returns the action type. Must not be {@code null}. + */ + MiddlemanType getActionType(); + + /** + * The action type. + */ + public enum MiddlemanType { + + /** A normal action. */ + NORMAL, + + /** A normal middleman, which just encapsulates a list of artifacts. */ + AGGREGATING_MIDDLEMAN, + + /** + * A middleman that enforces action ordering, is not validated by the dependency checker, but + * allows errors to be propagated. + */ + ERROR_PROPAGATING_MIDDLEMAN, + + /** + * A runfiles middleman, which is validated by the dependency checker, but is not expanded + * in blaze. Instead, the runfiles manifest is sent to remote execution client, which + * performs the expansion. + */ + RUNFILES_MIDDLEMAN; + + public boolean isMiddleman() { + return this != NORMAL; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java new file mode 100644 index 0000000..2525c8d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java
@@ -0,0 +1,341 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action.MiddlemanType; +import com.google.devtools.build.lib.actions.cache.ActionCache; +import com.google.devtools.build.lib.actions.cache.Digest; +import com.google.devtools.build.lib.actions.cache.Metadata; +import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Checks whether an {@link Action} needs to be executed, or whether it has not changed since it was + * last stored in the action cache. Must be informed of the new Action data after execution as well. + * + * <p>The fingerprint, input files names, and metadata (either mtimes or MD5sums) of each action are + * cached in the action cache to avoid unnecessary rebuilds. Middleman artifacts are handled + * specially, avoiding the need to create actual files corresponding to the middleman artifacts. + * Instead of that, results of MiddlemanAction dependency checks are cached internally and then + * reused whenever an input middleman artifact is encountered. + * + * <p>While instances of this class hold references to action and metadata cache instances, they are + * otherwise lightweight, and should be constructed anew and discarded for each build request. + */ +public class ActionCacheChecker { + private final ActionCache actionCache; + private final Predicate<? super Action> executionFilter; + private final ArtifactResolver artifactResolver; + // True iff --verbose_explanations flag is set. + private final boolean verboseExplanations; + + public ActionCacheChecker(ActionCache actionCache, ArtifactResolver artifactResolver, + Predicate<? super Action> executionFilter, boolean verboseExplanations) { + this.actionCache = actionCache; + this.executionFilter = executionFilter; + this.artifactResolver = artifactResolver; + this.verboseExplanations = verboseExplanations; + } + + public boolean isActionExecutionProhibited(Action action) { + return !executionFilter.apply(action); + } + + /** + * Checks whether one of existing output paths is already used as a key. + * If yes, returns it - otherwise uses first output file as a key + */ + private ActionCache.Entry getCacheEntry(Action action) { + for (Artifact output : action.getOutputs()) { + ActionCache.Entry entry = actionCache.get(output.getExecPathString()); + if (entry != null) { + return entry; + } + } + return null; + } + + /** + * Validate metadata state for action input or output artifacts. + * + * @param entry cached action information. + * @param action action to be validated. + * @param metadataHandler provider of metadata for the artifacts this action interacts with. + * @param checkOutput true to validate output artifacts, Otherwise, just + * validate inputs. + * + * @return true if at least one artifact has changed, false - otherwise. + */ + private boolean validateArtifacts(ActionCache.Entry entry, Action action, + MetadataHandler metadataHandler, boolean checkOutput) { + Iterable<Artifact> artifacts = checkOutput + ? Iterables.concat(action.getOutputs(), action.getInputs()) + : action.getInputs(); + Map<String, Metadata> mdMap = new HashMap<>(); + for (Artifact artifact : artifacts) { + mdMap.put(artifact.getExecPathString(), metadataHandler.getMetadataMaybe(artifact)); + } + return !Digest.fromMetadata(mdMap).equals(entry.getFileDigest()); + } + + private void reportCommand(EventHandler handler, Action action) { + if (handler != null) { + if (verboseExplanations) { + String keyDescription = action.describeKey(); + reportRebuild(handler, action, + keyDescription == null ? "action command has changed" : + "action command has changed.\nNew action: " + keyDescription); + } else { + reportRebuild(handler, action, + "action command has changed (try --verbose_explanations for more info)"); + } + } + } + + protected boolean unconditionalExecution(Action action) { + return !isActionExecutionProhibited(action) && action.executeUnconditionally(); + } + + /** + * Checks whether {@code action} needs to be executed and returns a non-null Token if so. + * + * <p>The method checks if any of the action's inputs or outputs have changed. Returns a non-null + * {@link Token} if the action needs to be executed, and null otherwise. + * + * <p>If this method returns non-null, indicating that the action will be executed, the + * metadataHandler's {@link MetadataHandler#discardMetadata} method must be called, so that it + * does not serve stale metadata for the action's outputs after the action is executed. + */ + // Note: the handler should only be used for DEPCHECKER events; there's no + // guarantee it will be available for other events. + public Token getTokenIfNeedToExecute(Action action, EventHandler handler, + MetadataHandler metadataHandler) { + // TODO(bazel-team): (2010) For RunfilesAction/SymlinkAction and similar actions that + // produce only symlinks we should not check whether inputs are valid at all - all that matters + // that inputs and outputs are still exist (and new inputs have not appeared). All other checks + // are unnecessary. In other words, the only metadata we should check for them is file existence + // itself. + + MiddlemanType middlemanType = action.getActionType(); + if (middlemanType.isMiddleman()) { + // Some types of middlemen are not checked because they should not + // propagate invalidation of their inputs. + if (middlemanType != MiddlemanType.ERROR_PROPAGATING_MIDDLEMAN) { + checkMiddlemanAction(action, handler, metadataHandler); + } + return null; + } + ActionCache.Entry entry = null; // Populated lazily. + + // Update action inputs from cache, if necessary. + boolean inputsKnown = action.inputsKnown(); + if (!inputsKnown) { + Preconditions.checkState(action.discoversInputs()); + entry = getCacheEntry(action); + updateActionInputs(action, entry); + } + if (mustExecute(action, entry, handler, metadataHandler)) { + return new Token(getKeyString(action)); + } + return null; + } + + protected boolean mustExecute(Action action, @Nullable ActionCache.Entry entry, + EventHandler handler, MetadataHandler metadataHandler) { + // Unconditional execution can be applied only for actions that are allowed to be executed. + if (unconditionalExecution(action)) { + Preconditions.checkState(action.isVolatile()); + reportUnconditionalExecution(handler, action); + return true; // must execute - unconditional execution is requested. + } + + if (entry == null) { + entry = getCacheEntry(action); + } + if (entry == null) { + reportNewAction(handler, action); + return true; // must execute -- no cache entry (e.g. first build) + } + + if (entry.isCorrupted()) { + reportCorruptedCacheEntry(handler, action); + return true; // cache entry is corrupted - must execute + } else if (validateArtifacts(entry, action, metadataHandler, true)) { + reportChanged(handler, action); + return true; // files have changed + } else if (!entry.getActionKey().equals(action.getKey())){ + reportCommand(handler, action); + return true; // must execute -- action key is different + } + + entry.getFileDigest(); + return false; // cache hit + } + + public void afterExecution(Action action, Token token, MetadataHandler metadataHandler) + throws IOException { + Preconditions.checkArgument(token != null); + String key = token.cacheKey; + ActionCache.Entry entry = actionCache.createEntry(action.getKey()); + for (Artifact output : action.getOutputs()) { + // Remove old records from the cache if they used different key. + String execPath = output.getExecPathString(); + if (!key.equals(execPath)) { + actionCache.remove(key); + } + // Output files *must* exist and be accessible after successful action execution. + Metadata metadata = metadataHandler.getMetadata(output); + Preconditions.checkState(metadata != null); + entry.addFile(output.getExecPath(), metadata); + } + for (Artifact input : action.getInputs()) { + entry.addFile(input.getExecPath(), metadataHandler.getMetadataMaybe(input)); + } + entry.getFileDigest(); + actionCache.put(key, entry); + } + + protected void updateActionInputs(Action action, ActionCache.Entry entry) { + if (entry == null || entry.isCorrupted()) { + return; + } + + List<PathFragment> outputs = new ArrayList<>(); + for (Artifact output : action.getOutputs()) { + outputs.add(output.getExecPath()); + } + List<PathFragment> inputs = new ArrayList<>(); + for (String path : entry.getPaths()) { + PathFragment execPath = new PathFragment(path); + // Code assumes that action has only 1-2 outputs and ArrayList.contains() will be + // most efficient. + if (!outputs.contains(execPath)) { + inputs.add(execPath); + } + } + action.updateInputsFromCache(artifactResolver, inputs); + } + + /** + * Special handling for the MiddlemanAction. Since MiddlemanAction output + * artifacts are purely fictional and used only to stay within dependency + * graph model limitations (action has to depend on artifacts, not on other + * actions), we do not need to validate metadata for the outputs - only for + * inputs. We also do not need to validate MiddlemanAction key, since action + * cache entry key already incorporates that information for the middlemen + * and we will experience a cache miss when it is different. Whenever it + * encounters middleman artifacts as input artifacts for other actions, it + * consults with the aggregated middleman digest computed here. + */ + protected void checkMiddlemanAction(Action action, EventHandler handler, + MetadataHandler metadataHandler) { + Artifact middleman = action.getPrimaryOutput(); + String cacheKey = middleman.getExecPathString(); + ActionCache.Entry entry = actionCache.get(cacheKey); + boolean changed = false; + if (entry != null) { + if (entry.isCorrupted()) { + reportCorruptedCacheEntry(handler, action); + changed = true; + } else if (validateArtifacts(entry, action, metadataHandler, false)) { + reportChanged(handler, action); + changed = true; + } + } else { + reportChangedDeps(handler, action); + changed = true; + } + if (changed) { + // Compute the aggregated middleman digest. + // Since we never validate action key for middlemen, we should not store + // it in the cache entry and just use empty string instead. + entry = actionCache.createEntry(""); + for (Artifact input : action.getInputs()) { + entry.addFile(input.getExecPath(), metadataHandler.getMetadataMaybe(input)); + } + } + + metadataHandler.setDigestForVirtualArtifact(middleman, entry.getFileDigest()); + if (changed) { + actionCache.put(cacheKey, entry); + } + } + + /** + * Returns an action key. It is always set to the first output exec path string. + */ + private static String getKeyString(Action action) { + Preconditions.checkState(!action.getOutputs().isEmpty()); + return action.getOutputs().iterator().next().getExecPathString(); + } + + + /** + * In most cases, this method should not be called directly - reportXXX() methods + * should be used instead. This is done to avoid cost associated with building + * the message. + */ + private static void reportRebuild(@Nullable EventHandler handler, Action action, String message) { + // For MiddlemanAction, do not report rebuild. + if (handler != null && !action.getActionType().isMiddleman()) { + handler.handle(new Event( + EventKind.DEPCHECKER, null, "Executing " + action.prettyPrint() + ": " + message + ".")); + } + } + + // Called by IncrementalDependencyChecker. + protected static void reportUnconditionalExecution( + @Nullable EventHandler handler, Action action) { + reportRebuild(handler, action, "unconditional execution is requested"); + } + + private static void reportChanged(@Nullable EventHandler handler, Action action) { + reportRebuild(handler, action, "One of the files has changed"); + } + + private static void reportChangedDeps(@Nullable EventHandler handler, Action action) { + reportRebuild(handler, action, "the set of files on which this action depends has changed"); + } + + private static void reportNewAction(@Nullable EventHandler handler, Action action) { + reportRebuild(handler, action, "no entry in the cache (action is new)"); + } + + private static void reportCorruptedCacheEntry(@Nullable EventHandler handler, Action action) { + reportRebuild(handler, action, "cache entry is corrupted"); + } + + /** Wrapper for all context needed by the ActionCacheChecker to handle a single action. */ + public static final class Token { + private final String cacheKey; + + private Token(String cacheKey) { + this.cacheKey = Preconditions.checkNotNull(cacheKey); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionCompletionEvent.java b/src/main/java/com/google/devtools/build/lib/actions/ActionCompletionEvent.java new file mode 100644 index 0000000..a2cb577 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionCompletionEvent.java
@@ -0,0 +1,33 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * An event that is fired after an action completes (either successfully or not). + */ +public final class ActionCompletionEvent { + + private final ActionMetadata actionMetadata; + + public ActionCompletionEvent(ActionMetadata actionMetadata) { + this.actionMetadata = actionMetadata; + } + + /** + * Returns the action metadata. + */ + public ActionMetadata getActionMetadata() { + return actionMetadata; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionContextConsumer.java b/src/main/java/com/google/devtools/build/lib/actions/ActionContextConsumer.java new file mode 100644 index 0000000..4c0eaa2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionContextConsumer.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +import java.util.Map; + +/** + * An object describing that actions require a particular implementation of an + * {@link ActionContext}. + * + * <p>This is expected to be implemented by modules that also implement actions which need these + * contexts. Other modules will provide implementations for various action contexts by implementing + * {@link ActionContextProvider}. + * + * <p>Example: a module requires {@code SpawnActionContext} to do its job, and it creates + * actions with the mnemonic <code>C++</code>. Then the {@link #getSpawnActionContexts} method of + * this module would return a map with the key <code>"C++"</code> in it. + * + * <p>The module can either decide for itself which implementation is needed and make the value + * associated with this key a constant or defer that decision to the user, for example, by + * providing a command line option and setting the value in the map based on that. + * + * <p>Other modules are free to provide different implementations of {@code SpawnActionContext}. + * This can be used, for example, to implement sandboxed or distributed execution of + * {@code SpawnAction}s in different ways, while giving the user control over how exactly they + * are executed. + */ +public interface ActionContextConsumer { + /** + * Returns a map from spawn action mnemonics created by this module to the name of the + * implementation of {@code SpawnActionContext} that the module wants to use for executing + * it. + * + * <p>If a spawn action is executed whose mnemonic maps to the empty string or is not + * present in the map at all, the choice of the implementation is left to Blaze. + */ + public Map<String, String> getSpawnActionContexts(); + + /** + * Returns a map from action context class to the implementation required by the module. + * + * <p>If the implementation name is the empty string, the choice is left to Blaze. + */ + public Map<Class<? extends ActionContext>, String> getActionContexts(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionContextMarker.java b/src/main/java/com/google/devtools/build/lib/actions/ActionContextMarker.java new file mode 100644 index 0000000..fcb2e3d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionContextMarker.java
@@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation for action contexts. Actions contexts should also implement {@link ActionContext}. + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface ActionContextMarker { + String name(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/actions/ActionContextProvider.java new file mode 100644 index 0000000..6e52a4a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionContextProvider.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +/** + * An object that provides execution strategies to {@link BlazeExecutor}. + * + * <p>For more information, see {@link ActionContextConsumer}. + */ +public interface ActionContextProvider { + /** + * Returns the execution strategies that are provided by this object. + * + * <p>These may or may not actually end up in the executor depending on the command line options + * and other factors influencing how the executor is set up. + */ + Iterable<ActionContext> getActionContexts(); + + /** + * Called when the executor is constructed. The parameter contains all the contexts that were + * selected for this execution phase. + */ + void executorCreated(Iterable<ActionContext> usedContexts) throws ExecutorInitException; + + /** + * Called when the execution phase is started. + */ + void executionPhaseStarting( + ActionInputFileCache actionInputFileCache, + ActionGraph actionGraph, + Iterable<Artifact> topLevelArtifacts) + throws ExecutorInitException, InterruptedException; + + /** + * Called when the execution phase is finished. + */ + void executionPhaseEnding(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutedEvent.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutedEvent.java new file mode 100644 index 0000000..00ef9b4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutedEvent.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * This event is fired during the build, when an action is executed. It contains information about + * the action: the Action itself, and the output file names its stdout and stderr are recorded in. + */ +public class ActionExecutedEvent { + private final Action action; + private final ActionExecutionException exception; + private final String stdout; + private final String stderr; + + public ActionExecutedEvent(Action action, + ActionExecutionException exception, String stdout, String stderr) { + this.action = action; + this.exception = exception; + this.stdout = stdout; + this.stderr = stderr; + } + + public Action getAction() { + return action; + } + + // null if action succeeded + public ActionExecutionException getException() { + return exception; + } + + public String getStdout() { + return stdout; + } + + public String getStderr() { + return stderr; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java new file mode 100644 index 0000000..d6d08fa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java
@@ -0,0 +1,73 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.actions.Artifact.MiddlemanExpander; +import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.util.io.FileOutErr; + +/** + * A class that groups services in the scope of the action. Like the FileOutErr object. + */ +public class ActionExecutionContext { + + private final Executor executor; + private final ActionInputFileCache actionInputFileCache; + private final MetadataHandler metadataHandler; + private final FileOutErr fileOutErr; + private final MiddlemanExpander middlemanExpander; + + public ActionExecutionContext(Executor executor, ActionInputFileCache actionInputFileCache, + MetadataHandler metadataHandler, FileOutErr fileOutErr, MiddlemanExpander middlemanExpander) { + this.actionInputFileCache = actionInputFileCache; + this.metadataHandler = metadataHandler; + this.fileOutErr = fileOutErr; + this.executor = executor; + this.middlemanExpander = middlemanExpander; + } + + public ActionInputFileCache getActionInputFileCache() { + return actionInputFileCache; + } + + public MetadataHandler getMetadataHandler() { + return metadataHandler; + } + + public Executor getExecutor() { + return executor; + } + + public MiddlemanExpander getMiddlemanExpander() { + return middlemanExpander; + } + + /** + * Provide that {@code FileOutErr} that the action should use for redirecting the output and error + * stream. + */ + public FileOutErr getFileOutErr() { + return fileOutErr; + } + + /** + * Allows us to create a new context that overrides the FileOutErr with another one. This is + * useful for muting the output for example. + */ + public ActionExecutionContext withFileOutErr(FileOutErr fileOutErr) { + return new ActionExecutionContext(executor, actionInputFileCache, metadataHandler, fileOutErr, + middlemanExpander); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionException.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionException.java new file mode 100644 index 0000000..0d0d908 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionException.java
@@ -0,0 +1,113 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Label; + +/** + * This exception gets thrown if {@link Action#execute(ActionExecutionContext)} is unsuccessful. + * Typically these are re-raised ExecException throwables. + */ +@ThreadSafe +public class ActionExecutionException extends Exception { + + private final Action action; + private final NestedSet<Label> rootCauses; + private final boolean catastrophe; + + public ActionExecutionException(Throwable cause, Action action, boolean catastrophe) { + super(cause.getMessage(), cause); + this.action = action; + this.rootCauses = rootCausesFromAction(action); + this.catastrophe = catastrophe; + } + + public ActionExecutionException(String message, + Throwable cause, Action action, boolean catastrophe) { + super(message + ": " + cause.getMessage(), cause); + this.action = action; + this.rootCauses = rootCausesFromAction(action); + this.catastrophe = catastrophe; + } + + public ActionExecutionException(String message, Action action, boolean catastrophe) { + super(message); + this.action = action; + this.rootCauses = rootCausesFromAction(action); + this.catastrophe = catastrophe; + } + + public ActionExecutionException(String message, Action action, + NestedSet<Label> rootCauses, boolean catastrophe) { + super(message); + this.action = action; + this.rootCauses = rootCauses; + this.catastrophe = catastrophe; + } + + public ActionExecutionException(String message, Throwable cause, Action action, + NestedSet<Label> rootCauses, boolean catastrophe) { + super(message, cause); + this.action = action; + this.rootCauses = rootCauses; + this.catastrophe = catastrophe; + } + + static NestedSet<Label> rootCausesFromAction(Action action) { + return action == null || action.getOwner() == null || action.getOwner().getLabel() == null + ? NestedSetBuilder.<Label>emptySet(Order.STABLE_ORDER) + : NestedSetBuilder.create(Order.STABLE_ORDER, action.getOwner().getLabel()); + } + + /** + * Returns the action that failed. + */ + public Action getAction() { + return action; + } + + /** + * Return the root causes that should be reported. Usually the owner of the action, but it can + * be the label of a missing artifact. + */ + public NestedSet<Label> getRootCauses() { + return rootCauses; + } + + /** + * Returns the location of the owner of this action. May be null. + */ + public Location getLocation() { + return action.getOwner().getLocation(); + } + + /** + * Catastrophic exceptions should stop builds, even if --keep_going. + */ + public boolean isCatastrophe() { + return catastrophe; + } + + /** + * Returns true if the error should be shown. + */ + public boolean showError() { + return getMessage() != null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporter.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporter.java new file mode 100644 index 0000000..34aadc4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporter.java
@@ -0,0 +1,262 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.Pair; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; + +import javax.annotation.Nullable; + +/** + * Implements "Still waiting..." message functionality, displaying current status for "in-flight" + * actions. Used by the ParallelBuilder. + * + * TODO(bazel-team): (2010) It would be nice if "duplicated" actions (e.g. test shards and multiple + * test runs) were merged into the single line. + */ +@ThreadSafe +public final class ActionExecutionStatusReporter { + // Maximum number of lines to output per each status category before truncation. + private static final int MAX_LINES = 10; + + private final EventHandler eventHandler; + private final Executor executor; + private final EventBus eventBus; + private final Clock clock; + + /** + * The status of each action "in flight", i.e. whose ExecuteBuildAction.call() method is active. + * Used for implementing the "still waiting" message. + */ + private final Map<ActionMetadata, Pair<String, Long>> actionStatus = + new ConcurrentHashMap<>(100); + + public static ActionExecutionStatusReporter create(EventHandler eventHandler) { + return create(eventHandler, null, null); + } + + @VisibleForTesting + static ActionExecutionStatusReporter create(EventHandler eventHandler, Clock clock) { + return create(eventHandler, null, null, clock); + } + + public static ActionExecutionStatusReporter create(EventHandler eventHandler, + @Nullable Executor executor, @Nullable EventBus eventBus) { + return create(eventHandler, executor, eventBus, null); + } + + private static ActionExecutionStatusReporter create(EventHandler eventHandler, + @Nullable Executor executor, @Nullable EventBus eventBus, @Nullable Clock clock) { + ActionExecutionStatusReporter result = new ActionExecutionStatusReporter(eventHandler, executor, + eventBus, clock == null ? BlazeClock.instance() : clock); + if (eventBus != null) { + eventBus.register(result); + } + return result; + } + + private ActionExecutionStatusReporter(EventHandler eventHandler, @Nullable Executor executor, + @Nullable EventBus eventBus, Clock clock) { + this.eventHandler = Preconditions.checkNotNull(eventHandler); + this.executor = executor; + this.eventBus = eventBus; + this.clock = Preconditions.checkNotNull(clock); + } + + public void unregisterFromEventBus() { + if (eventBus != null) { + eventBus.unregister(this); + } + } + + private void setStatus(ActionMetadata action, String message) { + actionStatus.put(action, Pair.of(message, clock.nanoTime())); + } + + /** + * Remove action from the list of active actions. + */ + public void remove(Action action) { + Preconditions.checkNotNull(actionStatus.remove(action), action); + } + + /** + * Set "Preparing" status. + */ + public void setPreparing(Action action) { + updateStatus(ActionStatusMessage.preparingStrategy(action)); + } + + public void setRunningFromBuildData(ActionMetadata action) { + updateStatus(ActionStatusMessage.runningStrategy(action)); + } + + @Subscribe + public void updateStatus(ActionStatusMessage statusMsg) { + String message = statusMsg.getMessage(); + ActionMetadata action = statusMsg.getActionMetadata(); + if (statusMsg.needsStrategy()) { + String strategy = action.describeStrategy(executor); + if (strategy == null) { + return; + } + message = String.format(message, strategy); + } + setStatus(action, message); + } + + public int getCount() { + return actionStatus.size(); + } + + private static void appendGroupStatus(StringBuilder buffer, + Map<ActionMetadata, Pair<String, Long>> statusMap, String status, long currentTime) { + List<Pair<Long, ActionMetadata>> actions = new ArrayList<>(); + for (Map.Entry<ActionMetadata, Pair<String, Long>> entry : statusMap.entrySet()) { + if (entry.getValue().first.equals(status)) { + actions.add(Pair.of(entry.getValue().second, entry.getKey())); + } + } + if (actions.size() == 0) { + return; + } + Collections.sort(actions, Pair.<Long, ActionMetadata>compareByFirst()); + + buffer.append("\n " + status + ":"); + + boolean truncateList = actions.size() > MAX_LINES; + for (Pair<Long, ActionMetadata> entry : actions.subList(0, + truncateList ? MAX_LINES - 1 : actions.size())) { + String message = entry.second.getProgressMessage(); + if (message == null) { + // Actions will a null progress message should run so + // fast we never see them here. In any case... + message = entry.second.prettyPrint(); + } + buffer.append("\n ").append(message); + long runTime = (currentTime - entry.first) / 1000000000L; // Convert to seconds. + buffer.append(", ").append(runTime).append(" s"); + } + if (truncateList) { + buffer.append("\n ... ").append(actions.size() - MAX_LINES + 1).append(" more jobs"); + } + } + + /** + * Get message showing currently executing actions. + */ + private String getExecutionStatusMessage(Map<ActionMetadata, Pair<String, Long>> statusMap) { + int count = statusMap.size(); + StringBuilder s = count != 1 + ? new StringBuilder("Still waiting for ").append(count).append(" jobs to complete:") + : new StringBuilder("Still waiting for 1 job to complete:"); + + long currentTime = clock.nanoTime(); + + // A tree is just as fast as HashSet for small data sets. + Set<String> statuses = new TreeSet<String>(); + for (Map.Entry<ActionMetadata, Pair<String, Long>> entry : statusMap.entrySet()) { + statuses.add(entry.getValue().first); + } + + for (String status : statuses) { + appendGroupStatus(s, statusMap, status, currentTime); + } + return s.toString(); + } + + /** + * Show currently executing actions. + */ + public void showCurrentlyExecutingActions(String progressPercentageMessage) { + // Defensive copy to ensure thread safety. + Map<ActionMetadata, Pair<String, Long>> statusMap = new HashMap<>(actionStatus); + if (statusMap.size() > 0) { + eventHandler.handle( + Event.progress(progressPercentageMessage + getExecutionStatusMessage(statusMap))); + } + } + + /** + * Warn about actions that are still being executed. + * Method is used to produce informative message when build is interrupted. + */ + void warnAboutCurrentlyExecutingActions() { + // Defensive copy to ensure thread safety. + Map<ActionMetadata, Pair<String, Long>> statusMap = new HashMap<>(actionStatus); + if (statusMap.size() == 0) { + // There are no tasks in the queue so there is nothing to report. + eventHandler.handle(Event.warn("There are no active jobs - stopping the build")); + return; + } + Iterator<ActionMetadata> iterator = statusMap.keySet().iterator(); + while (iterator.hasNext()) { + // Filter out actions that are not executed yet. + if (statusMap.get(iterator.next()).first.equals(ActionStatusMessage.PREPARING)) { + iterator.remove(); + } + } + if (statusMap.size() > 0) { + eventHandler.handle(Event.warn(getExecutionStatusMessage(statusMap) + + "\nBuild will be stopped after these tasks terminate")); + } else { + // It is possible that one or more tasks in "Preparing" state just started being executed. + // So warn user just in case. + eventHandler.handle(Event.warn("Still waiting for unfinished jobs")); + } + } + + /** + * Returns the number of seconds to wait before reporting slow progress again. + * + * @param userSpecifiedProgressInterval value of the --progress_report_interval flag; 0 means + * use default 10, then 30, then 60 seconds wait times + * @param previousWaitTime previous value returned by this method + */ + public static int getWaitTime(int userSpecifiedProgressInterval, int previousWaitTime) { + if (userSpecifiedProgressInterval > 0) { + return userSpecifiedProgressInterval; + } + + // Increase waitTime to 10, then to 30 and then to 60 seconds to reduce + // spamming during long wait periods. If the user specified a + // waitTime directly through progressReportInterval, then use + // that value. + if (previousWaitTime == 0) { + return 10; + } else if (previousWaitTime == 10) { + return 30; + } else { + return 60; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionGraph.java b/src/main/java/com/google/devtools/build/lib/actions/ActionGraph.java new file mode 100644 index 0000000..bb2b707 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionGraph.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import javax.annotation.Nullable; + +/** + * An action graph. + * + * <p>Provides lookups of generating actions for artifacts. + */ +public interface ActionGraph { + + /** + * Returns the Action that, when executed, gives rise to this file. + * + * <p>If this Artifact is a source file, null is returned. (We don't try to return a "no-op + * action" because that would require creating a new no-op Action for every source file, since + * each Action knows its outputs, so sharing all the no-ops is not an option.) + * + * <p>It's also possible for derived Artifacts to have null generating Actions when these actions + * are unknown. + */ + @Nullable + Action getGeneratingAction(Artifact artifact); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionGraphVisitor.java b/src/main/java/com/google/devtools/build/lib/actions/ActionGraphVisitor.java new file mode 100644 index 0000000..4ddda4c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionGraphVisitor.java
@@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * An abstract visitor for the action graph. Specializes {@link BipartiteVisitor} for artifacts and + * actions, and takes care of visiting the complete transitive closure. + */ +public abstract class ActionGraphVisitor extends BipartiteVisitor<Action, Artifact> { + + private final ActionGraph actionGraph; + + public ActionGraphVisitor(ActionGraph actionGraph) { + this.actionGraph = actionGraph; + } + + /** + * Called for all artifacts in the visitation. Hook for subclasses. + * + * @param artifact + */ + protected void visitArtifact(Artifact artifact) {} + + /** + * Called for all actions in the visitation. Hook for subclasses. + * + * @param action + */ + protected void visitAction(Action action) {} + + /** + * Whether the given action should be visited. If this returns false, the visitation stops here, + * so the dependencies of this action are also not visited. + * + * @param action + */ + protected boolean shouldVisit(Action action) { + return true; + } + + /** + * Whether the given artifact should be visited. If this returns false, the visitation stops here, + * so dependencies of this artifact (if it is a generated one) are also not visited. + * + * @param artifact + */ + protected boolean shouldVisit(Artifact artifact) { + return true; + } + + @SuppressWarnings("unused") + protected final void visitArtifacts(Iterable<Artifact> artifacts) { + for (Artifact artifact : artifacts) { + visitArtifact(artifact); + } + } + + @Override protected void white(Artifact artifact) { + Action action = actionGraph.getGeneratingAction(artifact); + visitArtifact(artifact); + if (action != null && shouldVisit(action)) { + visitBlackNode(action); + } + } + + @Override protected void black(Action action) { + visitAction(action); + for (Artifact input : action.getInputs()) { + if (shouldVisit(input)) { + visitWhiteNode(input); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionInput.java b/src/main/java/com/google/devtools/build/lib/actions/ActionInput.java new file mode 100644 index 0000000..c370591 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionInput.java
@@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * Represents an input file to a build action, with an appropriate relative path and digest + * value. + * + * <p>Artifact is the only notable implementer of the interface, but the interface remains + * because 1) some Google specific rules ship files that could be Artifacts to remote execution + * by instantiating ad-hoc derived classes of ActionInput. 2) historically, Google C++ rules + * allow underspecified C++ builds. For that case, we have extra logic to guess the undeclared + * header inclusions (eg. computed inclusions). The extra logic lives in a file that is not + * needed for remote execution, but is a dependency, and it is inserted as a non-Artifact + * ActionInput. + * + * <p>ActionInput is used as a cache "key" for ActionInputFileCache: for Artifacts, the + * digest/size is already stored in Artifact, but for non-artifacts, we use getExecPathString + * to find this data in a filesystem related cache. + */ +public interface ActionInput { + + /** + * @return the relative path to the input file. + */ + public String getExecPathString(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionInputFileCache.java b/src/main/java/com/google/devtools/build/lib/actions/ActionInputFileCache.java new file mode 100644 index 0000000..b45e9cd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionInputFileCache.java
@@ -0,0 +1,77 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.protobuf.ByteString; + +import java.io.File; +import java.io.IOException; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +/** + * The interface for Action inputs metadata (Digest and size). + * + * NOTE: Implementations must be thread safe. + */ +@ThreadSafe +public interface ActionInputFileCache { + /** + * Returns digest for the given artifact. This digest is current as of some time t >= the start of + * the present build. If the artifact is an output of an action that already executed at time p, + * then t >= p. Aside from these properties, t can be any value and may vary arbitrarily across + * calls. + * + * @param input the input to retrieve the digest for + * @return the artifact's digest or null if digest cannot be obtained (due to artifact + * non-existence, lookup errors, or any other reason) + * + * @throws DigestOfDirectoryException in case {@code input} is a directory. + * @throws IOException If the file cannot be digested. + * + */ + @Nullable + ByteString getDigest(ActionInput input) throws IOException; + + /** + * Retrieve the size of the file at the given path. Will usually return 0 on failure instead of + * throwing an IOException. Returns 0 for files inaccessible to user, but available to the + * execution environment. + * + * @param input the input. + * @return the file size in bytes. + * @throws IOException on failure. + */ + long getSizeInBytes(ActionInput input) throws IOException; + + /** + * Checks if the file is available locally, based on the assumption that previous operations on + * the ActionInputFileCache would have created a cache entry for it. + * + * @param digest the digest to lookup. + * @return true if the specified digest is backed by a locally-readable file, false otherwise + */ + boolean contentsAvailableLocally(ByteString digest); + + /** + * Concrete subclasses must implement this to provide a mapping from digest to file path, + * based on files previously seen as inputs. + * + * @param digest the digest. + * @return a File path. + */ + @Nullable + File getFileFromDigest(ByteString digest) throws IOException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionInputHelper.java b/src/main/java/com/google/devtools/build/lib/actions/ActionInputHelper.java new file mode 100644 index 0000000..0fed928 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionInputHelper.java
@@ -0,0 +1,160 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Functions; +import com.google.common.base.Preconditions; +import com.google.common.collect.Collections2; +import com.google.common.collect.Iterables; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Helper utility to create ActionInput instances. + */ +public final class ActionInputHelper { + private ActionInputHelper() { + } + + @VisibleForTesting + public static Artifact.MiddlemanExpander actionGraphMiddlemanExpander( + final ActionGraph actionGraph) { + return new Artifact.MiddlemanExpander() { + @Override + public void expand(Artifact mm, Collection<? super Artifact> output) { + // Skyframe is stricter in that it checks that "mm" is a input of the action, because + // it cannot expand arbitrary middlemen without access to a global action graph. + // We could check this constraint here too, but it seems unnecessary. This code is + // going away anyway. + Preconditions.checkArgument(mm.isMiddlemanArtifact(), + "%s is not a middleman artifact", mm); + Action middlemanAction = actionGraph.getGeneratingAction(mm); + Preconditions.checkState(middlemanAction != null, mm); + // TODO(bazel-team): Consider expanding recursively or throwing an exception here. + // Most likely, this code will cause silent errors if we ever have a middleman that + // contains a middleman. + if (middlemanAction.getActionType() == Action.MiddlemanType.AGGREGATING_MIDDLEMAN) { + Artifact.addNonMiddlemanArtifacts(middlemanAction.getInputs(), output, + Functions.<Artifact>identity()); + } + + } + }; + } + + /** + * Most ActionInputs are created and never used again. On the off chance that one is, however, we + * implement equality via path comparison. Since file caches are keyed by ActionInput, equality + * checking does come up. + */ + private static class BasicActionInput implements ActionInput { + private final String path; + public BasicActionInput(String path) { + this.path = Preconditions.checkNotNull(path); + } + + @Override + public String getExecPathString() { + return path; + } + + @Override + public int hashCode() { + return path.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null) { + return false; + } + if (!this.getClass().equals(other.getClass())) { + return false; + } + return this.path.equals(((BasicActionInput) other).path); + } + + @Override + public String toString() { + return "BasicActionInput: " + path; + } + } + + /** + * Creates an ActionInput with just the given relative path and no digest. + * + * @param path the relative path of the input. + * @return a ActionInput. + */ + public static ActionInput fromPath(String path) { + return new BasicActionInput(path); + } + + private static final Function<String, ActionInput> FROM_PATH = + new Function<String, ActionInput>() { + @Override + public ActionInput apply(String path) { + return fromPath(path); + } + }; + + /** + * Creates a sequence of {@link ActionInput}s from a sequence of string paths. + */ + public static Collection<ActionInput> fromPaths(Collection<String> paths) { + return Collections2.transform(paths, FROM_PATH); + } + + /** + * Expands middleman artifacts in a sequence of {@link ActionInput}s. + * + * <p>Non-middleman artifacts are returned untouched. + */ + public static List<ActionInput> expandMiddlemen(Iterable<? extends ActionInput> inputs, + Artifact.MiddlemanExpander middlemanExpander) { + + List<ActionInput> result = new ArrayList<>(); + List<Artifact> containedArtifacts = new ArrayList<>(); + for (ActionInput input : inputs) { + if (!(input instanceof Artifact)) { + result.add(input); + continue; + } + containedArtifacts.add((Artifact) input); + } + Artifact.addExpandedArtifacts(containedArtifacts, result, middlemanExpander); + return result; + } + + /** Formatter for execPath String output. Public because Artifact uses it directly. */ + public static final Function<ActionInput, String> EXEC_PATH_STRING_FORMATTER = + new Function<ActionInput, String>() { + @Override + public String apply(ActionInput input) { + return input.getExecPathString(); + } + }; + + public static Iterable<String> toExecPaths(Iterable<? extends ActionInput> artifacts) { + return Iterables.transform(artifacts, EXEC_PATH_STRING_FORMATTER); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionLogBufferPathGenerator.java b/src/main/java/com/google/devtools/build/lib/actions/ActionLogBufferPathGenerator.java new file mode 100644 index 0000000..2c80e8a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionLogBufferPathGenerator.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.Path; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A source for generating unique action log paths. + */ +public final class ActionLogBufferPathGenerator { + + private final AtomicInteger actionCounter = new AtomicInteger(); + + private final Path actionOutputRoot; + + public ActionLogBufferPathGenerator(Path actionOutputRoot) { + this.actionOutputRoot = actionOutputRoot; + } + + /** + * Generates a unique filename for an action to store its output. + */ + public FileOutErr generate() { + int actionId = actionCounter.incrementAndGet(); + return new FileOutErr(actionOutputRoot.getRelative("stdout-" + actionId), + actionOutputRoot.getRelative("stderr-" + actionId)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionMetadata.java b/src/main/java/com/google/devtools/build/lib/actions/ActionMetadata.java new file mode 100644 index 0000000..3569f07 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionMetadata.java
@@ -0,0 +1,217 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +import javax.annotation.Nullable; + +/** + * Side-effect free query methods for information about an {@link Action}. + * + * <p>This method is intended for use in situations when the intention is to pass around information + * about an action without allowing actual execution of the action. + * + * <p>The split between {@link Action} and {@link ActionMetadata} is somewhat arbitrary, other than + * that all methods with side effects must belong to the former. + */ +public interface ActionMetadata { + /** + * If this executable can supply verbose information, returns a string that can be used as a + * progress message while this executable is running. A return value of {@code null} indicates no + * message should be reported. + */ + @Nullable + public String getProgressMessage(); + + /** + * Returns the owner of this executable if this executable can supply verbose information. This is + * typically the rule that constructed it; see ActionOwner class comment for details. Returns + * {@code null} if no owner can be determined. + * + * <p>If this executable does not supply verbose information, this function may throw an + * IllegalStateException. + */ + public ActionOwner getOwner(); + + /** + * Returns a mnemonic (string constant) for this kind of action; written into + * the master log so that the appropriate parser can be invoked for the output + * of the action. Effectively a public method as the value is used by the + * extra_action feature to match actions. + */ + String getMnemonic(); + + /** + * Returns a pretty string representation of this action, suitable for use in + * progress messages or error messages. + */ + String prettyPrint(); + + /** + * Returns a string that can be used to describe the execution strategy. + * For example, "local". + * + * May return null if the action chooses to update its strategy + * locality "manually", via ActionLocalityMessage. + * + * @param executor the application-specific value passed to the + * executor parameter of the top-level call to + * Builder.buildArtifacts(). + */ + public String describeStrategy(Executor executor); + + /** + * Returns true iff the getInputs set is known to be complete. + * + * <p>For most Actions, this always returns true, but in some cases (e.g. C++ compilation), inputs + * are dynamically discovered from the previous execution of the Action, and so before the initial + * execution, this method will return false in those cases. + * + * <p>Any builder <em>must</em> unconditionally execute an Action for which inputsKnown() returns + * false, regardless of all other inferences made by its dependency analysis. In addition, all + * prerequisites mentioned in the (possibly incomplete) value returned by getInputs must also be + * built first, as usual. + */ + @ThreadSafe + boolean inputsKnown(); + + /** + * Returns true iff inputsKnown() may ever return false. + */ + @ThreadSafe + boolean discoversInputs(); + + /** + * Returns the input Artifacts that this Action depends upon. May be empty. + * + * <p>For subclasses overriding getInputs(), if getInputs() could return different values in the + * lifetime of an object, {@link #getInputCount()} must also be overridden. + * + * <p>During execution, the {@link Iterable} returned by {@code getInputs} <em>must not</em> be + * concurrently modified before the value is fully read in {@code JavaDistributorDriver#exec} (via + * the {@code Iterable<ActionInput>} argument there). Violating this would require somewhat + * pathological behavior by the {@link Action}, since it would have to modify its inputs, as a + * list, say, without reassigning them. This should never happen with any Action subclassing + * AbstractAction, since AbstractAction's implementation of getInputs() returns an immutable + * iterable. + */ + Iterable<Artifact> getInputs(); + + /** + * Returns the number of input Artifacts that this Action depends upon. + * + * <p>Must be consistent with {@link #getInputs()}. + */ + int getInputCount(); + + /** + * Returns the (unordered, immutable) set of output Artifacts that + * this action generates. (It would not make sense for this to be empty.) + */ + ImmutableSet<Artifact> getOutputs(); + + /** + * Returns the "primary" input of this action, if applicable. + * + * <p>For example, a C++ compile action would return the .cc file which is being compiled, + * irrespective of the other inputs. + * + * <p>May return null. + */ + Artifact getPrimaryInput(); + + /** + * Returns the "primary" output of this action. + * + * <p>For example, the linked library would be the primary output of a LinkAction. + * + * <p>Never returns null. + */ + Artifact getPrimaryOutput(); + + /** + * Returns an iterable of input Artifacts that MUST exist prior to executing an action. In other + * words, in case when action is scheduled for execution, builder will ensure that all artifacts + * returned by this method are present in the filesystem (artifact.getPath().exists() is true) or + * action execution will be aborted with an error that input file does not exist. While in + * majority of cases this method will return all action inputs, for some actions (e.g. + * CppCompileAction) it can return a subset of inputs because that not all action inputs might be + * mandatory for action execution to succeed (e.g. header files retrieved from *.d file from the + * previous build). + */ + Iterable<Artifact> getMandatoryInputs(); + + /** + * <p>Returns a string encoding all of the significant behaviour of this + * Action that might affect the output. The general contract of + * <code>getKey</code> is this: if the work to be performed by the + * execution of this action changes, the key must change. </p> + * + * <p>As a corollary, the build system is free to omit the execution of an + * Action <code>a1</code> if (a) at some time in the past, it has already + * executed an Action <code>a0</code> with the same key as + * <code>a1</code>, and (b) the names and contents of the input files listed + * by <code>a1.getInputs()</code> are identical to the names and contents of + * the files listed by <code>a0.getInputs()</code>. </p> + * + * <p>Examples of changes that should affect the key are: + * <ul> + * <li>Changes to the BUILD file that materially affect the rule which gave + * rise to this Action.</li> + * + * <li>Changes to the command-line options, environment, or other global + * configuration resources which affect the behaviour of this kind of Action + * (other than changes to the names of the input/output files, which are + * handled externally).</li> + * + * <li>An upgrade to the build tools which changes the program logic of this + * kind of Action (typically this is achieved by incorporating a UUID into + * the key, which is changed each time the program logic of this action + * changes).</li> + * + * </ul></p> + */ + String getKey(); + + /** + * Returns a human-readable description of the inputs to {@link #getKey()}. + * Used in the output from '--explain', and in error messages for + * '--check_up_to_date' and '--check_tests_up_to_date'. + * May return null, meaning no extra information is available. + * + * <p>If the return value is non-null, for consistency it should be a multiline message of the + * form: + * <pre> + * <var>Summary</var> + * <var>Fieldname</var>: <var>value</var> + * <var>Fieldname</var>: <var>value</var> + * ... + * </pre> + * where each line after the first one is intended two spaces, and where any fields that might + * contain newlines or other funny characters are escaped using {@link + * com.google.devtools.build.lib.shell.ShellUtils#shellEscape}. + * For example: + * <pre> + * Compiling foo.cc + * Command: /usr/bin/gcc + * Argument: '-c' + * Argument: foo.cc + * Argument: '-o' + * Argument: foo.o + * </pre> + */ + @Nullable String describeKey(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionMiddlemanEvent.java b/src/main/java/com/google/devtools/build/lib/actions/ActionMiddlemanEvent.java new file mode 100644 index 0000000..855d97a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionMiddlemanEvent.java
@@ -0,0 +1,54 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; + +/** + * This event is fired during the build, when a middleman action is executed. Middleman actions + * don't usually do any computation but we need them in the critical path because they depend on + * other actions. + */ +public class ActionMiddlemanEvent { + + private final Action action; + private final long nanoTimeStart; + + /** + * Create an event for action that has been started. + * + * @param action the middleman action. + * @param nanoTimeStart the time when the action was started. This allow us to record more + * accurately the time spent by the middleman action, since even for middleman actions we execute + * some. + */ + public ActionMiddlemanEvent(Action action, long nanoTimeStart) { + Preconditions.checkArgument(action.getActionType().isMiddleman(), + "Only middleman actions should be passed: %s", action); + this.action = action; + this.nanoTimeStart = nanoTimeStart; + } + + /** + * Returns the associated action. + */ + public Action getAction() { + return action; + } + + public long getNanoTimeStart() { + return nanoTimeStart; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionOwner.java b/src/main/java/com/google/devtools/build/lib/actions/ActionOwner.java new file mode 100644 index 0000000..ab59f63 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionOwner.java
@@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Label; + +/** + * The owner of an action is responsible for reporting conflicts in the action + * graph (two actions attempting to generate the same artifact). + * + * Typically an action's owner is the RuleConfiguredTarget instance responsible + * for creating it, but to avoid coupling between the view and actions + * packages, the RuleConfiguredTarget is hidden behind this interface, which + * exposes only the error reporting functionality. + */ +public interface ActionOwner { + + /** + * Returns the location of this ActionOwner, if any; null otherwise. + */ + Location getLocation(); + + /** + * Returns the label for this ActionOwner, if any; null otherwise. + */ + Label getLabel(); + + /** + * Returns the name of the configuration of the action owner. + */ + String getConfigurationName(); + + /** + * Returns the configuration's mnemonic. + */ + String getConfigurationMnemonic(); + + /** + * Returns the short cache key for the configuration of the action owner. + * + * <p>Special action owners that are not targets can return any string here as long as it is + * constant. If the configuration is null, this should return "null". + * + * <p>These requirements exist so that {@link ActionOwner} instances are consistent with + * {@code BuildView.ActionOwnerIdentity(ConfiguredTargetValue)}. + */ + String getConfigurationShortCacheKey(); + + /** + * Returns the target kind (rule class name) for this ActionOwner, if any; null otherwise. + */ + String getTargetKind(); + + /** + * Returns additional information that should be displayed in progress messages, or {@code null} + * if nothing should be added. + */ + String getAdditionalProgressInfo(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionRegistry.java b/src/main/java/com/google/devtools/build/lib/actions/ActionRegistry.java new file mode 100644 index 0000000..db7e350 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionRegistry.java
@@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; + +/** + * An interface for registering actions. + */ +public interface ActionRegistry { + /** + * This method notifies the registry new actions. + */ + void registerAction(Action... actions); + + /** + * Get the (Label and BuildConfiguration) of the ConfiguredTarget ultimately responsible for all + * these actions. + */ + ArtifactOwner getOwner(); + + /** + * An action registry that does exactly nothing. + */ + @VisibleForTesting + public static final ActionRegistry NOP = new ActionRegistry() { + @Override + public void registerAction(Action... actions) {} + + @Override + public ArtifactOwner getOwner() { + return ArtifactOwner.NULL_OWNER; + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionStartedEvent.java b/src/main/java/com/google/devtools/build/lib/actions/ActionStartedEvent.java new file mode 100644 index 0000000..a2a978f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionStartedEvent.java
@@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * This event is fired during the build, when an action is started. + */ +public class ActionStartedEvent { + private final Action action; + private final long nanoTimeStart; + + /** + * Create an event for action that has been started. + * + * @param action the started action. + * @param nanoTimeStart the time when the action was started. This allow us to + * record more accurately the time spend by the action, since we execute some code before + * deciding if we execute the action or not. + */ + public ActionStartedEvent(Action action, long nanoTimeStart) { + this.action = action; + this.nanoTimeStart = nanoTimeStart; + } + + /** + * Returns the associated action. + */ + public Action getAction() { + return action; + } + + public long getNanoTimeStart() { + return nanoTimeStart; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionStatusMessage.java b/src/main/java/com/google/devtools/build/lib/actions/ActionStatusMessage.java new file mode 100644 index 0000000..c932a9e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionStatusMessage.java
@@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * A message used to update in-flight action status. An action's status may change low down in the + * execution stack (for instance, from running remotely to running locally), so this message can be + * used to notify any interested parties. + */ +public class ActionStatusMessage { + private final ActionMetadata action; + private final String message; + public static final String PREPARING = "Preparing"; + + public ActionStatusMessage(ActionMetadata action, String message) { + this.action = action; + this.message = message; + } + + public ActionMetadata getActionMetadata() { + return action; + } + + public String getMessage() { + return message; + } + + /** Returns whether the message needs further interpolation of a 'strategy' when printed. */ + public boolean needsStrategy() { + return false; + } + + /** Creates "Analyzing" status message. */ + public static ActionStatusMessage analysisStrategy(ActionMetadata action) { + return new ActionStatusMessage(action, "Analyzing"); + } + + /** Creates "Preparing" status message. */ + public static ActionStatusMessage preparingStrategy(ActionMetadata action) { + return new ActionStatusMessage(action, PREPARING); + } + + /** Creates "Scheduling" status message. */ + public static ActionStatusMessage schedulingStrategy(ActionMetadata action) { + return new ActionStatusMessage(action, "Scheduling"); + } + + /** Creates "Running (%s)" status message (needs strategy interpolated). */ + public static ActionStatusMessage runningStrategy(ActionMetadata action) { + return new ActionStatusMessage(action, "Running (%s)") { + @Override + public boolean needsStrategy() { + return true; + } + }; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/Actions.java b/src/main/java/com/google/devtools/build/lib/actions/Actions.java new file mode 100644 index 0000000..fb2b834 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/Actions.java
@@ -0,0 +1,79 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.Iterables; +import com.google.common.escape.Escaper; +import com.google.common.escape.Escapers; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.syntax.Label; + +/** + * Helper class for actions. + */ +@ThreadSafe +public final class Actions { + private static final Escaper PATH_ESCAPER = Escapers.builder() + .addEscape('_', "_U") + .addEscape('/', "_S") + .addEscape('\\', "_B") + .addEscape(':', "_C") + .build(); + + /** + * Checks if the two actions are equivalent. This method exists to support sharing actions between + * configured targets for cases where there is no canonical target that could own the action. In + * the action graph construction this case shows up as two actions generating the same output + * file. + * + * <p>This method implements an equivalence relationship across actions, based on the action + * class, the key, and the list of inputs and outputs. + */ + public static boolean canBeShared(Action a, Action b) { + if (!a.getMnemonic().equals(b.getMnemonic())) { + return false; + } + if (!a.getKey().equals(b.getKey())) { + return false; + } + // Don't bother to check input and output counts first; the expected result for these tests is + // to always be true (i.e., that this method returns true). + if (!Iterables.elementsEqual(a.getMandatoryInputs(), b.getMandatoryInputs())) { + return false; + } + if (!Iterables.elementsEqual(a.getOutputs(), b.getOutputs())) { + return false; + } + return true; + } + + /** + * Returns the escaped name for a given relative path as a string. This takes + * a short relative path and turns it into a string suitable for use as a + * filename. Invalid filename characters are escaped with an '_' + a single + * character token. + */ + public static String escapedPath(String path) { + return PATH_ESCAPER.escape(path); + } + + /** + * Returns a string that is usable as a unique path component for a label. It is guaranteed + * that no other label maps to this string. + */ + public static String escapeLabel(Label label) { + return PATH_ESCAPER.escape(label.getPackageName() + ":" + label.getName()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/AlreadyReportedActionExecutionException.java b/src/main/java/com/google/devtools/build/lib/actions/AlreadyReportedActionExecutionException.java new file mode 100644 index 0000000..4ce258b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/AlreadyReportedActionExecutionException.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * This wrapper exception is used as a marker class to already reported errors. Errors are reported + * at {@code AbstractBuilder.executeActionTask()} method in case of the builder not aborting in case + * of exceptions (For example keepgoing). + * + * <p>Then In upper levels we wrap catch the exception and throw a BuildFailedException + * unconditionally, that is caught and shown as error in AbstractBuildCommand (because the message + * of the exception is !=null). + * + * With this exception we detect that the error was already shown and we wrap it in a + * BuildFailedException without message. + */ +public class AlreadyReportedActionExecutionException extends ActionExecutionException { + + public AlreadyReportedActionExecutionException(ActionExecutionException cause) { + super(cause.getMessage(), cause.getCause(), cause.getAction(), cause.getRootCauses(), + cause.isCatastrophe()); + } + + @Override + public boolean showError() { + return false; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/Artifact.java b/src/main/java/com/google/devtools/build/lib/actions/Artifact.java new file mode 100644 index 0000000..2f2272b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/Artifact.java
@@ -0,0 +1,654 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Functions; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action.MiddlemanType; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import javax.annotation.Nullable; + +/** + * An Artifact represents a file used by the build system, whether it's a source + * file or a derived (output) file. Not all Artifacts have a corresponding + * FileTarget object in the <code>build.packages</code> API: for example, + * low-level intermediaries internal to a given rule, such as a Java class files + * or C++ object files. However all FileTargets have a corresponding Artifact. + * + * <p>In any given call to Builder#buildArtifacts(), no two Artifacts in the + * action graph may refer to the same path. + * + * <p>Artifacts generally fall into two classifications, source and derived, but + * there exist a few other cases that are fuzzy and difficult to classify. The + * following cases exist: + * <ul> + * <li>Well-formed source Artifacts will have null generating Actions and a root + * that is orthogonal to execRoot. (With the root coming from the package path.) + * <li>Well-formed derived Artifacts will have non-null generating Actions, and + * a root that is below execRoot. + * <li>Symlinked include source Artifacts under the output/include tree will + * appear to be derived artifacts with null generating Actions. + * <li>Some derived Artifacts, mostly in the genfiles tree and mostly discovered + * during include validation, will also have null generating Actions. + * </ul> + * + * <p>This class is "theoretically" final; it should not be subclassed except by + * {@link SpecialArtifact}. + */ +@Immutable +@SkylarkModule(name = "File", + doc = "This type represents a file used by the build system. It can be " + + "either a source file or a derived file produced by a rule.") +public class Artifact implements FileType.HasFilename, Comparable<Artifact>, ActionInput { + + /** An object that can expand middleman artifacts. */ + public interface MiddlemanExpander { + + /** + * Expands the middleman artifact "mm", and populates "output" with the result. + * + * <p>{@code mm.isMiddlemanArtifact()} must be true. Only aggregating middlemen are expanded. + */ + void expand(Artifact mm, Collection<? super Artifact> output); + } + + public static final ImmutableList<Artifact> NO_ARTIFACTS = ImmutableList.of(); + + /** + * A Predicate that evaluates to true if the Artifact is not a middleman artifact. + */ + public static final Predicate<Artifact> MIDDLEMAN_FILTER = new Predicate<Artifact>() { + @Override + public boolean apply(Artifact input) { + return !input.isMiddlemanArtifact(); + } + }; + + private final Path path; + private final Root root; + private final PathFragment execPath; + private final PathFragment rootRelativePath; + // Non-final only for use when dealing with deserialized artifacts. + private ArtifactOwner owner; + + /** + * Constructs an artifact for the specified path, root and execPath. The root must be an ancestor + * of path, and execPath must be a non-absolute tail of path. Outside of testing, this method + * should only be called by ArtifactFactory. The ArtifactOwner may be null. + * + * <p>In a source Artifact, the path tail after the root will be identical to the execPath, but + * the root will be orthogonal to execRoot. + * <pre> + * [path] == [/root/][execPath] + * </pre> + * + * <p>In a derived Artifact, the execPath will overlap with part of the root, which in turn will + * be below of the execRoot. + * <pre> + * [path] == [/root][pathTail] == [/execRoot][execPath] == [/execRoot][rootPrefix][pathTail] + * <pre> + */ + @VisibleForTesting + public Artifact(Path path, Root root, PathFragment execPath, ArtifactOwner owner) { + if (root == null || !path.startsWith(root.getPath())) { + throw new IllegalArgumentException(root + ": illegal root for " + path); + } + if (execPath == null || execPath.isAbsolute() || !path.asFragment().endsWith(execPath)) { + throw new IllegalArgumentException(execPath + ": illegal execPath for " + path); + } + this.path = path; + this.root = root; + this.execPath = execPath; + // These two lines establish the invariant that + // execPath == rootRelativePath <=> execPath.equals(rootRelativePath) + // This is important for isSourceArtifact. + PathFragment rootRel = path.relativeTo(root.getPath()); + if (!execPath.endsWith(rootRel)) { + throw new IllegalArgumentException(execPath + ": illegal execPath doesn't end with " + + rootRel + " at " + path + " with root " + root); + } + this.rootRelativePath = rootRel.equals(execPath) ? execPath : rootRel; + this.owner = Preconditions.checkNotNull(owner, path); + } + + /** + * Constructs an artifact for the specified path, root and execPath. The root must be an ancestor + * of path, and execPath must be a non-absolute tail of path. Should only be called for testing. + * + * <p>In a source Artifact, the path tail after the root will be identical to the execPath, but + * the root will be orthogonal to execRoot. + * <pre> + * [path] == [/root/][execPath] + * </pre> + * + * <p>In a derived Artifact, the execPath will overlap with part of the root, which in turn will + * be below of the execRoot. + * <pre> + * [path] == [/root][pathTail] == [/execRoot][execPath] == [/execRoot][rootPrefix][pathTail] + * <pre> + */ + @VisibleForTesting + public Artifact(Path path, Root root, PathFragment execPath) { + this(path, root, execPath, ArtifactOwner.NULL_OWNER); + } + + /** + * Constructs a source or derived Artifact for the specified path and specified root. The root + * must be an ancestor of the path. + */ + @VisibleForTesting // Only exists for testing. + public Artifact(Path path, Root root) { + this(path, root, root.getExecPath().getRelative(path.relativeTo(root.getPath())), + ArtifactOwner.NULL_OWNER); + } + + /** + * Constructs a source or derived Artifact for the specified root-relative path and root. + */ + @VisibleForTesting // Only exists for testing. + public Artifact(PathFragment rootRelativePath, Root root) { + this(root.getPath().getRelative(rootRelativePath), root, + root.getExecPath().getRelative(rootRelativePath), ArtifactOwner.NULL_OWNER); + } + + /** + * Returns the location of this Artifact on the filesystem. + */ + public final Path getPath() { + return path; + } + + /** + * Returns the base file name of this artifact. + */ + @Override + public final String getFilename() { + return getExecPath().getBaseName(); + } + + /** + * Returns the artifact owner. May be null. + */ + @Nullable public final Label getOwner() { + return owner.getLabel(); + } + + /** + * Get the {@code LabelAndConfiguration} of the {@code ConfiguredTarget} that owns this artifact, + * if it was set. Otherwise, this should be a dummy value -- either {@link + * ArtifactOwner#NULL_OWNER} or a dummy owner set in tests. Such a dummy value should only occur + * for source artifacts if created without specifying the owner, or for special derived artifacts, + * such as target completion middleman artifacts, build info artifacts, and the like. + * + * When deserializing artifacts we end up with a dummy owner. In that case, it must be set using + * {@link #setArtifactOwner} before this method is called. + */ + public final ArtifactOwner getArtifactOwner() { + Preconditions.checkState(owner != DESERIALIZED_MARKER_OWNER, this); + return owner; + } + + /** + * Sets the artifact owner of this artifact. Should only be called for artifacts that were created + * through deserialization, and so their owner was unknown at the time of creation. + */ + public final void setArtifactOwner(ArtifactOwner owner) { + if (this.owner == DESERIALIZED_MARKER_OWNER) { + // We tolerate multiple calls of this method to accommodate shared actions. + this.owner = Preconditions.checkNotNull(owner, this); + } + } + + /** + * Returns the root beneath which this Artifact resides, if any. This may be one of the + * package-path entries (for source Artifacts), or one of the bin, genfiles or includes dirs + * (for derived Artifacts). It will always be an ancestor of getPath(). + */ + public final Root getRoot() { + return root; + } + + /** + * Returns the exec path of this Artifact. The exec path is a relative path + * that is suitable for accessing this artifact relative to the execution + * directory for this build. + */ + public final PathFragment getExecPath() { + return execPath; + } + + /** + * Returns true iff this is a source Artifact as determined by its path and + * root relationships. Note that this will report all Artifacts in the output + * tree, including in the include symlink tree, as non-source. + */ + public final boolean isSourceArtifact() { + return execPath == rootRelativePath; + } + + /** + * Returns true iff this is a middleman Artifact as determined by its root. + */ + public final boolean isMiddlemanArtifact() { + return getRoot().isMiddlemanRoot(); + } + + /** + * Returns whether the artifact represents a Fileset. + */ + public boolean isFileset() { + return false; + } + + /** + * Returns true iff metadata cache must return constant metadata for the + * given artifact. + */ + public boolean isConstantMetadata() { + return false; + } + + /** + * Special artifact types. + * + * @see SpecialArtifact + */ + static enum SpecialArtifactType { + FILESET, + CONSTANT_METADATA, + } + + /** + * A special kind of artifact that either is a fileset or needs special metadata caching behavior. + * + * <p>We subclass {@link Artifact} instead of storing the special attributes inside in order + * to save memory. The proportion of artifacts that are special is very small, and by not having + * to keep around the attribute for the rest we save some memory. + */ + @Immutable + @VisibleForTesting + public static final class SpecialArtifact extends Artifact { + private final SpecialArtifactType type; + + SpecialArtifact(Path path, Root root, PathFragment execPath, ArtifactOwner owner, + SpecialArtifactType type) { + super(path, root, execPath, owner); + this.type = type; + } + + @Override + public final boolean isFileset() { + return type == SpecialArtifactType.FILESET; + } + + @Override + public boolean isConstantMetadata() { + return type == SpecialArtifactType.CONSTANT_METADATA; + } + } + + /** + * Returns the relative path to this artifact relative to its root. (Useful + * when deriving output filenames from input files, etc.) + */ + public final PathFragment getRootRelativePath() { + return rootRelativePath; + } + + /** + * Returns this.getExecPath().getPathString(). + */ + @Override + @SkylarkCallable(name = "path", structField = true, + doc = "The execution path of this file, relative to the execution directory.") + public final String getExecPathString() { + return getExecPath().getPathString(); + } + + @SkylarkCallable(name = "short_path", structField = true, + doc = "The path of this file relative to its root.") + public final String getRootRelativePathString() { + return getRootRelativePath().getPathString(); + } + + /** + * Returns a pretty string representation of the path denoted by this artifact, suitable for use + * in user error messages. Artifacts beneath a root will be printed relative to that root; other + * artifacts will be printed as an absolute path. + * + * <p>(The toString method is intended for developer messages since its more informative.) + */ + public final String prettyPrint() { + // toDetailString would probably be more useful to users, but lots of tests rely on the + // current values. + return rootRelativePath.toString(); + } + + @Override + public final boolean equals(Object other) { + if (!(other instanceof Artifact)) { + return false; + } + // We don't bother to check root in the equivalence relation, because we + // assume that 'root' is an ancestor of 'path', and that all possible roots + // are disjoint, so unless things are really screwed up, it's ok. + Artifact that = (Artifact) other; + return this.path.equals(that.path); + } + + @Override + public final int compareTo(Artifact o) { + // The artifact factory ensures that there is a unique artifact for a given path. + return this.path.compareTo(o.path); + } + + @Override + public final int hashCode() { + return path.hashCode(); + } + + @Override + public final String toString() { + return "Artifact:" + toDetailString(); + } + + /** + * Returns the root-part of a given path by trimming off the end specified by + * a given tail. Assumes that the tail is known to match, and simply relies on + * the segment lengths. + */ + private static PathFragment trimTail(PathFragment path, PathFragment tail) { + return path.subFragment(0, path.segmentCount() - tail.segmentCount()); + } + + /** + * Returns a string representing the complete artifact path information. + */ + public final String toDetailString() { + if (isSourceArtifact()) { + // Source Artifact: relPath == execPath, & real path is not under execRoot + return "[" + root + "]" + rootRelativePath; + } else { + // Derived Artifact: path and root are under execRoot + PathFragment execRoot = trimTail(path.asFragment(), execPath); + return "[[" + execRoot + "]" + root.getPath().asFragment().relativeTo(execRoot) + "]" + + rootRelativePath; + } + } + + /** + * Serializes this artifact to a string that has enough data to reconstruct the artifact. + */ + public final String serializeToString() { + // In theory, it should be enough to serialize execPath and rootRelativePath (which is a suffix + // of execPath). However, in practice there is code around that uses other attributes which + // needs cleaning up. + String result = execPath + " /" + rootRelativePath.toString().length(); + if (getOwner() != null) { + result += " " + getOwner(); + } + return result; + } + + //--------------------------------------------------------------------------- + // Static methods to assist in working with Artifacts + + /** + * Formatter for execPath PathFragment output. + */ + private static final Function<Artifact, PathFragment> EXEC_PATH_FORMATTER = + new Function<Artifact, PathFragment>() { + @Override + public PathFragment apply(Artifact input) { + return input.getExecPath(); + } + }; + + private static final Function<Artifact, String> ROOT_RELATIVE_PATH_STRING = + new Function<Artifact, String>() { + @Override + public String apply(Artifact artifact) { + return artifact.getRootRelativePath().getPathString(); + } + }; + + /** + * Converts a collection of artifacts into execution-time path strings, and + * adds those to a given collection. Middleman artifacts are ignored by this + * method. + */ + public static void addExecPaths(Iterable<Artifact> artifacts, Collection<String> output) { + addNonMiddlemanArtifacts(artifacts, output, ActionInputHelper.EXEC_PATH_STRING_FORMATTER); + } + + /** + * Converts a collection of artifacts into the outputs computed by + * outputFormatter and adds them to a given collection. Middleman artifacts + * are ignored. + */ + static <E> void addNonMiddlemanArtifacts(Iterable<Artifact> artifacts, + Collection<? super E> output, Function<? super Artifact, E> outputFormatter) { + for (Artifact artifact : artifacts) { + if (MIDDLEMAN_FILTER.apply(artifact)) { + output.add(outputFormatter.apply(artifact)); + } + } + } + + /** + * Lazily converts artifacts into root-relative path strings. Middleman artifacts are ignored by + * this method. + */ + public static Iterable<String> toRootRelativePaths(Iterable<Artifact> artifacts) { + return Iterables.transform( + Iterables.filter(artifacts, MIDDLEMAN_FILTER), + ROOT_RELATIVE_PATH_STRING); + } + + /** + * Lazily converts artifacts into execution-time path strings. Middleman artifacts are ignored by + * this method. + */ + public static Iterable<String> toExecPaths(Iterable<Artifact> artifacts) { + return ActionInputHelper.toExecPaths(Iterables.filter(artifacts, MIDDLEMAN_FILTER)); + } + + /** + * Converts a collection of artifacts into execution-time path strings, and + * returns those as an immutable list. Middleman artifacts are ignored by this method. + */ + public static List<String> asExecPaths(Iterable<Artifact> artifacts) { + return ImmutableList.copyOf(toExecPaths(artifacts)); + } + + /** + * Renders a collection of artifacts as execution-time paths and joins + * them into a single string. Middleman artifacts are ignored by this method. + */ + public static String joinExecPaths(String delimiter, Iterable<Artifact> artifacts) { + return Joiner.on(delimiter).join(toExecPaths(artifacts)); + } + + /** + * Renders a collection of artifacts as root-relative paths and joins + * them into a single string. Middleman artifacts are ignored by this method. + */ + public static String joinRootRelativePaths(String delimiter, Iterable<Artifact> artifacts) { + return Joiner.on(delimiter).join(toRootRelativePaths(artifacts)); + } + + /** + * Adds a collection of artifacts to a given collection, with + * {@link MiddlemanType#AGGREGATING_MIDDLEMAN} middleman actions expanded once. + */ + public static void addExpandedArtifacts(Iterable<Artifact> artifacts, + Collection<? super Artifact> output, MiddlemanExpander middlemanExpander) { + addExpandedArtifacts(artifacts, output, Functions.<Artifact>identity(), middlemanExpander); + } + + /** + * Converts a collection of artifacts into execution-time path strings, and + * adds those to a given collection. Middleman artifacts for + * {@link MiddlemanType#AGGREGATING_MIDDLEMAN} middleman actions are expanded + * once. + */ + @VisibleForTesting + public static void addExpandedExecPathStrings(Iterable<Artifact> artifacts, + Collection<String> output, + MiddlemanExpander middlemanExpander) { + addExpandedArtifacts(artifacts, output, ActionInputHelper.EXEC_PATH_STRING_FORMATTER, + middlemanExpander); + } + + /** + * Converts a collection of artifacts into execution-time path fragments, and + * adds those to a given collection. Middleman artifacts for + * {@link MiddlemanType#AGGREGATING_MIDDLEMAN} middleman actions are expanded + * once. + */ + public static void addExpandedExecPaths(Iterable<Artifact> artifacts, + Collection<PathFragment> output, MiddlemanExpander middlemanExpander) { + addExpandedArtifacts(artifacts, output, EXEC_PATH_FORMATTER, middlemanExpander); + } + + /** + * Converts a collection of artifacts into the outputs computed by + * outputFormatter and adds them to a given collection. Middleman artifacts + * are expanded once. + */ + private static <E> void addExpandedArtifacts(Iterable<Artifact> artifacts, + Collection<? super E> output, + Function<? super Artifact, E> outputFormatter, + MiddlemanExpander middlemanExpander) { + for (Artifact artifact : artifacts) { + if (artifact.isMiddlemanArtifact()) { + expandMiddlemanArtifact(artifact, output, outputFormatter, middlemanExpander); + } else { + output.add(outputFormatter.apply(artifact)); + } + } + } + + private static <E> void expandMiddlemanArtifact(Artifact middleman, + Collection<? super E> output, + Function<? super Artifact, E> outputFormatter, + MiddlemanExpander middlemanExpander) { + Preconditions.checkArgument(middleman.isMiddlemanArtifact()); + List<Artifact> artifacts = new ArrayList<>(); + middlemanExpander.expand(middleman, artifacts); + for (Artifact artifact : artifacts) { + output.add(outputFormatter.apply(artifact)); + } + } + + /** + * Converts a collection of artifacts into execution-time path strings, and + * returns those as a list. Middleman artifacts are expanded once. The + * returned list is mutable. + */ + public static List<String> asExpandedExecPathStrings(Iterable<Artifact> artifacts, + MiddlemanExpander middlemanExpander) { + List<String> result = new ArrayList<>(); + addExpandedExecPathStrings(artifacts, result, middlemanExpander); + return result; + } + + /** + * Converts a collection of artifacts into execution-time path fragments, and + * returns those as a list. Middleman artifacts are expanded once. The + * returned list is mutable. + */ + public static List<PathFragment> asExpandedExecPaths(Iterable<Artifact> artifacts, + MiddlemanExpander middlemanExpander) { + List<PathFragment> result = new ArrayList<>(); + addExpandedExecPaths(artifacts, result, middlemanExpander); + return result; + } + + /** + * Converts a collection of artifacts into execution-time path strings with + * the root-break delimited with a colon ':', and adds those to a given list. + * <pre> + * Source: sourceRoot/rootRelative => :rootRelative + * Derived: execRoot/rootPrefix/rootRelative => rootPrefix:rootRelative + * </pre> + */ + public static void addRootPrefixedExecPaths(Iterable<Artifact> artifacts, + List<String> output) { + for (Artifact artifact : artifacts) { + output.add(asRootPrefixedExecPath(artifact)); + } + } + + /** + * Convenience method to filter the files to build for a certain filetype. + * + * @param artifacts the files to filter + * @param allowedType the allowed filetype + * @return all members of filesToBuild that are of one of the + * allowed filetypes + */ + public static List<Artifact> filterFiles(Iterable<Artifact> artifacts, FileType allowedType) { + List<Artifact> filesToBuild = new ArrayList<>(); + for (Artifact artifact : artifacts) { + if (allowedType.matches(artifact.getFilename())) { + filesToBuild.add(artifact); + } + } + return filesToBuild; + } + + @VisibleForTesting + static String asRootPrefixedExecPath(Artifact artifact) { + PathFragment execPath = artifact.getExecPath(); + PathFragment rootRel = artifact.getRootRelativePath(); + if (execPath.equals(rootRel)) { + return ":" + rootRel.getPathString(); + } else { //if (execPath.endsWith(rootRel)) { + PathFragment rootPrefix = trimTail(execPath, rootRel); + return rootPrefix.getPathString() + ":" + rootRel.getPathString(); + } + } + + /** + * Converts artifacts into their exec paths. Returns an immutable list. + */ + public static List<PathFragment> asPathFragments(Iterable<Artifact> artifacts) { + return ImmutableList.copyOf(Iterables.transform(artifacts, EXEC_PATH_FORMATTER)); + } + + static final ArtifactOwner DESERIALIZED_MARKER_OWNER = new ArtifactOwner() { + @Override + public Label getLabel() { + return null; + }}; +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactDeserializer.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactDeserializer.java new file mode 100644 index 0000000..40996ac --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactDeserializer.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; + +/** + * An interface for creating artifacts from their serialized representations. + * + * @see ArtifactSerializer + */ +public interface ArtifactDeserializer { + + /** + * Looks up an artifact by an integer id. + * + * <p>This is a dual of {@link ArtifactSerializer#getArtifactId}. + */ + Artifact lookupArtifactById(int artifactId); + + /** + * Maps a list of artifact ids to a list of artifacts. + * + * <p>This is a batch version of {@link #lookupArtifactById}, provided for efficiency. It takes + * an iterable of boxed integers because that's what the proto wrapper provides. + */ + ImmutableList<Artifact> lookupArtifactsByIds(Iterable<Integer> artifactIds); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java new file mode 100644 index 0000000..99a53cb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java
@@ -0,0 +1,339 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import javax.annotation.Nullable; + +/** + * A cache of Artifacts, keyed by Path. + */ +@ThreadSafe +public class ArtifactFactory implements ArtifactResolver, ArtifactSerializer, ArtifactDeserializer { + + private final Path execRoot; + + /** + * The main Path to source artifact cache. There will always be exactly one canonical + * artifact for a given source path. + */ + private final Map<PathFragment, Artifact> pathToSourceArtifact = new HashMap<>(); + + /** + * Map of package names to source root paths so that we can create source + * artifact paths given execPaths in the symlink forest. + */ + private ImmutableMap<PackageIdentifier, Root> packageRoots; + + /** + * Reverse-ordered list of derived roots for use in looking up or (in rare cases) creating + * derived artifacts from execPaths. The reverse order is only significant for overlapping roots + * so that the longest is found first. + */ + private ImmutableCollection<Root> derivedRoots = ImmutableList.of(); + + private ArtifactIdRegistry artifactIdRegistry = new ArtifactIdRegistry(); + + /** + * Constructs a new artifact factory that will use a given execution root when + * creating artifacts. + * + * @param execRoot the execution root Path to use + */ + public ArtifactFactory(Path execRoot) { + this.execRoot = execRoot; + } + + /** + * Clear the cache. + */ + public synchronized void clear() { + pathToSourceArtifact.clear(); + packageRoots = null; + derivedRoots = ImmutableList.of(); + artifactIdRegistry = new ArtifactIdRegistry(); + clearDeserializedArtifacts(); + } + + /** + * Set the set of known packages and their corresponding source artifact + * roots. Must be called exactly once after construction or clear(). + * + * @param packageRoots the map of package names to source artifact roots to + * use. + */ + public synchronized void setPackageRoots(Map<PackageIdentifier, Root> packageRoots) { + this.packageRoots = ImmutableMap.copyOf(packageRoots); + } + + /** + * Set the set of known derived artifact roots. Must be called exactly once + * after construction or clear(). + * + * @param roots the set of derived artifact roots to use + */ + public synchronized void setDerivedArtifactRoots(Collection<Root> roots) { + derivedRoots = ImmutableSortedSet.<Root>reverseOrder().addAll(roots).build(); + } + + @Override + public Artifact getSourceArtifact(PathFragment execPath, Root root, ArtifactOwner owner) { + Preconditions.checkArgument(!execPath.isAbsolute()); + Preconditions.checkNotNull(owner, execPath); + execPath = execPath.normalize(); + return getArtifact(root.getPath().getRelative(execPath), root, execPath, owner, null); + } + + @Override + public Artifact getSourceArtifact(PathFragment execPath, Root root) { + return getSourceArtifact(execPath, root, ArtifactOwner.NULL_OWNER); + } + + /** + * Only for use by BinTools! Returns an artifact for a tool at the given path + * fragment, relative to the exec root, creating it if not found. This method + * only works for normalized, relative paths. + */ + public Artifact getDerivedArtifact(PathFragment execPath) { + Preconditions.checkArgument(!execPath.isAbsolute(), execPath); + Preconditions.checkArgument(execPath.isNormalized(), execPath); + // TODO(bazel-team): Check that either BinTools do not change over the life of the Blaze server, + // or require that a legitimate ArtifactOwner be passed in here to allow for ownership. + return getArtifact(execRoot.getRelative(execPath), Root.execRootAsDerivedRoot(execRoot), + execPath, ArtifactOwner.NULL_OWNER, null); + } + + private void validatePath(PathFragment rootRelativePath, Root root) { + Preconditions.checkArgument(!rootRelativePath.isAbsolute(), rootRelativePath); + Preconditions.checkArgument(rootRelativePath.isNormalized(), rootRelativePath); + Preconditions.checkArgument(root.getPath().startsWith(execRoot), "%s %s", root, execRoot); + Preconditions.checkArgument(!root.getPath().equals(execRoot), "%s %s", root, execRoot); + // TODO(bazel-team): this should only accept roots from derivedRoots. + //Preconditions.checkArgument(derivedRoots.contains(root), "%s not in %s", root, derivedRoots); + } + + /** + * Returns an artifact for a tool at the given root-relative path under the given root, creating + * it if not found. This method only works for normalized, relative paths. + * + * <p>The root must be below the execRoot, and the execPath of the resulting Artifact is computed + * as {@code root.getRelative(rootRelativePath).relativeTo(execRoot)}. + */ + // TODO(bazel-team): Don't allow root == execRoot. + public Artifact getDerivedArtifact(PathFragment rootRelativePath, Root root, + ArtifactOwner owner) { + validatePath(rootRelativePath, root); + Path path = root.getPath().getRelative(rootRelativePath); + return getArtifact(path, root, path.relativeTo(execRoot), owner, null); + } + + /** + * Returns an artifact that represents the output directory of a Fileset at the given + * root-relative path under the given root, creating it if not found. This method only works for + * normalized, relative paths. + * + * <p>The root must be below the execRoot, and the execPath of the resulting Artifact is computed + * as {@code root.getRelative(rootRelativePath).relativeTo(execRoot)}. + */ + public Artifact getFilesetArtifact(PathFragment rootRelativePath, Root root, + ArtifactOwner owner) { + validatePath(rootRelativePath, root); + Path path = root.getPath().getRelative(rootRelativePath); + return getArtifact(path, root, path.relativeTo(execRoot), owner, SpecialArtifactType.FILESET); + } + + public Artifact getConstantMetadataArtifact(PathFragment rootRelativePath, Root root, + ArtifactOwner owner) { + validatePath(rootRelativePath, root); + Path path = root.getPath().getRelative(rootRelativePath); + return getArtifact( + path, root, path.relativeTo(execRoot), owner, SpecialArtifactType.CONSTANT_METADATA); + } + + /** + * Returns the Artifact for the specified path, creating one if not found and + * setting the <code>root</code> and <code>execPath</code> to the + * specified values. + */ + private synchronized Artifact getArtifact(Path path, Root root, PathFragment execPath, + ArtifactOwner owner, @Nullable SpecialArtifactType type) { + Preconditions.checkNotNull(root); + Preconditions.checkNotNull(execPath); + + if (!root.isSourceRoot()) { + return createArtifact(path, root, execPath, owner, type); + } + + Artifact artifact = pathToSourceArtifact.get(execPath); + + if (artifact == null || !Objects.equals(artifact.getArtifactOwner(), owner)) { + // There really should be a safety net that makes it impossible to create two Artifacts + // with the same exec path but a different Owner, but we also need to reuse Artifacts from + // previous builds. + artifact = createArtifact(path, root, execPath, owner, type); + pathToSourceArtifact.put(execPath, artifact); + } else { + // TODO(bazel-team): Maybe we should check for equality of the fileset bit. However, that + // would require us to differentiate between artifact-creating and artifact-getting calls to + // getDerivedArtifact(). + Preconditions.checkState(root.equals(artifact.getRoot()), + "root for path %s changed from %s to %s", path, artifact.getRoot(), root); + Preconditions.checkState(execPath.equals(artifact.getExecPath()), + "execPath for path %s changed from %s to %s", path, artifact.getExecPath(), execPath); + } + return artifact; + } + + private Artifact createArtifact(Path path, Root root, PathFragment execPath, ArtifactOwner owner, + @Nullable SpecialArtifactType type) { + Preconditions.checkNotNull(owner, path); + if (type == null) { + return new Artifact(path, root, execPath, owner); + } else { + return new Artifact.SpecialArtifact(path, root, execPath, owner, type); + } + } + + @Override + public synchronized Artifact resolveSourceArtifact(PathFragment execPath) { + execPath = execPath.normalize(); + // First try a quick map lookup to see if the artifact already exists. + Artifact a = pathToSourceArtifact.get(execPath); + if (a != null) { + return a; + } + // Don't create an artifact if it's derived. + if (findDerivedRoot(execRoot.getRelative(execPath)) != null) { + return null; + } + // Must be a new source artifact, so probe the known packages to find the longest package + // prefix, and then use the corresponding source root to create a new artifact. + for (PathFragment dir = execPath.getParentDirectory(); dir != null; + dir = dir.getParentDirectory()) { + Root sourceRoot = packageRoots.get(PackageIdentifier.createInDefaultRepo(dir)); + if (sourceRoot != null) { + return getSourceArtifact(execPath, sourceRoot, ArtifactOwner.NULL_OWNER); + } + } + return null; // not a path that we can find... + } + + /** + * Finds the derived root for a full path by comparing against the known + * derived artifact roots. + * + * @param path a Path to resolve the root for + * @return the root for the path or null if no root can be determined + */ + @VisibleForTesting // for our own unit tests only. + synchronized Root findDerivedRoot(Path path) { + for (Root prefix : derivedRoots) { + if (path.startsWith(prefix.getPath())) { + return prefix; + } + } + return null; + } + + /** + * Returns all source artifacts created by the artifact factory. + */ + public synchronized Iterable<Artifact> getSourceArtifacts() { + return ImmutableList.copyOf(pathToSourceArtifact.values()); + } + + // Non-final only because clear()ing a map does not actually free the memory it took up, so we + // assign it to a new map in lieu of clearing. + private ConcurrentMap<PathFragment, Artifact> deserializedArtifacts = + new ConcurrentHashMap<>(); + + /** + * Returns the map of all artifacts that were deserialized this build. The caller should process + * them and then call {@link #clearDeserializedArtifacts}. + */ + public Map<PathFragment, Artifact> getDeserializedArtifacts() { + return deserializedArtifacts; + } + + /** Clears the map of deserialized artifacts. */ + public void clearDeserializedArtifacts() { + deserializedArtifacts = new ConcurrentHashMap<>(); + } + + /** + * Resolves an artifact based on its deserialized representation. The artifact can be either a + * source or a derived one. + * + * <p>Note: this method represents a hole in the usual contract that artifacts with a random path + * cannot be created. Unfortunately, we currently need this in some cases. + * + * @param execPath the exec path of the artifact + */ + public Artifact deserializeArtifact(PathFragment execPath, PackageRootResolver resolver) { + Preconditions.checkArgument(!execPath.isAbsolute(), execPath); + Path path = execRoot.getRelative(execPath); + Root root = findDerivedRoot(path); + + Artifact result; + if (root != null) { + result = getDerivedArtifact(path.relativeTo(root.getPath()), root, + Artifact.DESERIALIZED_MARKER_OWNER); + Artifact oldResult = deserializedArtifacts.putIfAbsent(execPath, result); + if (oldResult != null) { + result = oldResult; + } + return result; + } else { + Map<PathFragment, Root> sourceRoots = resolver.findPackageRoots(Lists.newArrayList(execPath)); + if (sourceRoots == null || sourceRoots.get(execPath) == null) { + return null; + } + return getSourceArtifact(execPath, sourceRoots.get(execPath), ArtifactOwner.NULL_OWNER); + } + } + + @Override + public Artifact lookupArtifactById(int artifactId) { + return artifactIdRegistry.lookupArtifactById(artifactId); + } + + @Override + public ImmutableList<Artifact> lookupArtifactsByIds(Iterable<Integer> artifactIds) { + return artifactIdRegistry.lookupArtifactsByIds(artifactIds); + } + + @Override + public int getArtifactId(Artifact artifact) { + return artifactIdRegistry.getArtifactId(artifact); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactIdRegistry.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactIdRegistry.java new file mode 100644 index 0000000..9edf684 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactIdRegistry.java
@@ -0,0 +1,108 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.MapMaker; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * A registry that keeps a map of artifacts to unique integer ids. + */ +class ArtifactIdRegistry implements ArtifactSerializer, ArtifactDeserializer { + + /** + * A sequence of registered artifacts. The position in the list is the artifact's id. + * + * <p>Synchronized using {@link #artifactIdsLock}. + */ + private final List<Artifact> serializedArtifactList = new ArrayList<>(); + + /** + * A map of artifacts to unique integer ids. + * + * <p>Writes to this map must be synchronized using {@link #artifactIdsLock}, in order to + * maintain consistency with {@link #serializedArtifactList}. + */ + private final ConcurrentMap<Artifact, Integer> serializedArtifactIds = + new MapMaker().concurrencyLevel(1).makeMap(); + + /** + * A lock for keeping {@code serializedArtifactList} and {@code serializedArtifactIds} in sync. + */ + private ReadWriteLock artifactIdsLock = new ReentrantReadWriteLock(); + + ArtifactIdRegistry() { + } + + @Override + public int getArtifactId(Artifact artifact) { + Integer artifactId = serializedArtifactIds.get(artifact); + if (artifactId == null) { + artifactId = assignArtifactId(artifact); + } + return artifactId; + } + + private Integer assignArtifactId(Artifact artifact) { + artifactIdsLock.writeLock().lock(); + try { + Integer artifactId = serializedArtifactIds.get(artifact); + if (artifactId == null) { + artifactId = serializedArtifactList.size(); + serializedArtifactList.add(artifact); + serializedArtifactIds.put(artifact, artifactId); + } + return artifactId; + } finally { + artifactIdsLock.writeLock().unlock(); + } + } + + @Override + public Artifact lookupArtifactById(int artifactId) { + artifactIdsLock.readLock().lock(); + try { + return serializedArtifactList.get(artifactId); + } finally { + artifactIdsLock.readLock().unlock(); + } + } + + @Override + public ImmutableList<Artifact> lookupArtifactsByIds(Iterable<Integer> artifactIds) { + int size = Iterables.size(artifactIds); + Artifact[] result = new Artifact[size]; + + int i = 0; + + artifactIdsLock.readLock().lock(); + try { + for (int artifactId : artifactIds) { + result[i] = serializedArtifactList.get(artifactId); + i++; + } + } finally { + artifactIdsLock.readLock().unlock(); + } + + return ImmutableList.copyOf(result); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactOwner.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactOwner.java new file mode 100644 index 0000000..458fb42 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactOwner.java
@@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.lib.syntax.Label; + +/** + * An interface for {@code LabelAndConfiguration}, or at least for a {@link Label}. Only tests and + * internal {@link Artifact}-generators should implement this interface -- otherwise, + * {@code LabelAndConfiguration} should be the only implementation. + */ +public interface ArtifactOwner { + Label getLabel(); + + @VisibleForTesting + public static final ArtifactOwner NULL_OWNER = new ArtifactOwner() { + @Override + public Label getLabel() { + return null; + } + + @Override + public String toString() { + return "NULL_OWNER"; + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactPrefixConflictException.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactPrefixConflictException.java new file mode 100644 index 0000000..42ad285 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactPrefixConflictException.java
@@ -0,0 +1,33 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Exception to indicate that one {@link Action} has an output artifact whose path is a prefix of an + * output of another action. Since the first path cannot be both a directory and a file, this would + * lead to an error if both actions were executed in the same build. + */ +public class ArtifactPrefixConflictException extends Exception { + public ArtifactPrefixConflictException(PathFragment firstPath, PathFragment secondPath, + Label firstOwner, Label secondOwner) { + super(String.format( + "output path '%s' (belonging to %s) is a prefix of output path '%s' (belonging to %s). " + + "These actions cannot be simultaneously present; please rename one of the output files " + + "or, as a last resort, run 'blaze clean' and then build just one of them", + firstPath, firstOwner, secondPath, secondOwner)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactResolver.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactResolver.java new file mode 100644 index 0000000..b74c17b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactResolver.java
@@ -0,0 +1,56 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * An interface for resolving artifact names to {@link Artifact} objects. Should only be used + * in the internal machinery of Blaze: rule implementations are not allowed to do this. + */ +public interface ArtifactResolver { + /** + * Returns the source Artifact for the specified path, creating it if not found and setting its + * root and execPath. + * + * @param execPath the path of the source artifact relative to the source root + * @param root the source root prefix of the path + * @param owner the artifact owner. + * @return the canonical source artifact for the given path + */ + Artifact getSourceArtifact(PathFragment execPath, Root root, ArtifactOwner owner); + + /** + * Returns the source Artifact for the specified path, creating it if not found and setting its + * root and execPath. + * + * @see #getSourceArtifact(PathFragment, Root, ArtifactOwner) + */ + Artifact getSourceArtifact(PathFragment execPath, Root root); + + /** + * Resolves a source Artifact given an execRoot-relative path. + * + * <p>Never creates or returns derived artifacts, only source artifacts. + * + * <p>Note: this method should only be used when the roots are unknowable, such as from the + * post-compile .d or manifest scanning methods. + * + * @param execPath the exec path of the artifact to resolve + * @return an existing or new source Artifact for the given execPath. Returns null if + * the root can not be determined and the artifact did not exist before. + */ + Artifact resolveSourceArtifact(PathFragment execPath); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactSerializer.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactSerializer.java new file mode 100644 index 0000000..703eeb7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactSerializer.java
@@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * An interface for creating artifacts from their serialized representations. + * + * @see ArtifactDeserializer + */ +public interface ArtifactSerializer { + + /** + * Returns a number that uniquely identifies an artifact. + * + * <p>The artifact can be retrieved again later by calling + * {@link ArtifactDeserializer#lookupArtifactById}. + */ + int getArtifactId(Artifact artifact); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BaseSpawn.java b/src/main/java/com/google/devtools/build/lib/actions/BaseSpawn.java new file mode 100644 index 0000000..dd879c2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/BaseSpawn.java
@@ -0,0 +1,214 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.extra.EnvironmentVariable; +import com.google.devtools.build.lib.actions.extra.SpawnInfo; +import com.google.devtools.build.lib.util.CommandDescriptionForm; +import com.google.devtools.build.lib.util.CommandFailureUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * Base implementation of a Spawn. + */ +@Immutable +public class BaseSpawn implements Spawn { + private final ImmutableList<String> arguments; + private final ImmutableMap<String, String> environment; + private final ImmutableMap<String, String> executionInfo; + private final ImmutableMap<PathFragment, Artifact> runfilesManifests; + private final ActionMetadata action; + private final ResourceSet localResources; + + /** + * Returns a new Spawn. The caller must not modify the parameters after the call; neither will + * this method. + */ + public BaseSpawn(List<String> arguments, + Map<String, String> environment, + Map<String, String> executionInfo, + Map<PathFragment, Artifact> runfilesManifests, + ActionMetadata action, + ResourceSet localResources) { + this.arguments = ImmutableList.copyOf(arguments); + this.environment = ImmutableMap.copyOf(environment); + this.executionInfo = ImmutableMap.copyOf(executionInfo); + this.runfilesManifests = ImmutableMap.copyOf(runfilesManifests); + this.action = action; + this.localResources = localResources; + } + + /** + * Returns a new Spawn. + */ + public BaseSpawn(List<String> arguments, + Map<String, String> environment, + Map<String, String> executionInfo, + // TODO(bazel-team): have this always be non-null. + @Nullable Artifact runfilesManifest, + ActionMetadata action, + ResourceSet localResources) { + this(arguments, environment, executionInfo, + ((runfilesManifest != null) + ? ImmutableMap.of(runfilesForFragment(new PathFragment(arguments.get(0))), + runfilesManifest) + : ImmutableMap.<PathFragment, Artifact>of()), + action, localResources); + } + + public static PathFragment runfilesForFragment(PathFragment pathFragment) { + return pathFragment.getParentDirectory().getChild(pathFragment.getBaseName() + ".runfiles"); + } + + /** + * Returns a new Spawn. + */ + public BaseSpawn(List<String> arguments, + Map<String, String> environment, + Map<String, String> executionInfo, + ActionMetadata action, + ResourceSet localResources) { + this(arguments, environment, executionInfo, + ImmutableMap.<PathFragment, Artifact>of(), action, localResources); + } + + @Override + public boolean isRemotable() { + return !executionInfo.containsKey("local"); + } + + @Override + public final ImmutableMap<String, String> getExecutionInfo() { + return executionInfo; + } + + @Override + public String asShellCommand(Path workingDir) { + return asShellCommand(getArguments(), workingDir, getEnvironment()); + } + + @Override + public ImmutableMap<PathFragment, Artifact> getRunfilesManifests() { + return runfilesManifests; + } + + @Override + public ImmutableList<Artifact> getFilesetManifests() { + return ImmutableList.<Artifact>of(); + } + + @Override + public SpawnInfo getExtraActionInfo() { + SpawnInfo.Builder info = SpawnInfo.newBuilder(); + + info.addAllArgument(getArguments()); + for (Map.Entry<String, String> variable : getEnvironment().entrySet()) { + info.addVariable(EnvironmentVariable.newBuilder() + .setName(variable.getKey()) + .setValue(variable.getValue()).build()); + } + for (ActionInput input : getInputFiles()) { + // Explicitly ignore middleman artifacts here. + if (!(input instanceof Artifact) || !((Artifact) input).isMiddlemanArtifact()) { + info.addInputFile(input.getExecPathString()); + } + } + info.addAllOutputFile(ActionInputHelper.toExecPaths(getOutputFiles())); + return info.build(); + } + + @Override + public ImmutableList<String> getArguments() { + // TODO(bazel-team): this method should be final, as the correct value of the args can be + // injected in the ctor. + return arguments; + } + + @Override + public ImmutableMap<String, String> getEnvironment() { + if (getRunfilesManifests().size() != 1) { + return environment; + } + + ImmutableMap.Builder<String, String> env = ImmutableMap.builder(); + env.putAll(environment); + for (Map.Entry<PathFragment, Artifact> e : getRunfilesManifests().entrySet()) { + // TODO(bazel-team): Unify these into a single env variable. + env.put("JAVA_RUNFILES", e.getKey().getPathString() + "/"); + env.put("PYTHON_RUNFILES", e.getKey().getPathString() + "/"); + } + return env.build(); + } + + @Override + public Iterable<? extends ActionInput> getInputFiles() { + return action.getInputs(); + } + + @Override + public Collection<? extends ActionInput> getOutputFiles() { + return action.getOutputs(); + } + + @Override + public ActionMetadata getResourceOwner() { + return action; + } + + @Override + public ResourceSet getLocalResources() { + return localResources; + } + + @Override + public ActionOwner getOwner() { return action.getOwner(); } + + @Override + public String getMnemonic() { return action.getMnemonic(); } + + /** + * Convert a working dir + environment map + arg list into a Bourne shell + * command. + */ + public static String asShellCommand(Collection<String> arguments, + Path workingDirectory, + Map<String, String> environment) { + // We print this command out in such a way that it can safely be + // copied+pasted as a Bourne shell command. This is extremely valuable for + // debugging. + return CommandFailureUtils.describeCommand(CommandDescriptionForm.COMPLETE, + arguments, environment, workingDirectory.getPathString()); + } + + /** + * A local spawn requiring zero resources. + */ + public static class Local extends BaseSpawn { + public Local(List<String> arguments, Map<String, String> environment, ActionMetadata action) { + super(arguments, environment, ImmutableMap.<String, String>of("local", ""), + action, ResourceSet.ZERO); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BipartiteVisitor.java b/src/main/java/com/google/devtools/build/lib/actions/BipartiteVisitor.java new file mode 100644 index 0000000..61803c8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/BipartiteVisitor.java
@@ -0,0 +1,98 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import java.util.HashMap; +import java.util.Map; + +/** + * A visitor helper class for bipartite graphs. The alternate kinds of nodes + * are arbitrarily designated "black" or "white". + * + * <p> Subclasses implement the black() and white() hook functions which are + * called as nodes are visited. The class holds a mapping from each node to a + * small integer; this is available to subclasses if they wish. + */ +public abstract class BipartiteVisitor<BLACK, WHITE> { + + protected BipartiteVisitor() {} + + private int nextNodeId = 0; + + // Maps each visited black node to a small integer. + protected final Map<BLACK, Integer> visitedBlackNodes = new HashMap<>(); + + // Maps each visited white node to a small integer. + protected final Map<WHITE, Integer> visitedWhiteNodes = new HashMap<>(); + + /** + * Visit the specified black node. If this node has not already been + * visited, the black() hook is called and true is returned; otherwise, + * false is returned. + */ + public final boolean visitBlackNode(BLACK blackNode) { + if (blackNode == null) { throw new NullPointerException(); } + if (!visitedBlackNodes.containsKey(blackNode)) { + visitedBlackNodes.put(blackNode, nextNodeId++); + black(blackNode); + return true; + } + return false; + } + + /** + * Visit all specified black nodes. + */ + public final void visitBlackNodes(Iterable<BLACK> blackNodes) { + for (BLACK blackNode : blackNodes) { + visitBlackNode(blackNode); + } + } + + /** + * Visit the specified white node. If this node has not already been + * visited, the white() hook is called and true is returned; otherwise, + * false is returned. + */ + public final boolean visitWhiteNode(WHITE whiteNode) { + if (whiteNode == null) { + throw new NullPointerException(); + } + if (!visitedWhiteNodes.containsKey(whiteNode)) { + visitedWhiteNodes.put(whiteNode, nextNodeId++); + white(whiteNode); + return true; + } + return false; + } + + /** + * Visit all specified white nodes. + */ + public final void visitWhiteNodes(Iterable<WHITE> whiteNodes) { + for (WHITE whiteNode : whiteNodes) { + visitWhiteNode(whiteNode); + } + } + + /** + * Called whenever a white node is visited. Hook for subclasses. + */ + protected abstract void white(WHITE whiteNode); + + /** + * Called whenever a black node is visited. Hook for subclasses. + */ + protected abstract void black(BLACK blackNode); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BlazeExecutor.java b/src/main/java/com/google/devtools/build/lib/actions/BlazeExecutor.java new file mode 100644 index 0000000..fd3c3d9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/BlazeExecutor.java
@@ -0,0 +1,233 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.OptionsClassProvider; + +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * The Executor class provides a dynamic abstraction of the various actual primitive system + * operations that might be performed during a build step. + * + * <p>Constructions of this class might perform distributed execution, "virtual" execution for + * testing purposes, or just print out the sequence of commands that would be executed, like Make's + * "-n" option. + */ +@ThreadSafe +public final class BlazeExecutor implements Executor { + + private final Path outputPath; + private final boolean verboseFailures; + private final boolean showSubcommands; + private final Path execRoot; + private final Reporter reporter; + private final EventBus eventBus; + private final Clock clock; + private final OptionsClassProvider options; + private AtomicBoolean inExecutionPhase; + + private final Map<String, SpawnActionContext> spawnActionContextMap; + private final Map<Class<? extends ActionContext>, ActionContext> contextMap = + new HashMap<>(); + + /** + * Constructs an Executor, bound to a specified output base path, and which + * will use the specified reporter to announce SUBCOMMAND events, + * the given event bus to delegate events and the given output streams + * for streaming output. The list of + * strategy implementation classes is used to construct instances of the + * strategies mapped by their declared abstract type. This list is uniquified + * before using. Each strategy instance is created with a reference to this + * Executor as well as the given options object. + * <p> + * Don't forget to call startBuildRequest() and stopBuildRequest() for each + * request, and shutdown() when you're done with this executor. + */ + public BlazeExecutor(Path execRoot, + Path outputPath, + Reporter reporter, + EventBus eventBus, + Clock clock, + OptionsClassProvider options, + boolean verboseFailures, + boolean showSubcommands, + List<ActionContext> contextImplementations, + Map<String, ActionContext> spawnContextMap, + Iterable<ActionContextProvider> contextProviders) + throws ExecutorInitException { + this.outputPath = outputPath; + this.verboseFailures = verboseFailures; + this.showSubcommands = showSubcommands; + this.execRoot = execRoot; + this.reporter = reporter; + this.eventBus = eventBus; + this.clock = clock; + this.options = options; + this.inExecutionPhase = new AtomicBoolean(false); + + // We need to keep only the last occurrences of the entries in contextImplementations + // (so we respect insertion order but also instantiate them only once). + LinkedHashSet<ActionContext> allContexts = new LinkedHashSet<>(); + allContexts.addAll(contextImplementations); + + ImmutableMap.Builder<String, SpawnActionContext> spawnMapBuilder = ImmutableMap.builder(); + for (Map.Entry<String, ActionContext> entry: spawnContextMap.entrySet()) { + spawnMapBuilder.put(entry.getKey(), (SpawnActionContext) entry.getValue()); + allContexts.add(entry.getValue()); + } + + for (ActionContext context : contextImplementations) { + ExecutionStrategy annotation = context.getClass().getAnnotation(ExecutionStrategy.class); + if (annotation != null) { + contextMap.put(annotation.contextType(), context); + } + } + this.spawnActionContextMap = spawnMapBuilder.build(); + + for (ActionContextProvider factory : contextProviders) { + factory.executorCreated(allContexts); + } + } + + @Override + public Path getExecRoot() { + return execRoot; + } + + @Override + public EventHandler getEventHandler() { + return reporter; + } + + @Override + public EventBus getEventBus() { + return eventBus; + } + + @Override + public Clock getClock() { + return clock; + } + + @Override + public boolean reportsSubcommands() { + return showSubcommands; + } + + /** + * Report a subcommand event to this Executor's Reporter and, if action + * logging is enabled, post it on its EventBus. + */ + @Override + public void reportSubcommand(String reason, String message) { + reporter.handle(new Event(EventKind.SUBCOMMAND, null, "# " + reason + "\n" + message)); + } + + /** + * This method is called before the start of the execution phase of each + * build request. + */ + public void executionPhaseStarting() { + Preconditions.checkState(!inExecutionPhase.getAndSet(true)); + Profiler.instance().startTask(ProfilerTask.INFO, "Initializing executors"); + Profiler.instance().completeTask(ProfilerTask.INFO); + } + + /** + * This method is called after the end of the execution phase of each build + * request (even if there was an interrupt). + */ + public void executionPhaseEnding() { + if (!inExecutionPhase.get()) { + return; + } + + Profiler.instance().startTask(ProfilerTask.INFO, "Shutting down executors"); + Profiler.instance().completeTask(ProfilerTask.INFO); + inExecutionPhase.set(false); + } + + public static void shutdownHelperPool(EventHandler reporter, ExecutorService pool, + String name) { + pool.shutdownNow(); + + boolean interrupted = false; + while (true) { + try { + if (!pool.awaitTermination(10, TimeUnit.SECONDS)) { + reporter.handle(Event.warn(name + " threadpool shutdown took greater than ten seconds")); + } + break; + } catch (InterruptedException e) { + interrupted = true; + } + } + + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + @Override + public <T extends ActionContext> T getContext(Class<? extends T> type) { + Preconditions.checkArgument(type != SpawnActionContext.class, + "should use getSpawnActionContext instead"); + return type.cast(contextMap.get(type)); + } + + /** + * Returns the {@link SpawnActionContext} to use for the given mnemonic. If no execution mode is + * set, then it returns the default strategy for spawn actions. + */ + @Override + public SpawnActionContext getSpawnActionContext(String mnemonic) { + SpawnActionContext context = spawnActionContextMap.get(mnemonic); + return context == null ? spawnActionContextMap.get("") : context; + } + + /** Returns true iff the --verbose_failures option was enabled. */ + @Override + public boolean getVerboseFailures() { + return verboseFailures; + } + + /** Returns the options associated with the execution. */ + @Override + public OptionsClassProvider getOptions() { + return options; + } + + public Path getOutputPath() { + return outputPath; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BuildFailedException.java b/src/main/java/com/google/devtools/build/lib/actions/BuildFailedException.java new file mode 100644 index 0000000..088ba60 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/BuildFailedException.java
@@ -0,0 +1,81 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.syntax.Label; + +/** + * This exception gets thrown if there were errors during the execution phase of + * the build. + * + * <p>The argument to the constructor may be null if the thrower has already + * printed an error message; in this case, no error message should be printed by + * the catcher. (Typically, this happens when the builder is unsuccessful and + * {@code --keep_going} was specified. This error corresponds to one or more + * actions failing, but since those actions' failures will be reported + * separately, the exception carries no message and is just used for control + * flow.) + */ +@ThreadSafe +public class BuildFailedException extends Exception { + private final boolean catastrophic; + private final Action action; + private final Iterable<Label> rootCauses; + private final boolean errorAlreadyShown; + + public BuildFailedException() { + this(null); + } + + public BuildFailedException(String message) { + this(message, false, null, ImmutableList.<Label>of()); + } + + public BuildFailedException(String message, boolean catastrophic) { + this(message, catastrophic, null, ImmutableList.<Label>of()); + } + + public BuildFailedException(String message, boolean catastrophic, + Action action, Iterable<Label> rootCauses) { + this(message, catastrophic, action, rootCauses, false); + } + + public BuildFailedException(String message, boolean catastrophic, + Action action, Iterable<Label> rootCauses, boolean errorAlreadyShown) { + super(message); + this.catastrophic = catastrophic; + this.rootCauses = ImmutableList.copyOf(rootCauses); + this.action = action; + this.errorAlreadyShown = errorAlreadyShown; + } + + public boolean isCatastrophic() { + return catastrophic; + } + + public Action getAction() { + return action; + } + + public Iterable<Label> getRootCauses() { + return rootCauses; + } + + public boolean isErrorAlreadyShown() { + return errorAlreadyShown || getMessage() == null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BuilderUtils.java b/src/main/java/com/google/devtools/build/lib/actions/BuilderUtils.java new file mode 100644 index 0000000..72ce13335 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/BuilderUtils.java
@@ -0,0 +1,57 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * Methods needed by {@code SkyframeBuilder}. + */ +public final class BuilderUtils { + + private BuilderUtils() {} + + /** + * Figure out why an action's execution failed and rethrow the right kind of exception. + */ + public static void rethrowCause(Exception e) throws BuildFailedException, TestExecException { + Throwable cause = e.getCause(); + Throwable innerCause = cause.getCause(); + if (innerCause instanceof TestExecException) { + throw (TestExecException) innerCause; + } + if (cause instanceof ActionExecutionException) { + ActionExecutionException actionExecutionCause = (ActionExecutionException) cause; + // Sometimes ActionExecutionExceptions are caused by Actions with no owner. + String message = + (actionExecutionCause.getLocation() != null) ? + (actionExecutionCause.getLocation().print() + " " + cause.getMessage()) : + e.getMessage(); + throw new BuildFailedException(message, actionExecutionCause.isCatastrophe(), + actionExecutionCause.getAction(), actionExecutionCause.getRootCauses(), + /*errorAlreadyShown=*/ !actionExecutionCause.showError()); + } else if (cause instanceof MissingInputFileException) { + throw new BuildFailedException(cause.getMessage()); + } else if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } else if (cause instanceof Error) { + throw (Error) cause; + } else { + /* + * This should never happen - we should only get exceptions listed in the exception + * specification for ExecuteBuildAction.call(). + */ + throw new IllegalArgumentException("action terminated with " + + "unexpected exception: " + cause.getMessage(), cause); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/CachedActionEvent.java b/src/main/java/com/google/devtools/build/lib/actions/CachedActionEvent.java new file mode 100644 index 0000000..35db7d6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/CachedActionEvent.java
@@ -0,0 +1,45 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * This event is fired during the build if an action was in the action cache. + */ +public class CachedActionEvent { + + private final Action action; + private final long nanoTimeStart; + + /** + * Create an event for an action that was cached. + * + * @param action the cached action + * @param nanoTimeStart the time when the action was started. This allow us to + * record more accurately the time spend by the action, since we execute some code before + * deciding if we execute the action or not. + */ + public CachedActionEvent(Action action, long nanoTimeStart) { + this.action = action; + this.nanoTimeStart = nanoTimeStart; + } + + public Action getAction() { + return action; + } + + public long getNanoTimeStart() { + return nanoTimeStart; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ChangedArtifactsMessage.java b/src/main/java/com/google/devtools/build/lib/actions/ChangedArtifactsMessage.java new file mode 100644 index 0000000..9177cba --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ChangedArtifactsMessage.java
@@ -0,0 +1,34 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableSet; + +import java.util.Set; + +/** + * Used to signal when the incremental builder has found the set of changed artifacts. + */ +public class ChangedArtifactsMessage { + + private final Set<Artifact> artifacts; + + public ChangedArtifactsMessage(Set<Artifact> changedArtifacts) { + this.artifacts = ImmutableSet.copyOf(changedArtifacts); + } + + public Set<Artifact> getChangedArtifacts() { + return artifacts; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ChangedFilesMessage.java b/src/main/java/com/google/devtools/build/lib/actions/ChangedFilesMessage.java new file mode 100644 index 0000000..56f6882 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ChangedFilesMessage.java
@@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Set; + +/** + * A message sent conveying a set of changed files. + */ +public class ChangedFilesMessage { + + private final Set<PathFragment> changedFiles; + + public ChangedFilesMessage(Set<PathFragment> changedFiles) { + this.changedFiles = ImmutableSet.copyOf(changedFiles); + } + + public Set<PathFragment> getChangedFiles() { + return changedFiles; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElement.java b/src/main/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElement.java new file mode 100644 index 0000000..d9d875d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElement.java
@@ -0,0 +1,220 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile; + +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; + +import javax.annotation.Nullable; + +/** + * A Multimap-like object that is actually a {@link ConcurrentMap} of {@code SmallSet}s to avoid + * the memory penalties of a {@code Multimap} while preserving concurrency guarantees, and + * retrieving a consistent "head" element. Operations are guaranteed to reflect a consistent view of + * a {@code SetMultimap}, although most methods are not implemented. + */ +final class ConcurrentMultimapWithHeadElement<K, V> { + private final ConcurrentMap<K, SmallSet<V>> map = Maps.newConcurrentMap(); + + /** + * Remove (key, val) pair from the multimap. If this removes the current 'head' element + * for a key, then another randomly chosen element becomes the current head. + * + * <p>Until the next (possibly concurrent) {@link #putAndGet}(key, val) call, {@link #get}(key) + * will never return val. + */ + void remove(K key, V val) { + SmallSet<V> entry = getEntry(key); + if (entry != null) { + entry.remove(val); + if (entry.get() == null) { + // Remove entry completely from map if dead. + map.remove(key, entry); + } + } + } + + /** + * Return some value val such that (key, val) is in the multimap. If there is always at least one + * entry for key in the multimap during the lifetime of this method call, it will not return null. + */ + @Nullable V get(K key) { + SmallSet<V> entry = getEntry(key); + return (entry != null) ? entry.get() : null; + } + + /** + * Adds (key, val) to the multimap. Returns the head element for key, either val or another + * already-stored value. + */ + V putAndGet(K key, V val) { + V result = null; + while (result == null) { + // If another thread concurrently removes the only remaining value from the entry, this + // putAndGet will return null, since the entry is about to be removed from the map. In that + // case, we obtain a fresh entry from the map and do the put on it. + result = getOrCreateEntry(key).putAndGet(val); + } + return result; + } + + /** + * Obtain the entry for key, adding it to the underlying map if no entry was previously present. + */ + private SmallSet<V> getOrCreateEntry(K key) { + SmallSet<V> entry = new SmallSet<V>(); + SmallSet<V> oldEntry = map.putIfAbsent(key, entry); + if (oldEntry != null) { + return oldEntry; + } + return entry; + } + + /** + * Obtain the entry for key, returning null if no entry was present in the underlying map. + */ + private SmallSet<V> getEntry(K key) { + return map.get(key); + } + + /** + * Clears the multimap. May not be called concurrently with any other methods. + */ + @ThreadHostile + void clear() { + map.clear(); + } + + /** + * Wrapper for a {@code #Set} that will probably have at most one element. Keeps the first element + * in a separate variable for fast reading/writing and to save space if more than one element is + * never written to this set. We always have the invariant that {@link #first} is null only if + * {@link #rest} is null. + */ + private static class SmallSet<T> { + /* + * What is this 'volatile' on first and where's the lock on the read path? + * + * Volatile is an alternative to locking that works only in very limited situations, such as + * simple field reads and writes. Writes from one thread to 'first' happen before reads from + * other threads. When used correctly, it can have the same correctness properties as a + * 'synchronized' but is much faster on most hardware. + * + * Here, volatile is used to eliminate locks on the read path. Since get() is merely fetching + * the contents of 'first', it meets the criteria for a safe volatile read. In the mutator + * methods, care is taken to write only correct values to 'first'; intermediate and incomplete + * values do not get written to the field. This means that whenever 'first' is replaced, it is + * immediately replaced with the next correct value. Therefore, it is a safe volatile write. + * + * Other more complex relationships that need to be maintained during the mutate are maintained + * with the Object monitor. Since they do not impact the read path (only 'first' matters), the + * lock is sufficient for writes and unnecessary for 'first' reads. + * + * Documentation on volatile: + * http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/package-summary.html#MemoryVisibility + * (java.util.concurrent package docs) + */ + + private volatile T first = null; + private Set<T> rest = null; + + /* + * We may have a race where one thread tries to remove a small set from the map while another + * thread tries to add to it. If the second thread loses the race, it will add to a set that is + * no longer in the map. To prevent that, once a small set is ever empty, we mark it "dead" by + * setting {@code rest} to a {@code TOMBSTONE} value, and (and subsequently remove it from the + * map). No modifications to a set can happen after the {@code TOMBSTONE} value is set. Thus, + * the thread trying to add a new value to a set will fail, and knows to retrieve the entry anew + * from the map and try again. + */ + private static final Set<Object> TOMBSTONE = ImmutableSet.of(); + + /** + * Return some value in the SmallSet. + * + * <p>If there is always at least one value in the SmallSet during the lifetime of this call, + * it will not return null, since by the invariant, {@link #first} must be non-null. + */ + private T get() { + return first; + } + + /** + * Adds val to the SmallSet. Returns some element of the SmallSet. + */ + private synchronized T putAndGet(T elt) { + Preconditions.checkNotNull(elt); + if (isDead()) { + return null; + } + if (elt.equals(first)) { + return first; + } + if (first == null) { + Preconditions.checkState(rest == null, elt); + first = elt; + return first; + } + if (rest == null) { + rest = Sets.newHashSet(); + } + rest.add(elt); + return first; + } + + /** + * Remove val from the SmallSet, if it is present. + */ + private synchronized void remove(T elt) { + Preconditions.checkNotNull(elt); + if (isDead()) { + return; + } + if (elt.equals(first)) { + // Normalize to enforce invariant "first is null only if rest is empty." + if (rest != null) { + Iterator<T> it = rest.iterator(); + first = it.next(); + it.remove(); + if (!it.hasNext()) { + rest = null; + } + } else { + first = null; + markDead(); + } + } else if ((rest != null) && rest.remove(elt) && rest.isEmpty()) { // side-effect: remove + rest = null; + } + } + + private boolean isDead() { + Preconditions.checkState(rest != TOMBSTONE || first == null, + "%s present in tombstoned SmallSet, but tombstoned SmallSets should be empty", first); + return rest == TOMBSTONE; + } + + @SuppressWarnings("unchecked") // Cast of TOMBSTONE. Ok since TOMBSTONE is empty immutable set. + private void markDead() { + rest = (Set<T>) TOMBSTONE; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/DelegateSpawn.java b/src/main/java/com/google/devtools/build/lib/actions/DelegateSpawn.java new file mode 100644 index 0000000..4b4048b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/DelegateSpawn.java
@@ -0,0 +1,106 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.extra.SpawnInfo; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; + +/** + * A delegating spawn that allow us to overwrite certain methods while maintaining the original + * behavior for non-overwritten methods. + */ +public class DelegateSpawn implements Spawn { + + private final Spawn spawn; + + public DelegateSpawn(Spawn spawn){ + this.spawn = spawn; + } + + @Override + public final ImmutableMap<String, String> getExecutionInfo() { + return spawn.getExecutionInfo(); + } + + @Override + public boolean isRemotable() { + return spawn.isRemotable(); + } + + @Override + public ImmutableList<Artifact> getFilesetManifests() { + return spawn.getFilesetManifests(); + } + + @Override + public String asShellCommand(Path workingDir) { + return spawn.asShellCommand(workingDir); + } + + @Override + public ImmutableMap<PathFragment, Artifact> getRunfilesManifests() { + return spawn.getRunfilesManifests(); + } + + @Override + public SpawnInfo getExtraActionInfo() { + return spawn.getExtraActionInfo(); + } + + @Override + public ImmutableList<String> getArguments() { + return spawn.getArguments(); + } + + @Override + public ImmutableMap<String, String> getEnvironment() { + return spawn.getEnvironment(); + } + + @Override + public Iterable<? extends ActionInput> getInputFiles() { + return spawn.getInputFiles(); + } + + @Override + public Collection<? extends ActionInput> getOutputFiles() { + return spawn.getOutputFiles(); + } + + @Override + public ActionMetadata getResourceOwner() { + return spawn.getResourceOwner(); + } + + @Override + public ResourceSet getLocalResources() { + return spawn.getLocalResources(); + } + + @Override + public ActionOwner getOwner() { + return spawn.getOwner(); + } + + @Override + public String getMnemonic() { + return spawn.getMnemonic(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/DigestOfDirectoryException.java b/src/main/java/com/google/devtools/build/lib/actions/DigestOfDirectoryException.java new file mode 100644 index 0000000..92cc12b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/DigestOfDirectoryException.java
@@ -0,0 +1,28 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import java.io.IOException; + +/** + * Exception thrown when we try to digest a directory in {@code ActionInputFileCache}. + * + */ +public class DigestOfDirectoryException extends IOException { + + public DigestOfDirectoryException(String message) { + super(message); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/EnvironmentalExecException.java b/src/main/java/com/google/devtools/build/lib/actions/EnvironmentalExecException.java new file mode 100644 index 0000000..e2c493c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/EnvironmentalExecException.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * An ExecException which is results from an external problem on the user's + * local system. + * + * <p>Note that this is fundamentally different exception then the higher level + * LocalEnvironmentException, which is thrown from the BuildTool. That exception + * is thrown when the higher levels of Blaze decide to exit. + * + * <p>This exception is thrown when a low level error is encountered in the + * strategy or client protocol layers. This does not necessarily mean we will + * exit; we may just retry the action. + */ +public class EnvironmentalExecException extends ExecException { + + public EnvironmentalExecException(String message, Throwable cause) { + super(message, cause); + } + + public EnvironmentalExecException(String message) { + super(message); + } + + public EnvironmentalExecException(String message, Throwable cause, boolean catastrophe) { + super(message, cause, catastrophe); + } + + public EnvironmentalExecException(String message, boolean catastrophe) { + super(message, catastrophe); + } + + @Override + public ActionExecutionException toActionExecutionException(String messagePrefix, + boolean verboseFailures, Action action) { + if (verboseFailures) { + return new ActionExecutionException(messagePrefix + " failed" + getMessage(), this, action, + isCatastrophic()); + } else { + return new ActionExecutionException(messagePrefix + " failed" + getMessage(), action, + isCatastrophic()); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ExecException.java b/src/main/java/com/google/devtools/build/lib/actions/ExecException.java new file mode 100644 index 0000000..a2edc84 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ExecException.java
@@ -0,0 +1,96 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * An exception indication that the execution of an action has failed OR could + * not be attempted OR could not be finished OR had something else wrong. + * + * <p>The four main kinds of failure are broadly defined as follows: + * + * <p>USER_INPUT which means it had something to do with what the user told us + * to do. This failure should satisfy the invariant that it would happen + * identically again if all other things are equal. + * + * <p>ENVIRONMENT which is loosely defined as anything which is generally out of + * scope for a blaze evaluation. As a rule of thumb, these are any errors would + * not necessarily happen again given constant input. + * + * <p>INTERRUPTION conditions arise from being unable to complete an evaluation + * for whatever reason. + * + * <p>INTERNAL_ERROR would happen because of anything which arises from within + * blaze itself but is generally unexpected to ever occur for any user input. + * + * <p>The class is a catch-all for both failures of actions and failures to + * evaluate actions properly. + * + * <p>Invariably, all low level ExecExceptions are caught by various specific + * ConfigurationAction classes and re-raised as ActionExecutionExceptions. + */ +public abstract class ExecException extends Exception { + + private final boolean catastrophe; + + public ExecException(String message, boolean catastrophe) { + super(message); + this.catastrophe = catastrophe; + } + + public ExecException(String message) { + this(message, false); + } + + public ExecException(String message, Throwable cause, boolean catastrophe) { + super(message + ": " + cause.getMessage(), cause); + this.catastrophe = catastrophe; + } + + public ExecException(String message, Throwable cause) { + this(message, cause, false); + } + + /** + * Catastrophic exceptions should stop the build, even if --keep_going. + */ + public boolean isCatastrophic() { + return catastrophe; + } + + /** + * Returns a new ActionExecutionException without a message prefix. + * @param action failed action + * @return ActionExecutionException object describing the action failure + */ + public ActionExecutionException toActionExecutionException(Action action) { + // In all ExecException implementations verboseFailures argument used only to determine should + // we pass ExecException as cause of ActionExecutionException. So use this method only + // if you need this information inside of ActionExecutionexception. + return toActionExecutionException("", true, action); + } + + /** + * Returns a new ActionExecutionException given a message prefix describing the action type as a + * noun. When appropriate (we use some heuristics to decide), produces an abbreviated message + * incorporating just the termination status if available. + * + * @param messagePrefix describes the action type as noun + * @param verboseFailures true if user requested verbose output with flag --verbose_failures + * @param action failed action + * @return ActionExecutionException object describing the action failure + */ + public abstract ActionExecutionException toActionExecutionException(String messagePrefix, + boolean verboseFailures, Action action); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ExecutionStrategy.java b/src/main/java/com/google/devtools/build/lib/actions/ExecutionStrategy.java new file mode 100644 index 0000000..d86ea6c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ExecutionStrategy.java
@@ -0,0 +1,37 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation that marks strategies that extend the execution phase behavior of Blaze. + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface ExecutionStrategy { + /** + * The names this strategy is available under on the command line. + */ + String[] name() default {}; + + /** + * Returns the action context this strategy implements. + */ + Class<? extends ActionContext> contextType(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/Executor.java b/src/main/java/com/google/devtools/build/lib/actions/Executor.java new file mode 100644 index 0000000..f3904bb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/Executor.java
@@ -0,0 +1,103 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.OptionsClassProvider; + +/** + * The Executor provides the context for the execution of actions. It is only valid during the + * execution phase, and references should not be cached. + * + * <p>This class provides the actual logic to execute actions. The platonic ideal of this system + * is that {@link Action}s are immutable objects that tell Blaze <b>what</b> to do and + * <link>ActionContext</link>s tell Blaze <b>how</b> to do it (however, we do have an "execute" + * method on actions now). + * + * <p>In theory, most of the methods below would not exist and they would be methods on action + * contexts, but in practice, that would require some refactoring work so we are stuck with these + * for the time being. + * + * <p>In theory, we could also merge {@link Executor} with {@link ActionExecutionContext}, since + * they both provide services to actions being executed and are passed to almost the same places. + */ +public interface Executor { + /** + * A marker interface for classes that provide services for actions during execution. + * + * <p>Interfaces extending this one should also be annotated with {@link ActionContextMarker}. + */ + public interface ActionContext { + } + + /** + * Returns the execution root. This is the directory underneath which Blaze builds its entire + * output working tree, including the source symlink forest. All build actions are executed + * relative to this directory. + */ + Path getExecRoot(); + + /** + * Returns a clock. This is not hermetic, and should only be used for build info actions or + * performance measurements / reporting. + */ + Clock getClock(); + + /** + * The EventBus for the current build. + */ + EventBus getEventBus(); + + /** + * Returns whether failures should have verbose error messages. + */ + boolean getVerboseFailures(); + + /** + * Returns the command line options of the Blaze command being executed. + */ + OptionsClassProvider getOptions(); + + /** + * Whether this Executor reports subcommands. If not, reportSubcommand has no effect. + * This is provided so the caller of reportSubcommand can avoid wastefully constructing the + * subcommand string. + */ + boolean reportsSubcommands(); + + /** + * Report a subcommand event to this Executor's Reporter and, if action + * logging is enabled, post it on its EventBus. + */ + void reportSubcommand(String reason, String message); + + /** + * An event listener to report messages to. Errors that signal a action failure should + * use ActionExecutionException. + */ + EventHandler getEventHandler(); + + /** + * Looks up and returns an action context implementation of the given interface type. + */ + <T extends ActionContext> T getContext(Class<? extends T> type); + + /** + * Returns the action context implementation for spawn actions with a given mnemonic. + */ + SpawnActionContext getSpawnActionContext(String mnemonic); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ExecutorInitException.java b/src/main/java/com/google/devtools/build/lib/actions/ExecutorInitException.java new file mode 100644 index 0000000..21369fb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ExecutorInitException.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; + +/** + * An exception that is thrown when an executor can't be initialized. + */ +public class ExecutorInitException extends AbruptExitException { + + public ExecutorInitException(String message) { + this(message, ExitCode.LOCAL_ENVIRONMENTAL_ERROR); + } + + public ExecutorInitException(String message, ExitCode exitCode) { + super(message, exitCode); + } + + public ExecutorInitException(String message, Throwable cause) { + super(message + ": " + cause.getMessage(), + ExitCode.LOCAL_ENVIRONMENTAL_ERROR, + cause); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FailAction.java b/src/main/java/com/google/devtools/build/lib/actions/FailAction.java new file mode 100644 index 0000000..6c15b31 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/FailAction.java
@@ -0,0 +1,73 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +/** + * FailAction is an Action that always fails to execute. (Used as scaffolding + * for rules we haven't yet implemented. Also useful for testing.) + */ +@ThreadSafe +public final class FailAction extends AbstractAction { + + private static final String GUID = "626cb78a-810f-4af3-979c-ee194955f04c"; + + private final String errorMessage; + + public FailAction(ActionOwner owner, Iterable<Artifact> outputs, String errorMessage) { + super(owner, ImmutableList.<Artifact>of(), outputs); + this.errorMessage = errorMessage; + } + + @Override + public Artifact getPrimaryInput() { + return null; + } + + @Override + public void execute( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException { + throw new ActionExecutionException(errorMessage, this, false); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + + @Override + protected String computeKey() { + return GUID; + } + + @Override + protected String getRawProgressMessage() { + return "Building unsupported rule " + getOwner().getLabel() + + " located at " + getOwner().getLocation(); + } + + @Override + public String describeStrategy(Executor executor) { + return ""; + } + + @Override + public String getMnemonic() { + return "Fail"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FilesetOutputSymlink.java b/src/main/java/com/google/devtools/build/lib/actions/FilesetOutputSymlink.java new file mode 100644 index 0000000..1ae15ba --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/FilesetOutputSymlink.java
@@ -0,0 +1,81 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** Definition of a symlink in the output tree of a Fileset rule. */ +public final class FilesetOutputSymlink { + private static final String STRIPPED_METADATA = "<stripped-for-testing>"; + + /** Final name of the symlink relative to the Fileset's output directory. */ + public final PathFragment name; + + /** Target of the symlink. Depending on FilesetEntry.symlinks it may be relative or absolute. */ + public final PathFragment target; + + /** Opaque metadata about the link and its target; should change if either of them changes. */ + public final String metadata; + + @VisibleForTesting + public FilesetOutputSymlink(PathFragment name, PathFragment target) { + this.name = name; + this.target = target; + this.metadata = STRIPPED_METADATA; + } + + /** + * @param name relative path under the Fileset's output directory, including FilesetEntry.destdir + * with and FilesetEntry.strip_prefix applied (if applicable) + * @param target relative or absolute value of the link + * @param metadata opaque metadata about the link and its target; should change if either the link + * or its target changes + */ + public FilesetOutputSymlink(PathFragment name, PathFragment target, String metadata) { + this.name = Preconditions.checkNotNull(name); + this.target = Preconditions.checkNotNull(target); + this.metadata = Preconditions.checkNotNull(metadata); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || !obj.getClass().equals(getClass())) { + return false; + } + FilesetOutputSymlink o = (FilesetOutputSymlink) obj; + return name.equals(o.name) && target.equals(o.target) && metadata.equals(o.metadata); + } + + @Override + public int hashCode() { + return Objects.hashCode(name, target, metadata); + } + + @Override + public String toString() { + if (metadata.equals(STRIPPED_METADATA)) { + return String.format("FilesetOutputSymlink(%s -> %s)", + name.getPathString(), target.getPathString()); + } else { + return String.format("FilesetOutputSymlink(%s -> %s | metadata=%s)", + name.getPathString(), target.getPathString(), metadata); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParams.java b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParams.java new file mode 100644 index 0000000..1464f73 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParams.java
@@ -0,0 +1,165 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Optional; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; + +import java.util.Set; + +/** + * Parameters of a filesystem traversal requested by a Fileset rule. + * + * <p>This object stores the details of the traversal request, e.g. whether it's a direct or nested + * traversal (see {@link #getDirectTraversal()} and {@link #getNestedTraversal()}) or who the owner + * of the traversal is. + */ +public interface FilesetTraversalParams { + + /** + * Abstraction of the root directory of a {@link DirectTraversal}. + * + * <ul> + * <li>The root of package traversals is the package directory, i.e. the parent of the BUILD file. + * <li>The root of "recursive" directory traversals is the directory's path. + * <li>The root of "file" traversals is the path of the file (or directory, or symlink) itself. + * </ul> + * + * <p>For the meaning of "recursive" and "file" traversals see {@link DirectTraversal}. + */ + interface DirectTraversalRoot { + + /** + * Returns the root part of the full path. + * + * <p>This is typically the workspace root or some output tree's root (e.g. genfiles, binfiles). + */ + Path getRootPart(); + + /** + * Returns the {@link #getRootPart() root}-relative part of the path. + * + * <p>This is typically the source directory under the workspace or the output file under an + * output directory. + */ + PathFragment getRelativePart(); + + /** Returns a {@link RootedPath} composed of the root and relative parts. */ + RootedPath asRootedPath(); + } + + /** + * Describes a request for a direct filesystem traversal. + * + * <p>"Direct" means this corresponds to an actual filesystem traversal as opposed to traversing + * another Fileset rule, which is called a "nested" traversal. + * + * <p>Direct traversals can further be divided into two categories, "file" traversals and + * "recursive" traversals. + * + * <p>File traversal requests are created when the FilesetEntry.files attribute is defined; one + * file traversal request is created for each entry. + * + * <p>Recursive traversal requests are created when the FilesetEntry.files attribute is + * unspecified; one recursive traversal request is created for the FilesetEntry.srcdir. + * + * <p>See {@link DirectTraversal#getRoot()} for more details. + */ + interface DirectTraversal { + + /** Returns the root of the traversal; see {@link DirectTraversalRoot}. */ + DirectTraversalRoot getRoot(); + + /** + * Returns true if this traversal refers to a whole package. + * + * <p>In that case the root (see {@link #getRoot()}) refers to the path of the package. + * + * <p>Package traversals are always recursive (see {@link #isRecursive()}) and are never + * generated (see {@link #isGenerated()}). + */ + boolean isPackage(); + + /** + * Returns true if this is a "recursive traversal", i.e. created from FilesetEntry.srcdir. + * + * <p>This type of traversal is created when the FilesetEntry doesn't define a "files" list. + * When it does, the traversal is referred to as a "file traversal". When it doesn't, but the + * srcdir points to another Fileset, it is called a "nested" traversal. + * + * <p>Recursive traversals got their name from recursively traversing a directory structure. + * These are usually whole-package traversals, i.e. when FilesetEntry.srcdir refers to a BUILD + * file (see {@link #isPackage()}), but sometimes the srcdir references a input or output + * directory (the latter being generated by a local genrule) or a symlink (which must point to a + * directory; enforced during action execution). + * + * <p>The files in the results of a recursive traversal are all under the {@link #getRoot() + * root}. The root's path is stripped from the results. + * + * <p>N.B.: "file traversals" can also be recursive if the entry in FilesetEntry.files, for + * which the traversal parameters were created, turned out to be a directory. The difference + * lies in how the output paths are computed (with recursive traversals, the directory's name + * is stripped; with file traversals it is not, modulo usage of strip_prefix and the excludes + * attributes), and how directory symlinks are handled (in "recursive traversals" they are + * expanded just like normal directories, subsequent directory symlinks under them are *not* + * expanded though; they are not expanded at all in "file traversals"). + */ + boolean isRecursive(); + + /** Returns true if the root points to a generated file, symlink or directory. */ + boolean isGenerated(); + + /** Returns true if input symlinks should be dereferenced; false if copied. */ + boolean isFollowingSymlinks(); + + /** Returns the desired behavior when the traversal hits a subpackage. */ + boolean getCrossPackageBoundary(); + } + + /** Label of the Fileset rule that owns this traversal. */ + Label getOwnerLabel(); + + /** Returns the directory under the output path where the files will be mapped. May be empty. */ + PathFragment getDestPath(); + + /** Returns a list of file basenames to be excluded from the output. May be empty. */ + Set<String> getExcludedFiles(); + + /** + * Returns the parameters of the direct traversal request, if any. + * + * <p>A direct traversal is anything that's not a nested traversal, e.g. traversal of a package or + * directory (when FilesetEntry.srcdir is specified) or traversal of a single file (when + * FilesetEntry.files is specified). See {@link DirectTraversal} for more detail. + * + * <p>The value is present if and only if {@link #getNestedTraversal} is absent. + */ + Optional<DirectTraversal> getDirectTraversal(); + + /** + * Returns the parameters of the nested traversal request, if any. + * + * <p>A nested traversal is the traversal of another Fileset referenced by FilesetEntry.srcdir. + * + * <p>The value is present if and only if {@link #getDirectTraversal} is absent. + */ + Optional<FilesetTraversalParams> getNestedTraversal(); + + /** Adds the fingerprint of this traversal object. */ + void fingerprint(Fingerprint fp); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParamsFactory.java b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParamsFactory.java new file mode 100644 index 0000000..acc0e86 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParamsFactory.java
@@ -0,0 +1,314 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Ordering; +import com.google.devtools.build.lib.actions.FilesetTraversalParams.DirectTraversal; +import com.google.devtools.build.lib.actions.FilesetTraversalParams.DirectTraversalRoot; +import com.google.devtools.build.lib.syntax.FilesetEntry.SymlinkBehavior; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; + +import java.util.Set; + +import javax.annotation.Nullable; + +/** Factory of {@link FilesetTraversalParams}. */ +public final class FilesetTraversalParamsFactory { + + /** + * Creates parameters for a recursive traversal request in a package. + * + * <p>"Recursive" means that a directory is traversed along with all of its subdirectories. Such + * a traversal is created when FilesetEntry.files is unspecified. + * + * @param ownerLabel the rule that created this object + * @param buildFile path of the BUILD file of the package to traverse + * @param destPath path in the Fileset's output directory that will be the root of files found + * in this directory + * @param excludes optional; set of files directly under this package's directory to exclude; + * files in subdirectories cannot be excluded + * @param symlinkBehaviorMode what to do with symlinks + * @param crossPkgBoundary whether to traverse a subdirectory if it's also a subpackage (contains + * a BUILD file) + */ + public static FilesetTraversalParams recursiveTraversalOfPackage(Label ownerLabel, + Artifact buildFile, PathFragment destPath, @Nullable Set<String> excludes, + SymlinkBehavior symlinkBehaviorMode, boolean crossPkgBoundary) { + Preconditions.checkState(buildFile.isSourceArtifact(), "%s", buildFile); + return new DirectoryTraversalParams(ownerLabel, DirectTraversalRootImpl.forPackage(buildFile), + true, destPath, excludes, symlinkBehaviorMode, crossPkgBoundary, true, false); + } + + /** + * Creates parameters for a recursive traversal request in a directory. + * + * <p>"Recursive" means that a directory is traversed along with all of its subdirectories. Such + * a traversal is created when FilesetEntry.files is unspecified. + * + * @param ownerLabel the rule that created this object + * @param directoryToTraverse path of the directory to traverse + * @param destPath path in the Fileset's output directory that will be the root of files found + * in this directory + * @param excludes optional; set of files directly below this directory to exclude; files in + * subdirectories cannot be excluded + * @param symlinkBehaviorMode what to do with symlinks + * @param crossPkgBoundary whether to traverse a subdirectory if it's also a subpackage (contains + * a BUILD file) + */ + public static FilesetTraversalParams recursiveTraversalOfDirectory(Label ownerLabel, + Artifact directoryToTraverse, PathFragment destPath, @Nullable Set<String> excludes, + SymlinkBehavior symlinkBehaviorMode, boolean crossPkgBoundary) { + return new DirectoryTraversalParams(ownerLabel, + DirectTraversalRootImpl.forFileOrDirectory(directoryToTraverse), false, destPath, + excludes, symlinkBehaviorMode, crossPkgBoundary, true, + !directoryToTraverse.isSourceArtifact()); + } + + /** + * Creates parameters for a file traversal request. + * + * <p>Such a traversal is created for every entry in FilesetEntry.files, when it is specified. + * + * @param ownerLabel the rule that created this object + * @param fileToTraverse the file to traverse; "traversal" means that if this file is actually a + * directory or a symlink to one then it'll be traversed as one + * @param destPath path in the Fileset's output directory that will be the name of this file's + * respective symlink there, or the root of files found (in case this is a directory) + * @param symlinkBehaviorMode what to do with symlinks + * @param crossPkgBoundary whether to traverse a subdirectory if it's also a subpackage (contains + * a BUILD file) + */ + public static FilesetTraversalParams fileTraversal(Label ownerLabel, Artifact fileToTraverse, + PathFragment destPath, SymlinkBehavior symlinkBehaviorMode, boolean crossPkgBoundary) { + return new DirectoryTraversalParams(ownerLabel, + DirectTraversalRootImpl.forFileOrDirectory(fileToTraverse), false, destPath, null, + symlinkBehaviorMode, crossPkgBoundary, false, !fileToTraverse.isSourceArtifact()); + } + + /** + * Creates traversal request parameters for a FilesetEntry wrapping another Fileset. + * + * @param ownerLabel the rule that created this object + * @param nested the traversal params that were used for the nested (inner) Fileset + * @param destDir path in the Fileset's output directory that will be the root of files coming + * from the nested Fileset + * @param excludes optional; set of files directly below (not in a subdirectory of) the nested + * Fileset that should be excluded from the outer Fileset + */ + public static FilesetTraversalParams nestedTraversal(Label ownerLabel, + FilesetTraversalParams nested, PathFragment destDir, @Nullable Set<String> excludes) { + // When srcdir is another Fileset, then files must be null so strip_prefix must also be null. + return new NestedTraversalParams(ownerLabel, nested, destDir, excludes); + } + + private abstract static class ParamsCommon implements FilesetTraversalParams { + private final Label ownerLabel; + private final PathFragment destDir; + private final ImmutableSet<String> excludes; + + ParamsCommon(Label ownerLabel, PathFragment destDir, @Nullable Set<String> excludes) { + this.ownerLabel = ownerLabel; + this.destDir = destDir; + if (excludes == null) { + this.excludes = ImmutableSet.<String>of(); + } else { + // Order the set for the sake of deterministic fingerprinting. + this.excludes = ImmutableSet.copyOf(Ordering.natural().immutableSortedCopy(excludes)); + } + } + + @Override + public Label getOwnerLabel() { + return ownerLabel; + } + + @Override + public Set<String> getExcludedFiles() { + return excludes; + } + + @Override + public PathFragment getDestPath() { + return destDir; + } + + protected final void commonFingerprint(Fingerprint fp) { + fp.addPath(destDir); + if (!excludes.isEmpty()) { + fp.addStrings(excludes); + } + } + } + + private static final class DirectTraversalImpl implements DirectTraversal { + private final DirectTraversalRoot root; + private final boolean isPackage; + private final boolean followSymlinks; + private final boolean crossPkgBoundary; + private final boolean isRecursive; + private final boolean isGenerated; + + DirectTraversalImpl(DirectTraversalRoot root, boolean isPackage, boolean followSymlinks, + boolean crossPkgBoundary, boolean isRecursive, boolean isGenerated) { + this.root = root; + this.isPackage = isPackage; + this.followSymlinks = followSymlinks; + this.crossPkgBoundary = crossPkgBoundary; + this.isRecursive = isRecursive; + this.isGenerated = isGenerated; + } + + @Override + public DirectTraversalRoot getRoot() { + return root; + } + + @Override + public boolean isPackage() { + return isPackage; + } + + @Override + public boolean isRecursive() { + return isRecursive; + } + + @Override + public boolean isGenerated() { + return isGenerated; + } + + @Override + public boolean isFollowingSymlinks() { + return followSymlinks; + } + + @Override + public boolean getCrossPackageBoundary() { + return crossPkgBoundary; + } + + void fingerprint(Fingerprint fp) { + fp.addPath(root.asRootedPath().asPath()); + fp.addBoolean(isPackage); + fp.addBoolean(followSymlinks); + fp.addBoolean(isRecursive); + fp.addBoolean(isGenerated); + fp.addBoolean(crossPkgBoundary); + } + } + + private static final class DirectoryTraversalParams extends ParamsCommon { + private final DirectTraversalImpl traversal; + + DirectoryTraversalParams(Label ownerLabel, + DirectTraversalRoot root, + boolean isPackage, + PathFragment destPath, + @Nullable Set<String> excludes, + SymlinkBehavior symlinkBehaviorMode, + boolean crossPkgBoundary, + boolean isRecursive, + boolean isGenerated) { + super(ownerLabel, destPath, excludes); + traversal = new DirectTraversalImpl(root, isPackage, + symlinkBehaviorMode == SymlinkBehavior.DEREFERENCE, crossPkgBoundary, isRecursive, + isGenerated); + } + + @Override + public Optional<DirectTraversal> getDirectTraversal() { + return Optional.<DirectTraversal>of(traversal); + } + + @Override + public Optional<FilesetTraversalParams> getNestedTraversal() { + return Optional.absent(); + } + + @Override + public void fingerprint(Fingerprint fp) { + commonFingerprint(fp); + traversal.fingerprint(fp); + } + } + + private static final class NestedTraversalParams extends ParamsCommon { + private final FilesetTraversalParams nested; + + public NestedTraversalParams(Label ownerLabel, FilesetTraversalParams nested, + PathFragment destDir, @Nullable Set<String> excludes) { + super(ownerLabel, destDir, excludes); + this.nested = nested; + } + + @Override + public Optional<DirectTraversal> getDirectTraversal() { + return Optional.absent(); + } + + @Override + public Optional<FilesetTraversalParams> getNestedTraversal() { + return Optional.of(nested); + } + + @Override + public void fingerprint(Fingerprint fp) { + commonFingerprint(fp); + nested.fingerprint(fp); + } + } + + private static final class DirectTraversalRootImpl implements DirectTraversalRoot { + private final Path rootDir; + private final PathFragment relativeDir; + + static DirectTraversalRoot forPackage(Artifact buildFile) { + return new DirectTraversalRootImpl(buildFile.getRoot().getPath(), + buildFile.getRootRelativePath().getParentDirectory()); + } + + static DirectTraversalRoot forFileOrDirectory(Artifact fileOrDirectory) { + return new DirectTraversalRootImpl(fileOrDirectory.getRoot().getPath(), + fileOrDirectory.getRootRelativePath()); + } + + private DirectTraversalRootImpl(Path rootDir, PathFragment relativeDir) { + this.rootDir = rootDir; + this.relativeDir = relativeDir; + } + + @Override + public Path getRootPart() { + return rootDir; + } + + @Override + public PathFragment getRelativePart() { + return relativeDir; + } + + @Override + public RootedPath asRootedPath() { + return RootedPath.toRootedPath(rootDir, relativeDir); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java new file mode 100644 index 0000000..5fc6013 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
@@ -0,0 +1,302 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Splitter; +import com.google.common.io.Files; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.ProcMeminfoParser; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * This class estimates the local host's resource capacity. + */ +@ThreadCompatible +public final class LocalHostCapacity { + + private static final Logger LOG = Logger.getLogger(LocalHostCapacity.class.getName()); + + /** + * Stores parsed /proc/stat CPU time counters. + * See {@link LocalHostCapacity#getCpuTimes(String)} for details. + */ + @Immutable + private final static class CpuTimes { + private final long idleJiffies; + private final long totalJiffies; + + CpuTimes(long idleJiffies, long totalJiffies) { + this.idleJiffies = idleJiffies; + this.totalJiffies = totalJiffies; + } + + /** + * Return idle CPU ratio using current and previous CPU readings or 0 if + * ratio is undefined. + */ + double getIdleRatio(CpuTimes prevTimes) { + if (prevTimes.totalJiffies == 0 || totalJiffies == prevTimes.totalJiffies) { + return 0; + } + return ((double)(idleJiffies - prevTimes.idleJiffies) / + (double)(totalJiffies - prevTimes.totalJiffies)); + } + } + + /** + * Used to store available local CPU and RAM resources information. + * See {@link LocalHostCapacity#getFreeResources(FreeResources)} for details. + */ + public static final class FreeResources { + + private final Clock clock; + private final CpuTimes cpuTimes; + private final long lastTimestamp; + private final double freeCpu; + private final double freeMb; + private final long interval; + + private FreeResources(Clock localClock, ProcMeminfoParser memInfo, String statContent, + FreeResources prevStats) { + clock = localClock; + lastTimestamp = localClock.nanoTime(); + freeMb = ProcMeminfoParser.kbToMb(memInfo.getFreeRamKb()); + cpuTimes = getCpuTimes(statContent); + if (prevStats == null) { + interval = 0; + freeCpu = 0.0; + } else { + interval = lastTimestamp - prevStats.lastTimestamp; + freeCpu = getLocalHostCapacity().getCpuUsage() * cpuTimes.getIdleRatio(prevStats.cpuTimes); + } + } + + /** + * Returns amount of available RAM in MB. + */ + public double getFreeMb() { return freeMb; } + + /** + * Returns average available CPU resources (as a fraction of the CPU core, + * so one fully CPU-bound thread should consume exactly 1.0 CPU resource). + */ + public double getAvgFreeCpu() { return freeCpu; } + + /** + * Returns interval in ms between CPU load measurements used to calculate + * average available CPU resources. + */ + public long getInterval() { return interval / 1000000; } + + /** + * Returns age of available resource data in ms. + */ + public long getReadingAge() { + return (clock.nanoTime() - lastTimestamp) / 1000000; + } + } + + // Disables getFreeResources() if error occured during reading or parsing + // /proc/* information. + @VisibleForTesting + static boolean isDisabled; + + // If /proc/* information is not available, assume 3000 MB and 2 CPUs. + private static ResourceSet DEFAULT_RESOURCES = new ResourceSet(3000.0, 2.0, 1.0); + + private LocalHostCapacity() {} + + /** + * Estimates of the local host's resource capacity, + * obtained by reading /proc/cpuinfo and /proc/meminfo. + */ + private static ResourceSet localHostCapacity; + + /** + * Estimates of the local host's resource capacity, + * obtained by reading /proc/cpuinfo and /proc/meminfo. + */ + public static ResourceSet getLocalHostCapacity() { + if (localHostCapacity == null) { + localHostCapacity = getLocalHostCapacity("/proc/cpuinfo", "/proc/meminfo"); + } + return localHostCapacity; + } + + /** + * Returns new FreeResources object populated with free RAM information from + * /proc/meminfo and CPU load information from the /proc/stat. First call + * should be made with null parameter to instantiate new FreeResources object. + * Subsequent calls will use information inside it to calculate average CPU + * load over the time between calls and to calculate amount of free CPU + * resources and generate new FreeResources() instance. + * + * If information is not available due to error, functionality will be disabled + * and method will always return null. + */ + public static FreeResources getFreeResources(FreeResources stats) { + return getFreeResources(BlazeClock.instance(), "/proc/meminfo", "/proc/stat", stats); + } + + private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n').omitEmptyStrings(); + + @VisibleForTesting + static int getLogicalCpuCount(String cpuinfoContent) { + Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoContent); + int count = 0; + for (String line : lines) { + if(line.startsWith("processor")) { + count++; + } + } + if (count == 0) { + throw new IllegalArgumentException("Can't locate processor in the /proc/cpuinfo"); + } + return count; + } + + @VisibleForTesting + static int getPhysicalCpuCount(String cpuinfoContent, int logicalCpuCount) { + Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoContent); + Set<String> uniq = new HashSet<>(); + for (String line : lines) { + if(line.startsWith("physical id")) { + uniq.add(line); + } + } + int physicalCpuCount = uniq.size(); + if (physicalCpuCount == 0) { + physicalCpuCount = logicalCpuCount; + } + return physicalCpuCount; + } + + @VisibleForTesting + static int getCoresPerCpu(String cpuinfoFileContent) { + Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoFileContent); + Set<String> uniq = new HashSet<>(); + for (String line : lines) { + if(line.startsWith("core id")) { + uniq.add(line); + } + } + int coresPerCpu = uniq.size(); + if (coresPerCpu == 0) { + coresPerCpu = 1; + } + return coresPerCpu; + } + + /** + * Parses cpu line of the /proc/stats, calculates number of idle and total + * CPU jiffies and returns CpuTimes instance with that information. + * + * Total CPU time includes <b>all</b> time reported to be spent by the CPUs, + * including so-called "stolen" time - time spent by other VMs on the same + * workstation. + */ + private static CpuTimes getCpuTimes(String statContent) { + String[] cpuStats = statContent.substring(0, statContent.indexOf('\n')).trim().split(" +"); + // Supported versions of /proc/stat (Linux kernel 2.6.x) must contain either + // 9 or 10 fields: + // "cpu" utime ultime stime idle iowait irq softirq steal(since 2.6.11) 0 + // We are interested in total time (sum of all columns) and idle time. + if (cpuStats.length < 9 | cpuStats.length > 10) { + throw new IllegalArgumentException("Unrecognized /proc/stat format"); + } + if (!cpuStats[0].equals("cpu")) { + throw new IllegalArgumentException("/proc/stat does not start with cpu keyword"); + } + long idleCpuJiffies = Long.parseLong(cpuStats[4]); // "idle" column. + long totalJiffies = 0; + for (int i = 1; i < cpuStats.length; i++) { + totalJiffies += Long.parseLong(cpuStats[i]); + } + long totalCpuJiffies = totalJiffies; + return new CpuTimes(idleCpuJiffies, totalCpuJiffies); + } + + @VisibleForTesting + static ResourceSet getLocalHostCapacity(String cpuinfoFile, String meminfoFile) { + try { + String cpuinfoContent = readContent(cpuinfoFile); + ProcMeminfoParser memInfo = new ProcMeminfoParser(meminfoFile); + int logicalCpuCount = getLogicalCpuCount(cpuinfoContent); + int physicalCpuCount = getPhysicalCpuCount(cpuinfoContent, logicalCpuCount); + int coresPerCpu = getCoresPerCpu(cpuinfoContent); + int totalCores = coresPerCpu * physicalCpuCount; + boolean hyperthreading = (logicalCpuCount != totalCores); + double ramMb = ProcMeminfoParser.kbToMb(memInfo.getTotalKb()); + final double EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU = 0.6; + return new ResourceSet( + ramMb, + logicalCpuCount * (hyperthreading ? EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU + : 1.0), + 1.0); + } catch (IOException | IllegalArgumentException e) { + disableProcFsUse(e); + return DEFAULT_RESOURCES; + } + } + + @VisibleForTesting + static FreeResources getFreeResources(Clock localClock, String meminfoFile, String statFile, + FreeResources prevStats) { + if (isDisabled) { return null; } + try { + String statContent = readContent(statFile); + return new FreeResources(localClock, new ProcMeminfoParser(meminfoFile), + statContent, prevStats); + } catch (IOException | IllegalArgumentException e) { + disableProcFsUse(e); + return null; + } + } + + /** + * For testing purposes only. Do not use it. + */ + @VisibleForTesting + static void setLocalHostCapacity(ResourceSet resources) { + localHostCapacity = resources; + isDisabled = false; + } + + private static String readContent(String filename) throws IOException { + return Files.toString(new File(filename), Charset.defaultCharset()); + } + + /** + * Disables use of /proc filesystem. Called internally when unexpected + * exception is caught. + */ + private static void disableProcFsUse(Throwable cause) { + LoggingUtil.logToRemote(Level.WARNING, "Unable to read system load or capacity", cause); + LOG.log(Level.WARNING, "Unable to read system load or capacity", cause); + isDisabled = true; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/MapBasedActionGraph.java b/src/main/java/com/google/devtools/build/lib/actions/MapBasedActionGraph.java new file mode 100644 index 0000000..2788f2f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/MapBasedActionGraph.java
@@ -0,0 +1,64 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +/** + * An action graph that resolves generating actions by looking them up in a map. + */ +@ThreadSafe +public final class MapBasedActionGraph implements MutableActionGraph { + + private final ConcurrentMultimapWithHeadElement<Artifact, Action> generatingActionMap = + new ConcurrentMultimapWithHeadElement<Artifact, Action>(); + + @Override + @Nullable + public Action getGeneratingAction(Artifact artifact) { + return generatingActionMap.get(artifact); + } + + @Override + public void registerAction(Action action) throws ActionConflictException { + for (Artifact artifact : action.getOutputs()) { + Action previousAction = generatingActionMap.putAndGet(artifact, action); + if (previousAction != null && previousAction != action + && !Actions.canBeShared(action, previousAction)) { + generatingActionMap.remove(artifact, action); + throw new ActionConflictException(artifact, previousAction, action); + } + } + } + + @Override + public void unregisterAction(Action action) { + for (Artifact artifact : action.getOutputs()) { + generatingActionMap.remove(artifact, action); + Action otherAction = generatingActionMap.get(artifact); + Preconditions.checkState(otherAction == null + || (otherAction != action && Actions.canBeShared(action, otherAction)), + "%s %s", action, otherAction); + } + } + + @Override + public void clear() { + generatingActionMap.clear(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/MiddlemanAction.java b/src/main/java/com/google/devtools/build/lib/actions/MiddlemanAction.java new file mode 100644 index 0000000..9d3a2b8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/MiddlemanAction.java
@@ -0,0 +1,107 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.syntax.Label; + +/** + * An action that depends on a set of inputs and creates a single output file whenever it + * runs. This is useful for bundling up a bunch of dependencies that are shared + * between individual targets in the action graph; for example generated header files. + */ +public class MiddlemanAction extends AbstractAction { + + public static final String MIDDLEMAN_MNEMONIC = "Middleman"; + private final String description; + private final MiddlemanType middlemanType; + + /** + * Constructs a new {@link MiddlemanAction}. + * + * @param owner the owner of the action, usually a {@code ConfiguredTarget} + * @param inputs inputs of the middleman, i.e. the files it acts as a placeholder for + * @param stampFile the output of the middleman expansion; must be a middleman artifact (see + * {@link Artifact#isMiddlemanArtifact()}) + * @param description a short description for the action, for progress messages + * @param middlemanType the type of the middleman + * @throws IllegalArgumentException if {@code stampFile} is not a middleman artifact + */ + public MiddlemanAction(ActionOwner owner, Iterable<Artifact> inputs, Artifact stampFile, + String description, MiddlemanType middlemanType) { + super(owner, inputs, ImmutableList.of(stampFile)); + Preconditions.checkNotNull(middlemanType); + Preconditions.checkArgument(stampFile.isMiddlemanArtifact(), stampFile); + this.description = description; + this.middlemanType = middlemanType; + } + + @Override + public final void execute( + ActionExecutionContext actionExecutionContext) { + throw new IllegalStateException("MiddlemanAction should never be executed"); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + + @Override + protected String computeKey() { + // TODO(bazel-team): Need to take middlemanType into account here. + // Only the set of inputs matters, and the dependency checker is + // responsible for considering those. + return ""; + } + + /** + * Returns the type of the middleman. + */ + @Override + public MiddlemanType getActionType() { + return middlemanType; + } + + @Override + protected String getRawProgressMessage() { + return null; // users don't really want to know about Middlemen. + } + + @Override + public String prettyPrint() { + return description + " for " + Label.print(getOwner().getLabel()); + } + + @Override + public String describeStrategy(Executor executor) { + return ""; + } + + @Override + public String getMnemonic() { + return MIDDLEMAN_MNEMONIC; + } + + /** + * Creates a new middleman action. + */ + public static Action create(ActionRegistry env, ActionOwner owner, + Iterable<Artifact> inputs, Artifact stampFile, String purpose, MiddlemanType middlemanType) { + MiddlemanAction action = new MiddlemanAction(owner, inputs, stampFile, purpose, middlemanType); + env.registerAction(action); + return action; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/MiddlemanFactory.java b/src/main/java/com/google/devtools/build/lib/actions/MiddlemanFactory.java new file mode 100644 index 0000000..85a1450 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/MiddlemanFactory.java
@@ -0,0 +1,188 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action.MiddlemanType; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Iterator; + +/** + * A factory to create middleman objects. + */ +@ThreadSafe +public final class MiddlemanFactory { + + private final ArtifactFactory artifactFactory; + private final ActionRegistry actionRegistry; + + public MiddlemanFactory( + ArtifactFactory artifactFactory, ActionRegistry actionRegistry) { + this.artifactFactory = Preconditions.checkNotNull(artifactFactory); + this.actionRegistry = Preconditions.checkNotNull(actionRegistry); + } + + /** + * Creates a {@link MiddlemanType#AGGREGATING_MIDDLEMAN aggregating} middleman. + * + * @param owner the owner of the action that will be created; must not be null + * @param purpose the purpose for which this middleman is created. This should be a string which + * is suitable for use as a filename. A single rule may have many middlemen with distinct + * purposes. + * @param inputs the set of artifacts for which the created artifact is to be the middleman. + * @param middlemanDir the directory in which to place the middleman. + * @return null iff {@code inputs} is empty; the single element of {@code inputs} if there's only + * one; a new aggregating middleman for the {@code inputs} otherwise + */ + public Artifact createAggregatingMiddleman( + ActionOwner owner, String purpose, Iterable<Artifact> inputs, Root middlemanDir) { + if (hasExactlyOneInput(inputs)) { // Optimization: No middleman for just one input. + return Iterables.getOnlyElement(inputs); + } + Pair<Artifact, Action> result = createMiddleman( + owner, Label.print(owner.getLabel()), purpose, inputs, middlemanDir, + MiddlemanType.AGGREGATING_MIDDLEMAN); + return result == null ? null : result.getFirst(); + } + + /** + * Returns <code>null</code> iff inputs is empty. Returns the sole element + * of inputs iff <code>inputs.size()==1</code>. Otherwise, returns a + * middleman artifact and creates a middleman action that generates that + * artifact. + * + * @param owner the owner of the action that will be created. + * @param owningArtifact the artifact of the file for which the runfiles + * should be created. There may be at most one set of runfiles for + * an owning artifact, unless the owning artifact is null. There + * may be at most one set of runfiles per owner with a null + * owning artifact. + * Further, if the owning Artifact is non-null, the owning Artifacts' + * root-relative path must be unique and the artifact must be part + * of the runfiles tree for which this middleman is created. Usually + * this artifact will be an executable program. + * @param inputs the set of artifacts for which the created artifact is to be + * the middleman. + * @param middlemanDir the directory in which to place the middleman. + */ + public Artifact createRunfilesMiddleman( + ActionOwner owner, Artifact owningArtifact, Iterable<Artifact> inputs, Root middlemanDir) { + if (hasExactlyOneInput(inputs)) { // Optimization: No middleman for just one input. + return Iterables.getOnlyElement(inputs); + } + String middlemanPath = owningArtifact == null + ? Label.print(owner.getLabel()) + : owningArtifact.getRootRelativePath().getPathString(); + return createMiddleman(owner, middlemanPath, "runfiles", inputs, middlemanDir, + MiddlemanType.RUNFILES_MIDDLEMAN).getFirst(); + } + + private <T> boolean hasExactlyOneInput(Iterable<T> iterable) { + Iterator<T> it = iterable.iterator(); + if (!it.hasNext()) { + return false; + } + it.next(); + return !it.hasNext(); + } + + /** + * Creates a {@link MiddlemanType#ERROR_PROPAGATING_MIDDLEMAN error-propagating} middleman. + * + * @param owner the owner of the action that will be created. May not be null. + * @param middlemanName a unique file name for the middleman artifact in the {@code middlemanDir}; + * in practice this is usually the owning rule's label (so it gets escaped as such) + * @param purpose the purpose for which this middleman is created. This should be a string which + * is suitable for use as a filename. A single rule may have many middlemen with distinct + * purposes. + * @param inputs the set of artifacts for which the created artifact is to be the middleman; must + * not be null or empty + * @param middlemanDir the directory in which to place the middleman. + * @return a middleman that enforces scheduling order (just like a scheduling middleman) and + * propagates errors, but is ignored by the dependency checker + * @throws IllegalArgumentException if {@code inputs} is null or empty + */ + public Artifact createErrorPropagatingMiddleman(ActionOwner owner, String middlemanName, + String purpose, Iterable<Artifact> inputs, Root middlemanDir) { + Preconditions.checkArgument(inputs != null); + Preconditions.checkArgument(!Iterables.isEmpty(inputs)); + // We must always create this middleman even if there is only one input. + return createMiddleman(owner, middlemanName, purpose, inputs, middlemanDir, + MiddlemanType.ERROR_PROPAGATING_MIDDLEMAN).getFirst(); + } + + /** + * Returns the same artifact as {@code createErrorPropagatingMiddleman} would return, + * but doesn't create any action. + */ + public Artifact getErrorPropagatingMiddlemanArtifact(String middlemanName, String purpose, + Root middlemanDir) { + return getStampFileArtifact(middlemanName, purpose, middlemanDir); + } + + /** + * Creates both normal and scheduling middlemen. + * + * <p>Note: there's no need to synchronize this method; the only use of a field is via a call to + * another synchronized method (getArtifact()). + * + * @return null iff {@code inputs} is null or empty; the middleman file and the middleman action + * otherwise + */ + private Pair<Artifact, Action> createMiddleman( + ActionOwner owner, String middlemanName, String purpose, Iterable<Artifact> inputs, + Root middlemanDir, MiddlemanType middlemanType) { + if (inputs == null || Iterables.isEmpty(inputs)) { + return null; + } + + Artifact stampFile = getStampFileArtifact(middlemanName, purpose, middlemanDir); + Action action = new MiddlemanAction(owner, inputs, stampFile, purpose, middlemanType); + actionRegistry.registerAction(action); + return Pair.of(stampFile, action); + } + + /** + * Creates a normal middleman. + * + * <p>If called multiple times, it always returns the same object depending on the {@code + * purpose}. It does not check that the list of inputs is identical. In contrast to other + * middleman methods, this one also returns an object if the list of inputs is empty. + * + * <p>Note: there's no need to synchronize this method; the only use of a field is via a call to + * another synchronized method (getArtifact()). + */ + public Artifact createMiddlemanAllowMultiple(ActionRegistry registry, + ActionOwner owner, String purpose, Iterable<Artifact> inputs, Root middlemanDir) { + PathFragment stampName = new PathFragment("_middlemen/" + purpose); + Artifact stampFile = artifactFactory.getDerivedArtifact(stampName, middlemanDir, + actionRegistry.getOwner()); + MiddlemanAction.create( + registry, owner, inputs, stampFile, purpose, MiddlemanType.AGGREGATING_MIDDLEMAN); + return stampFile; + } + + private Artifact getStampFileArtifact(String middlemanName, String purpose, Root middlemanDir) { + String escapedFilename = Actions.escapedPath(middlemanName); + PathFragment stampName = new PathFragment("_middlemen/" + escapedFilename + "-" + purpose); + Artifact stampFile = artifactFactory.getDerivedArtifact(stampName, middlemanDir, + actionRegistry.getOwner()); + return stampFile; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/MissingInputFileException.java b/src/main/java/com/google/devtools/build/lib/actions/MissingInputFileException.java new file mode 100644 index 0000000..52f9f27 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/MissingInputFileException.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.events.Location; + +/** + * This exception is thrown during a build when an input file is missing, but the file + * is not the input to any action being executed. + * + * If a missing input file is an input + * to an action, an {@link ActionExecutionException} is thrown instead. + */ +public class MissingInputFileException extends BuildFailedException { + private final Location location; + + public MissingInputFileException(String message, Location location) { + super(message); + this.location = location; + } + + /** + * Return a location where this input file is referenced. If there + * are multiple such locations, one is chosen arbitrarily. If there + * are none, return null. + */ + public Location getLocation() { + return location; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/MutableActionGraph.java b/src/main/java/com/google/devtools/build/lib/actions/MutableActionGraph.java new file mode 100644 index 0000000..8b84c31 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/MutableActionGraph.java
@@ -0,0 +1,151 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.common.collect.Sets.SetView; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.StringUtil; + +import java.util.Set; + +/** + * A mutable action graph. Implementations of this interface must be thread-safe. + */ +public interface MutableActionGraph extends ActionGraph { + + /** + * Attempts to register the action. If any of the action's outputs already has a generating + * action, and the two actions are not compatible, then an {@link ActionConflictException} is + * thrown. The internal data structure may be partially modified when that happens; it is not + * guaranteed that all potential conflicts are detected, but at least one of them is. + * + * <p>For example, take three actions A, B, and C, where A creates outputs a and b, B creates just + * b, and C creates c and b. There are two potential conflicts in this case, between A and B, and + * between B and C. Depending on the ordering of calls to this method and the ordering of outputs + * in the action output lists, either one or two conflicts are detected: if B is registered first, + * then both conflicts are detected; if either A or C is registered first, then only one conflict + * is detected. + */ + void registerAction(Action action) throws ActionConflictException; + + /** + * Removes an action from this action graph if it is present. + * + * <p>Throws {@link IllegalStateException} if one of the outputs of the action is in fact + * generated by a different {@link Action} instance (even if they are sharable). + */ + void unregisterAction(Action action); + + /** + * Clear the action graph. + */ + void clear(); + + /** + * This exception is thrown when a conflict between actions is detected. It contains information + * about the artifact for which the conflict is found, and data about the two conflicting actions + * and their owners. + */ + public static final class ActionConflictException extends Exception { + + private final Artifact artifact; + private final Action previousAction; + private final Action attemptedAction; + + public ActionConflictException(Artifact artifact, Action previousAction, + Action attemptedAction) { + super("for " + artifact); + this.artifact = artifact; + this.previousAction = previousAction; + this.attemptedAction = attemptedAction; + } + + public Artifact getArtifact() { + return artifact; + } + + public void reportTo(EventHandler eventListener) { + String msg = "file '" + artifact.prettyPrint() + + "' is generated by these conflicting actions:\n" + + suffix(attemptedAction, previousAction); + eventListener.handle(Event.error(msg)); + } + + private void addStringDetail(StringBuilder sb, String key, String valueA, String valueB) { + valueA = valueA != null ? valueA : "(null)"; + valueB = valueB != null ? valueB : "(null)"; + + sb.append(key).append(": ").append(valueA); + if (!valueA.equals(valueB)) { + sb.append(", ").append(valueB); + } + sb.append("\n"); + } + + private void addListDetail(StringBuilder sb, String key, + Iterable<Artifact> valueA, Iterable<Artifact> valueB) { + Set<Artifact> setA = ImmutableSet.copyOf(valueA); + Set<Artifact> setB = ImmutableSet.copyOf(valueB); + SetView<Artifact> diffA = Sets.difference(setA, setB); + SetView<Artifact> diffB = Sets.difference(setB, setA); + + sb.append(key).append(": "); + if (diffA.isEmpty() && diffB.isEmpty()) { + sb.append("are equal"); + } else { + if (!diffA.isEmpty() && !diffB.isEmpty()) { + sb.append("attempted action contains artifacts not in previous action and " + + "previous action contains artifacts not in attempted action."); + } else if (!diffA.isEmpty()) { + sb.append("attempted action contains artifacts not in previous action: "); + sb.append(StringUtil.joinEnglishList(diffA, "and")); + } else if (!diffB.isEmpty()) { + sb.append("previous action contains artifacts not in attempted action: "); + sb.append(StringUtil.joinEnglishList(diffB, "and")); + } + } + sb.append("\n"); + } + + // See also Actions.canBeShared() + private String suffix(Action a, Action b) { + // Note: the error message reveals to users the names of intermediate files that are not + // documented in the BUILD language. This error-reporting logic is rather elaborate but it + // does help to diagnose some tricky situations. + StringBuilder sb = new StringBuilder(); + ActionOwner aOwner = a.getOwner(); + ActionOwner bOwner = b.getOwner(); + boolean aNull = aOwner == null; + boolean bNull = bOwner == null; + + addStringDetail(sb, "Label", aNull ? null : Label.print(aOwner.getLabel()), + bNull ? null : Label.print(bOwner.getLabel())); + addStringDetail(sb, "RuleClass", aNull ? null : aOwner.getTargetKind(), + bNull ? null : bOwner.getTargetKind()); + addStringDetail(sb, "Configuration", aNull ? null : aOwner.getConfigurationName(), + bNull ? null : bOwner.getConfigurationName()); + addStringDetail(sb, "Mnemonic", a.getMnemonic(), b.getMnemonic()); + + addListDetail(sb, "MandatoryInputs", a.getMandatoryInputs(), b.getMandatoryInputs()); + addListDetail(sb, "Outputs", a.getOutputs(), b.getOutputs()); + + return sb.toString(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/NotifyOnActionCacheHit.java b/src/main/java/com/google/devtools/build/lib/actions/NotifyOnActionCacheHit.java new file mode 100644 index 0000000..fa9b54e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/NotifyOnActionCacheHit.java
@@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * An action which must know when it is skipped due to an action cache hit. + * + * Use should be rare, as the action graph is a functional model. + */ +public interface NotifyOnActionCacheHit extends Action { + + /** + * Called when action has "cache hit", and therefore need not be executed. + * + * @param executor the executor + */ + void actionCacheHit(Executor executor); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/PackageRootResolver.java b/src/main/java/com/google/devtools/build/lib/actions/PackageRootResolver.java new file mode 100644 index 0000000..90af136 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/PackageRootResolver.java
@@ -0,0 +1,34 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Represents logic that evaluates the root of the package containing path. + */ +public interface PackageRootResolver { + + /** + * Returns mapping from execPath to Root. Some roots can equal null if the corresponding + * package can't be found. Returns null if for some reason we can't evaluate it. + */ + @Nullable + Map<PathFragment, Root> findPackageRoots(Iterable<PathFragment> execPaths); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ParameterFile.java b/src/main/java/com/google/devtools/build/lib/actions/ParameterFile.java new file mode 100644 index 0000000..80df9e2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ParameterFile.java
@@ -0,0 +1,114 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; + +/** + * Support for parameter file generation (as used by gcc and other tools, e.g. + * {@code gcc @param_file}. Note that the parameter file needs to be explicitly + * deleted after use. Different tools require different parameter file formats, + * which can be selected via the {@link ParameterFileType} enum. + * + * <p>The default charset is ISO-8859-1 (latin1). This also has to match the + * expectation of the tool. + * + * <p>Don't use this class for new code. Use the ParameterFileWriteAction + * instead! + */ +public class ParameterFile { + + /** + * Different styles of parameter files. + */ + public static enum ParameterFileType { + /** + * A parameter file with every parameter on a separate line. This format + * cannot handle newlines in parameters. It is currently used for most + * tools, but may not be interpreted correctly if parameters contain + * white space or other special characters. It should be avoided for new + * development. + */ + UNQUOTED, + + /** + * A parameter file where each parameter is correctly quoted for shell + * use, and separated by white space (space, tab, newline). This format is + * safe for all characters, but must be specially supported by the tool. In + * particular, it must not be used with gcc and related tools, which do not + * support this format as it is. + */ + SHELL_QUOTED; + } + + // Parameter file location. + private final Path execRoot; + private final PathFragment execPath; + private final Charset charset; + private final ParameterFileType type; + + @VisibleForTesting + public static final FileType PARAMETER_FILE = FileType.of(".params"); + + /** + * Creates a parameter file with the given parameters. + */ + public ParameterFile(Path execRoot, PathFragment execPath, Charset charset, + ParameterFileType type) { + Preconditions.checkNotNull(type); + this.execRoot = execRoot; + this.execPath = execPath; + this.charset = Preconditions.checkNotNull(charset); + this.type = Preconditions.checkNotNull(type); + } + + /** + * Derives an exec path from a given exec path by appending <code>".params"</code>. + */ + public static PathFragment derivePath(PathFragment original) { + return original.replaceName(original.getBaseName() + "-2.params"); + } + + /** + * Returns the path for the parameter file. + */ + public Path getPath() { + return execRoot.getRelative(execPath); + } + + /** + * Writes the arguments from the list into the parameter file according to + * the style selected in the constructor. + */ + public void writeContent(List<String> arguments) throws ExecException { + Iterable<String> actualArgs = (type == ParameterFileType.SHELL_QUOTED) ? + ShellEscaper.escapeAll(arguments) : arguments; + Path file = getPath(); + try { + FileSystemUtils.writeLinesAs(file, charset, actualArgs); + } catch (IOException e) { + throw new EnvironmentalExecException("could not write param file '" + file + "'", e); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java new file mode 100644 index 0000000..929a106 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java
@@ -0,0 +1,472 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.Pair; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.CountDownLatch; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * CPU/RAM resource manager. Used to keep track of resources consumed by the Blaze action execution + * threads and throttle them when necessary. + * + * <p>Threads which are known to consume a significant amount of the local CPU or RAM resources + * should call {@link #acquireResources} method. This method will check whether requested resources + * are available and will either mark them as used and allow thread to proceed or will block the + * thread until requested resources will become available. When thread completes it task, it must + * release allocated resources by calling {@link #releaseResources} method. + * + * <p>Available resources can be calculated using one of three ways: + * <ol> + * <li>They can be preset using {@link #setAvailableResources(ResourceSet)} method. This is used + * mainly by the unit tests (however it is possible to provide a future option that would + * artificially limit amount of CPU/RAM consumed by the Blaze). + * <li>They can be preset based on the /proc/cpuinfo and /proc/meminfo information. Blaze will + * calculate amount of available CPU cores (adjusting for hyperthreading logical cores) and + * amount of the total available memory and will limit itself to the number of effective cores + * and 2/3 of the available memory. For details, please look at the {@link + * LocalHostCapacity#getLocalHostCapacity} method. + * <li>Blaze will periodically (every 3 seconds) poll {@code /proc/meminfo} and {@code /proc/stat} + * information to obtain how much RAM and CPU resources are currently idle at that moment. For + * calculation details, please look at the {@link LocalHostCapacity#getFreeResources} + * implementation. + * </ol> + * + * <p>The resource manager also allows a slight overallocation of the resources to account for the + * fact that requested resources are usually estimated using a pessimistic approximation. It also + * guarantees that at least one thread will always be able to acquire any amount of requested + * resources (even if it is greater than amount of available resources). Therefore, assuming that + * threads correctly release acquired resources, Blaze will never be fully blocked. + */ +@ThreadSafe +public class ResourceManager { + + private static final Logger LOG = Logger.getLogger(ResourceManager.class.getName()); + private final boolean FINE; + + private EventBus eventBus; + + private final ThreadLocal<Boolean> threadLocked = new ThreadLocal<Boolean>() { + @Override + protected Boolean initialValue() { + return false; + } + }; + + /** + * Singleton reference defined in a separate class to ensure thread-safe lazy + * initialization. + */ + private static class Singleton { + static ResourceManager instance = new ResourceManager(); + } + + /** + * Returns singleton instance of the resource manager. + */ + public static ResourceManager instance() { + return Singleton.instance; + } + + // Allocated resources are allowed to go "negative", but at least + // MIN_AVAILABLE_CPU_RATIO portion of CPU and MIN_AVAILABLE_RAM_RATIO portion + // of RAM should be available. + // Please note that this value is purely empirical - we assume that generally + // requested resources are somewhat pessimistic and thread would end up + // using less than requested amount. + private final static double MIN_NECESSARY_CPU_RATIO = 0.6; + private final static double MIN_NECESSARY_RAM_RATIO = 1.0; + private final static double MIN_NECESSARY_IO_RATIO = 1.0; + + // List of blocked threads. Associated CountDownLatch object will always + // be initialized to 1 during creation in the acquire() method. + private final List<Pair<ResourceSet, CountDownLatch>> requestList; + + // The total amount of resources on the local host. Must be set by + // an explicit call to setAvailableResources(), often using + // LocalHostCapacity.getLocalHostCapacity() as an argument. + private ResourceSet staticResources = null; + + private ResourceSet availableResources = null; + private LocalHostCapacity.FreeResources freeReading = null; + + // Used amount of CPU capacity (where 1.0 corresponds to the one fully + // occupied CPU core. Corresponds to the CPU resource definition in the + // ResourceSet class. + private double usedCpu; + + // Used amount of RAM capacity in MB. Corresponds to the RAM resource + // definition in the ResourceSet class. + private double usedRam; + + // Used amount of I/O resources. Corresponds to the I/O resource + // definition in the ResourceSet class. + private double usedIo; + + // Specifies how much of the RAM in staticResources we should allow to be used. + public static final int DEFAULT_RAM_UTILIZATION_PERCENTAGE = 67; + private int ramUtilizationPercentage = DEFAULT_RAM_UTILIZATION_PERCENTAGE; + + // Timer responsible for the periodic polling of the current system load. + private Timer timer = null; + + private ResourceManager() { + FINE = LOG.isLoggable(Level.FINE); + requestList = new LinkedList<Pair<ResourceSet, CountDownLatch>>(); + } + + @VisibleForTesting public static ResourceManager instanceForTestingOnly() { + return new ResourceManager(); + } + + /** + * Resets resource manager state and releases all thread locks. + * Note - it does not reset auto-sensing or available resources. Use + * separate call to setAvailableResoures() or to setAutoSensing(). + */ + public synchronized void resetResourceUsage() { + usedCpu = 0; + usedRam = 0; + usedIo = 0; + for (Pair<ResourceSet, CountDownLatch> request : requestList) { + // CountDownLatch can be set only to 0 or 1. + request.second.countDown(); + } + requestList.clear(); + } + + /** + * Sets available resources using given resource set. Must be called + * at least once before using resource manager. + * <p> + * Method will also disable auto-sensing if it was enabled. + */ + public synchronized void setAvailableResources(ResourceSet resources) { + Preconditions.checkNotNull(resources); + staticResources = resources; + setAutoSensing(false); + } + + public synchronized boolean isAutoSensingEnabled() { + return timer != null; + } + + /** + * Specify how much of the available RAM we should allow to be used. + * This has no effect if autosensing is enabled. + */ + public synchronized void setRamUtilizationPercentage(int percentage) { + ramUtilizationPercentage = percentage; + } + + /** + * Enables or disables secondary resource allocation algorithm that will + * periodically (when needed but at most once per 3 seconds) checks real + * amount of available memory (based on /proc/meminfo) and current CPU load + * (based on 1 second difference of /proc/stat) and allows additional resource + * acquisition if previous requests were overly pessimistic. + */ + public synchronized void setAutoSensing(boolean enable) { + // Create new Timer instance only if it does not exist already. + if (enable && !isAutoSensingEnabled()) { + Profiler.instance().logEvent(ProfilerTask.INFO, "Enable auto sensing"); + if(refreshFreeResources()) { + timer = new Timer("AutoSenseTimer", true); + timer.schedule(new TimerTask() { + @Override public void run() { refreshFreeResources(); } + }, 3000, 3000); + } + } else if (!enable) { + if (isAutoSensingEnabled()) { + Profiler.instance().logEvent(ProfilerTask.INFO, "Disable auto sensing"); + timer.cancel(); + timer = null; + } + if (staticResources != null) { + updateAvailableResources(false); + } + } + } + + /** + * Acquires requested resource set. Will block if resource is not available. + * NB! This method must be thread-safe! + */ + public void acquireResources(ActionMetadata owner, ResourceSet resources) + throws InterruptedException { + Preconditions.checkArgument(resources != null); + long startTime = Profiler.nanoTimeMaybe(); + CountDownLatch latch = null; + try { + waiting(owner); + latch = acquire(resources); + if (latch != null) { + latch.await(); + } + } finally { + threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0 + || resources.getIoUsage() != 0); + acquired(owner); + + // Profile acquisition only if it waited for resource to become available. + if (latch != null) { + Profiler.instance().logSimpleTask(startTime, ProfilerTask.ACTION_LOCK, owner); + } + } + } + + /** + * Acquires the given resources if available immediately. Does not block. + * @return true iff the given resources were locked (all or nothing). + */ + public boolean tryAcquire(ActionMetadata owner, ResourceSet resources) { + boolean acquired = false; + synchronized (this) { + if (areResourcesAvailable(resources)) { + incrementResources(resources); + acquired = true; + } + } + + if (acquired) { + threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0); + acquired(owner); + } + + return acquired; + } + + private void incrementResources(ResourceSet resources) { + usedCpu += resources.getCpuUsage(); + usedRam += resources.getMemoryMb(); + usedIo += resources.getIoUsage(); + } + + /** + * Return true if any resources have been claimed through this manager. + */ + public synchronized boolean inUse() { + return usedCpu != 0.0 || usedRam != 0.0 || usedIo != 0.0 || requestList.size() > 0; + } + + + /** + * Return true iff this thread has a lock on non-zero resources. + */ + public boolean threadHasResources() { + return threadLocked.get(); + } + + public void setEventBus(EventBus eventBus) { + Preconditions.checkState(this.eventBus == null); + this.eventBus = Preconditions.checkNotNull(eventBus); + } + + public void unsetEventBus() { + Preconditions.checkState(this.eventBus != null); + this.eventBus = null; + } + + private void waiting(ActionMetadata owner) { + if (eventBus != null) { + // Null only in tests. + eventBus.post(ActionStatusMessage.schedulingStrategy(owner)); + } + } + + private void acquired(ActionMetadata owner) { + if (eventBus != null) { + // Null only in tests. + eventBus.post(ActionStatusMessage.runningStrategy(owner)); + } + } + + /** + * Releases previously requested resource set. + * + * <p>NB! This method must be thread-safe! + */ + public void releaseResources(ActionMetadata owner, ResourceSet resources) { + boolean isConflict = false; + long startTime = Profiler.nanoTimeMaybe(); + try { + isConflict = release(resources); + } finally { + threadLocked.set(false); + + // Profile resource release only if it resolved at least one allocation request. + if (isConflict) { + Profiler.instance().logSimpleTask(startTime, ProfilerTask.ACTION_RELEASE, owner); + } + } + } + + private synchronized CountDownLatch acquire(ResourceSet resources) { + if (areResourcesAvailable(resources)) { + incrementResources(resources); + return null; + } + Pair<ResourceSet, CountDownLatch> request = + new Pair<>(resources, new CountDownLatch(1)); + requestList.add(request); + + // If we use auto sensing and there has not been an update within last + // 30 seconds, something has gone really wrong - disable it. + if (isAutoSensingEnabled() && freeReading.getReadingAge() > 30000) { + LoggingUtil.logToRemote(Level.WARNING, "Free resource readings were " + + "not updated for 30 seconds - auto-sensing is disabled", + new IllegalStateException()); + LOG.warning("Free resource readings were not updated for 30 seconds - " + + "auto-sensing is disabled"); + setAutoSensing(false); + } + return request.second; + } + + private synchronized boolean release(ResourceSet resources) { + usedCpu -= resources.getCpuUsage(); + usedRam -= resources.getMemoryMb(); + usedIo -= resources.getIoUsage(); + + // TODO(bazel-team): (2010) rounding error can accumulate and value below can end up being + // e.g. 1E-15. So if it is small enough, we set it to 0. But maybe there is a better solution. + if (usedCpu < 0.0001) { + usedCpu = 0; + } + if (usedRam < 0.0001) { + usedRam = 0; + } + if (usedIo < 0.0001) { + usedIo = 0; + } + if (requestList.size() > 0) { + processWaitingThreads(); + return true; + } + return false; + } + + + /** + * Tries to unblock one or more waiting threads if there are sufficient resources available. + */ + private synchronized void processWaitingThreads() { + Iterator<Pair<ResourceSet, CountDownLatch>> iterator = requestList.iterator(); + while (iterator.hasNext()) { + Pair<ResourceSet, CountDownLatch> request = iterator.next(); + if (areResourcesAvailable(request.first)) { + incrementResources(request.first); + request.second.countDown(); + iterator.remove(); + } + } + } + + // Method will return true if all requested resources are considered to be available. + private boolean areResourcesAvailable(ResourceSet resources) { + Preconditions.checkNotNull(availableResources); + // Comparison below is robust, since any calculation errors will be fixed + // by the release() method. + if (usedCpu == 0.0 && usedRam == 0.0 && usedIo == 0.0) { + return true; + } + // Use only MIN_NECESSARY_???_RATIO of the resource value to check for + // allocation. This is necessary to account for the fact that most of the + // requested resource sets use pessimistic estimations. Note that this + // ratio is used only during comparison - for tracking we will actually + // mark whole requested amount as used. + double cpu = resources.getCpuUsage() * MIN_NECESSARY_CPU_RATIO; + double ram = resources.getMemoryMb() * MIN_NECESSARY_RAM_RATIO; + double io = resources.getIoUsage() * MIN_NECESSARY_IO_RATIO; + + double availableCpu = availableResources.getCpuUsage(); + double availableRam = availableResources.getMemoryMb(); + double availableIo = availableResources.getIoUsage(); + + // Resources are considered available if any one of the conditions below is true: + // 1) If resource is not requested at all, it is available. + // 2) If resource is not used at the moment, it is considered to be + // available regardless of how much is requested. This is necessary to + // ensure that at any given time, at least one thread is able to acquire + // resources even if it requests more than available. + // 3) If used resource amount is less than total available resource amount. + return (cpu == 0.0 || usedCpu == 0.0 || usedCpu + cpu <= availableCpu) && + (ram == 0.0 || usedRam == 0.0 || usedRam + ram <= availableRam) && + (io == 0.0 || usedIo == 0.0 || usedIo + io <= availableIo); + } + + private synchronized void updateAvailableResources(boolean useFreeReading) { + Preconditions.checkNotNull(staticResources); + if (useFreeReading && isAutoSensingEnabled()) { + availableResources = new ResourceSet( + usedRam + freeReading.getFreeMb(), + usedCpu + freeReading.getAvgFreeCpu(), + staticResources.getIoUsage()); + if(FINE) { + LOG.fine("Free resources: " + Math.round(freeReading.getFreeMb()) + " MB," + + Math.round(freeReading.getAvgFreeCpu() * 100) + "% CPU"); + } + processWaitingThreads(); + } else { + availableResources = new ResourceSet( + staticResources.getMemoryMb() * this.ramUtilizationPercentage / 100.0, + staticResources.getCpuUsage(), + staticResources.getIoUsage()); + processWaitingThreads(); + } + } + + /** + * Called by the timer thread to update system load information. + * + * @return true if update was successful and false if error was detected and + * autosensing was disabled. + */ + private boolean refreshFreeResources() { + freeReading = LocalHostCapacity.getFreeResources(freeReading); + if (freeReading == null) { // Unable to read or parse /proc/* information. + LOG.warning("Unable to obtain system load - autosensing is disabled"); + setAutoSensing(false); + return false; + } + updateAvailableResources( + freeReading.getInterval() >= 1000 && freeReading.getInterval() <= 10000); + return true; + } + + @VisibleForTesting + synchronized int getWaitCount() { + return requestList.size(); + } + + @VisibleForTesting + synchronized boolean isAvailable(double ram, double cpu, double io) { + return areResourcesAvailable(new ResourceSet(ram, cpu, io)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java new file mode 100644 index 0000000..e7ab98f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java
@@ -0,0 +1,113 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Splitter; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Instances of this class represent an estimate of the resource consumption + * for a particular Action, or the total available resources. We plan to + * use this to do smarter scheduling of actions, for example making sure + * that we don't schedule jobs concurrently if they would use so much + * memory as to cause the machine to thrash. + */ +@Immutable +public class ResourceSet { + + /** For actions that consume negligible resources. */ + public static final ResourceSet ZERO = new ResourceSet(0.0, 0.0, 0.0); + + /** The amount of real memory (resident set size). */ + private final double memoryMb; + + /** The number of CPUs, or fractions thereof. */ + private final double cpuUsage; + + /** + * Relative amount of used I/O resources (with 1.0 being total available amount on an "average" + * workstation. + */ + private final double ioUsage; + + public ResourceSet(double memoryMb, double cpuUsage, double ioUsage) { + this.memoryMb = memoryMb; + this.cpuUsage = cpuUsage; + this.ioUsage = ioUsage; + } + + /** Returns the amount of real memory (resident set size) used in MB. */ + public double getMemoryMb() { + return memoryMb; + } + + /** + * Returns the number of CPUs (or fractions thereof) used. + * For a CPU-bound single-threaded process, this will be 1.0. + * For a single-threaded process which spends part of its + * time waiting for I/O, this will be somewhere between 0.0 and 1.0. + * For a multi-threaded or multi-process application, + * this may be more than 1.0. + */ + public double getCpuUsage() { + return cpuUsage; + } + + /** + * Returns the amount of I/O used. + * Full amount of available I/O resources on the "average" workstation is + * considered to be 1.0. + */ + public double getIoUsage() { + return ioUsage; + } + + public static class ResourceSetConverter implements Converter<ResourceSet> { + private static final Splitter SPLITTER = Splitter.on(','); + + @Override + public ResourceSet convert(String input) throws OptionsParsingException { + Iterator<String> values = SPLITTER.split(input).iterator(); + try { + double memoryMb = Double.parseDouble(values.next()); + double cpuUsage = Double.parseDouble(values.next()); + double ioUsage = Double.parseDouble(values.next()); + if (values.hasNext()) { + throw new OptionsParsingException("Expected exactly 3 comma-separated float values"); + } + if (memoryMb <= 0.0 || cpuUsage <= 0.0 || ioUsage <= 0.0) { + throw new OptionsParsingException("All resource values must be positive"); + } + return new ResourceSet(memoryMb, cpuUsage, ioUsage); + } catch (NumberFormatException nfe) { + throw new OptionsParsingException("Expected exactly 3 comma-separated float values", nfe); + } catch (NoSuchElementException nsee) { + throw new OptionsParsingException("Expected exactly 3 comma-separated float values", nsee); + } + } + + @Override + public String getTypeDescription() { + return "comma-separated available amount of RAM (in MB), CPU (in cores) and " + + "available I/O (1.0 being average workstation)"; + } + + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/Root.java b/src/main/java/com/google/devtools/build/lib/actions/Root.java new file mode 100644 index 0000000..284b85f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/Root.java
@@ -0,0 +1,163 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.Serializable; +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * A root for an artifact. The roots are the directories containing artifacts, and they are mapped + * together into a single directory tree to form the execution environment. There are two kinds of + * roots, source roots and derived roots. Source roots correspond to entries of the package path, + * and they can be anywhere on disk. Derived roots correspond to output directories; there are + * generally different output directories for different configurations, and different types of + * output (bin, genfiles, includes, etc.). + * + * <p>When mapping the roots into a single directory tree, the source roots are merged, such that + * each package is accessed in its entirety from a single source root. The package cache is + * responsible for determining that mapping. The derived roots, on the other hand, have to be + * distinct. (It is currently allowed to have a derived root that is the prefix of another one.) + * + * <p>The derived roots must have paths that point inside the exec root, i.e. below the directory + * that is the root of the merged directory tree. + */ +@SkylarkModule(name = "root", + doc = "A root for files. The roots are the directories containing files, and they are mapped " + + "together into a single directory tree to form the execution environment.") +public final class Root implements Comparable<Root>, Serializable { + + /** + * Returns the given path as a source root. The path may not be {@code null}. + */ + public static Root asSourceRoot(Path path) { + return new Root(null, path); + } + + /** + * DO NOT USE IN PRODUCTION CODE! + * + * <p>Returns the given path as a derived root. This method only exists as a convenience for + * tests, which don't need a proper Root object. + */ + @VisibleForTesting + public static Root asDerivedRoot(Path path) { + return new Root(path, path); + } + + /** + * Returns the given path as a derived root, relative to the given exec root. The root must be a + * proper sub-directory of the exec root (i.e. not equal). Neither may be {@code null}. + * + * <p>Be careful with this method - all derived roots must be registered with the artifact factory + * before the analysis phase. + */ + public static Root asDerivedRoot(Path execRoot, Path root) { + Preconditions.checkArgument(root.startsWith(execRoot)); + Preconditions.checkArgument(!root.equals(execRoot)); + return new Root(execRoot, root); + } + + public static Root middlemanRoot(Path execRoot, Path outputDir) { + Path root = outputDir.getRelative("internal"); + Preconditions.checkArgument(root.startsWith(execRoot)); + Preconditions.checkArgument(!root.equals(execRoot)); + return new Root(execRoot, root, true); + } + + /** + * Returns the exec root as a derived root. The exec root should never be treated as a derived + * root, but this is currently allowed. Do not add any further uses besides the ones that already + * exist! + */ + static Root execRootAsDerivedRoot(Path execRoot) { + return new Root(execRoot, execRoot); + } + + @Nullable private final Path execRoot; + private final Path path; + private final boolean isMiddlemanRoot; + + private Root(@Nullable Path execRoot, Path path, boolean isMiddlemanRoot) { + this.execRoot = execRoot; + this.path = Preconditions.checkNotNull(path); + this.isMiddlemanRoot = isMiddlemanRoot; + } + + private Root(@Nullable Path execRoot, Path path) { + this(execRoot, path, false); + } + + public Path getPath() { + return path; + } + + /** + * Returns the path fragment from the exec root to the actual root. For source roots, this returns + * the empty fragment. + */ + public PathFragment getExecPath() { + return isSourceRoot() ? PathFragment.EMPTY_FRAGMENT : path.relativeTo(execRoot); + } + + @SkylarkCallable(name = "path", structField = true, + doc = "Returns the relative path from the exec root to the actual root.") + public String getExecPathString() { + return getExecPath().getPathString(); + } + + public boolean isSourceRoot() { + return execRoot == null; + } + + public boolean isMiddlemanRoot() { + return isMiddlemanRoot; + } + + @Override + public int compareTo(Root o) { + return path.compareTo(o.path); + } + + @Override + public int hashCode() { + return Objects.hash(execRoot, path.hashCode()); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof Root)) { + return false; + } + Root r = (Root) o; + return path.equals(r.path) && Objects.equals(execRoot, r.execRoot); + } + + @Override + public String toString() { + return path.toString() + (isSourceRoot() ? "[source]" : "[derived]"); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/Spawn.java b/src/main/java/com/google/devtools/build/lib/actions/Spawn.java new file mode 100644 index 0000000..2f905d0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/Spawn.java
@@ -0,0 +1,122 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.extra.SpawnInfo; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; + +/** + * An object representing a subprocess to be invoked, including its command and + * arguments, its working directory, its environment, a boolean indicating + * whether remote execution is appropriate for this command, and if so, the set + * of files it is expected to read and write. + */ +public interface Spawn { + + /** + * Returns true iff this command may be executed remotely. + */ + boolean isRemotable(); + + /** + * Out-of-band data for this spawn. This can be used to signal hints (hardware requirements, + * local vs. remote) to the execution subsystem. + * + * <p>String tags from {@link + * com.google.devtools.build.lib.rules.test.TestTargetProperties#getExecutionInfo()} can be added + * as keys with arbitrary values to this map too. + */ + ImmutableMap<String, String> getExecutionInfo(); + + /** + * Returns this Spawn as a Bourne shell command. + * + * @param workingDir the initial working directory of the command + */ + String asShellCommand(Path workingDir); + + /** + * Returns the runfiles data for remote execution. Format is (directory, manifest file). + */ + ImmutableMap<PathFragment, Artifact> getRunfilesManifests(); + + /** + * Returns artifacts for filesets, so they can be scheduled on remote execution. + */ + ImmutableList<Artifact> getFilesetManifests(); + + /** + * Returns a protocol buffer describing this spawn for use by the extra_action functionality. + */ + SpawnInfo getExtraActionInfo(); + + /** + * Returns the command (the first element) and its arguments. + */ + ImmutableList<String> getArguments(); + + /** + * Returns the initial environment of the process. + * If null, the environment is inherited from the parent process. + */ + ImmutableMap<String, String> getEnvironment(); + + /** + * Returns the list of files that this command may read. + * + * <p>This method explicitly does not expand middleman artifacts. Pass the result + * to an appropriate utility method on {@link com.google.devtools.build.lib.actions.Artifact} to + * expand the middlemen. + * + * <p>This is for use with remote execution, so we can ship inputs before starting the + * command. Order stability across multiple calls should be upheld for performance reasons. + */ + Iterable<? extends ActionInput> getInputFiles(); + + /** + * Returns the collection of files that this command must write. Callers should not mutate + * the result. + * + * <p>This is for use with remote execution, so remote execution does not have to guess what + * outputs the process writes. While the order does not affect the semantics, it should be + * stable so it can be cached. + */ + Collection<? extends ActionInput> getOutputFiles(); + + /** + * Returns the resource owner for local fallback. + */ + ActionMetadata getResourceOwner(); + + /** + * Returns the amount of resources needed for local fallback. + */ + ResourceSet getLocalResources(); + + /** + * Returns the owner for this action. Production code should supply a non-null owner. + */ + ActionOwner getOwner(); + + /** + * Returns a mnemonic (string constant) for this kind of spawn. + */ + String getMnemonic(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/SpawnActionContext.java b/src/main/java/com/google/devtools/build/lib/actions/SpawnActionContext.java new file mode 100644 index 0000000..c2ea1b0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/SpawnActionContext.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + + +/** + * A context that allows execution of {@link Spawn} instances. + */ +@ActionContextMarker(name = "spawn") +public interface SpawnActionContext extends Executor.ActionContext { + + /** + * Executes the given spawn. + */ + void exec(Spawn spawn, ActionExecutionContext actionExecutionContext) + throws ExecException, InterruptedException; + + /** Returns the locality of running the spawn, i.e., "local". */ + String strategyLocality(String mnemonic, boolean remotable); + + /** + * This implements a tri-state mode. There are three possible cases: (1) implementations of this + * class can unconditionally execute spawns locally, (2) they can follow whatever is set for the + * corresponding spawn (see {@link Spawn#isRemotable}), or (3) they can unconditionally execute + * spawns remotely, i.e., force remote execution. + * + * <p>Passing the spawns remotable flag to this method returns whether the spawn will actually be + * executed remotely. + */ + boolean isRemotable(String mnemonic, boolean remotable); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/TargetOutOfDateException.java b/src/main/java/com/google/devtools/build/lib/actions/TargetOutOfDateException.java new file mode 100644 index 0000000..9092ab2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/TargetOutOfDateException.java
@@ -0,0 +1,25 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * An exception indicating that a target is out of date. + */ +public class TargetOutOfDateException extends ActionExecutionException { + + public TargetOutOfDateException(Action action) { + super (action.prettyPrint() + " is not up-to-date", action, false); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/TestExecException.java b/src/main/java/com/google/devtools/build/lib/actions/TestExecException.java new file mode 100644 index 0000000..c861338 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/TestExecException.java
@@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * An TestExecException that is related to the failure of a TestAction. + */ +public final class TestExecException extends ExecException { + + public TestExecException(String message) { + super(message); + } + + @Override + public ActionExecutionException toActionExecutionException(String messagePrefix, + boolean verboseFailures, Action action) { + String message = messagePrefix + " failed" + getMessage(); + if (verboseFailures) { + return new ActionExecutionException(message, this, action, isCatastrophic()); + } else { + return new ActionExecutionException(message, action, isCatastrophic()); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/TestMiddlemanObserver.java b/src/main/java/com/google/devtools/build/lib/actions/TestMiddlemanObserver.java new file mode 100644 index 0000000..128ac20 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/TestMiddlemanObserver.java
@@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * Used as a notification mechanism for the scheduling middleman mutations + * related to scheduling exclusive tests. + */ +public interface TestMiddlemanObserver { + + /** + * Called when the test removes the stale middleman. + * + * @param action the test action. + * @param middleman the scheduling middleman. + * @param middlemanAction the action generating the scheduling middleman + */ + void remove(Action action, Artifact middleman, Action middlemanAction); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/UserExecException.java b/src/main/java/com/google/devtools/build/lib/actions/UserExecException.java new file mode 100644 index 0000000..86a6eb0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/UserExecException.java
@@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * An ExecException that is related to the failure of an Action and therefore + * very likely the user's fault. + */ +public class UserExecException extends ExecException { + + public UserExecException(String message) { + super(message); + } + + public UserExecException(String message, Throwable cause) { + super(message, cause); + } + + @Override + public ActionExecutionException toActionExecutionException(String messagePrefix, + boolean verboseFailures, Action action) { + String message = messagePrefix + " failed: " + getMessage(); + if (verboseFailures) { + return new ActionExecutionException(message, this, action, isCatastrophic()); + } else { + return new ActionExecutionException(message, action, isCatastrophic()); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java new file mode 100644 index 0000000..2ac0ab8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java
@@ -0,0 +1,173 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions.cache; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * An interface defining a cache of already-executed Actions. + * + * <p>This class' naming is misleading; it doesn't cache the actual actions, but it stores a + * fingerprint of the action state (ie. a hash of the input and output files on disk), so + * we can tell if we need to rerun an action given the state of the file system. + * + * <p>Each action entry uses one of its output paths as a key (after conversion + * to the string). + */ +@ThreadCompatible +public interface ActionCache { + + /** + * Updates the cache entry for the specified key. + */ + void put(String key, ActionCache.Entry entry); + + /** + * Returns the corresponding cache entry for the specified key, if any, or + * null if not found. + */ + ActionCache.Entry get(String key); + + /** + * Removes entry from cache + */ + void remove(String key); + + /** + * Returns a new Entry instance. This method allows ActionCache subclasses to + * define their own Entry implementation. + */ + ActionCache.Entry createEntry(String key); + + /** + * An entry in the ActionCache that contains all action input and output + * artifact paths and their metadata plus action key itself. + * + * Cache entry operates under assumption that once it is fully initialized + * and getFileDigest() method is called, it becomes logically immutable (all methods + * will continue to return same result regardless of internal data transformations). + */ + public final class Entry { + private final String actionKey; + private final List<String> files; + // If null, digest is non-null and the entry is immutable. + private Map<String, Metadata> mdMap; + private Digest digest; + + public Entry(String key) { + actionKey = key; + files = new ArrayList<>(); + mdMap = new HashMap<>(); + } + + public Entry(String key, List<String> files, Digest digest) { + actionKey = key; + this.files = files; + this.digest = digest; + mdMap = null; + } + + /** + * Adds the artifact, specified by the executable relative path and its + * metadata into the cache entry. + */ + public void addFile(PathFragment relativePath, Metadata md) { + Preconditions.checkState(mdMap != null); + Preconditions.checkState(!isCorrupted()); + Preconditions.checkState(digest == null); + + String execPath = relativePath.getPathString(); + files.add(execPath); + mdMap.put(execPath, md); + } + + /** + * @return action key string. + */ + public String getActionKey() { + return actionKey; + } + + /** + * Returns the combined digest of the action's inputs and outputs. + * + * This may compresses the data into a more compact representation, and + * makes the object immutable. + */ + public Digest getFileDigest() { + if (digest == null) { + digest = Digest.fromMetadata(mdMap); + mdMap = null; + } + return digest; + } + + /** + * Returns true if this cache entry is corrupted and should be ignored. + */ + public boolean isCorrupted() { + return actionKey == null; + } + + /** + * @return stored path strings. + */ + public Collection<String> getPaths() { + return files; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append(" actionKey = ").append(actionKey).append("\n"); + builder.append(" digestKey = "); + if (digest == null) { + builder.append(Digest.fromMetadata(mdMap)).append(" (from mdMap)\n"); + } else { + builder.append(digest).append("\n"); + } + List<String> fileInfo = Lists.newArrayListWithCapacity(files.size()); + fileInfo.addAll(files); + Collections.sort(fileInfo); + for (String info : fileInfo) { + builder.append(" ").append(info).append("\n"); + } + return builder.toString(); + } + } + + /** + * Give persistent cache implementations a notification to write to disk. + * @return size in bytes of the serialized cache. + */ + long save() throws IOException; + + /** + * Dumps action cache content into the given PrintStream. + */ + void dump(PrintStream out); +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java new file mode 100644 index 0000000..24eb42e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java
@@ -0,0 +1,389 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.CompactStringIndexer; +import com.google.devtools.build.lib.util.PersistentMap; +import com.google.devtools.build.lib.util.StringIndexer; +import com.google.devtools.build.lib.util.VarInt; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.UnixGlob; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * An implementation of the ActionCache interface that uses + * {@link CompactStringIndexer} to reduce memory footprint and saves + * cached actions using the {@link PersistentMap}. + * + * <p>This cache is not fully correct: as hashes are xor'd together, a permutation of input + * file contents will erroneously be considered up to date. + */ +@ConditionallyThreadSafe // condition: each instance must instantiated with + // different cache root +public class CompactPersistentActionCache implements ActionCache { + private static final int SAVE_INTERVAL_SECONDS = 3; + private static final long NANOS_PER_SECOND = 1000 * 1000 * 1000; + + // Key of the action cache record that holds information used to verify referential integrity + // between action cache and string indexer. Must be < 0 to avoid conflict with real action + // cache records. + private static final int VALIDATION_KEY = -10; + + private static final int VERSION = 10; + + private final class ActionMap extends PersistentMap<Integer, byte[]> { + private final Clock clock; + private long nextUpdate; + + public ActionMap(Map<Integer, byte[]> map, Clock clock, Path mapFile, Path journalFile) + throws IOException { + super(VERSION, map, mapFile, journalFile); + this.clock = clock; + // Using nanoTime. currentTimeMillis may not provide enough granularity. + nextUpdate = clock.nanoTime() / NANOS_PER_SECOND + SAVE_INTERVAL_SECONDS; + load(); + } + + @Override + protected boolean updateJournal() { + // Using nanoTime. currentTimeMillis may not provide enough granularity. + long time = clock.nanoTime() / NANOS_PER_SECOND; + if (SAVE_INTERVAL_SECONDS == 0 || time > nextUpdate) { + nextUpdate = time + SAVE_INTERVAL_SECONDS; + // Force flushing of the PersistentStringIndexer instance. This is needed to ensure + // that filename index data on disk is always up-to-date when we save action cache + // data. + indexer.flush(); + return true; + } + return false; + } + + @Override + protected boolean keepJournal() { + // We must first flush the journal to get an accurate measure of its size. + forceFlush(); + try { + return journalSize() * 100 < cacheSize(); + } catch (IOException e) { + return false; + } + } + + @Override + protected Integer readKey(DataInputStream in) throws IOException { + return in.readInt(); + } + + @Override + protected byte[] readValue(DataInputStream in) + throws IOException { + int size = in.readInt(); + if (size < 0) { + throw new IOException("found negative array size: " + size); + } + byte[] data = new byte[size]; + in.readFully(data); + return data; + } + + @Override + protected void writeKey(Integer key, DataOutputStream out) + throws IOException { + out.writeInt(key); + } + + @Override + // TODO(bazel-team): (2010) This method, writeKey() and related Metadata methods + // should really use protocol messages. Doing so would allow easy inspection + // of the action cache content and, more importantly, would cut down on the + // need to change VERSION to different number every time we touch those + // methods. Especially when we'll start to add stuff like statistics for + // each action. + protected void writeValue(byte[] value, DataOutputStream out) + throws IOException { + out.writeInt(value.length); + out.write(value); + } + } + + private final PersistentMap<Integer, byte[]> map; + private final PersistentStringIndexer indexer; + static final ActionCache.Entry CORRUPTED = new ActionCache.Entry(null); + + public CompactPersistentActionCache(Path cacheRoot, Clock clock) throws IOException { + Path cacheFile = cacheFile(cacheRoot); + Path journalFile = journalFile(cacheRoot); + Path indexFile = cacheRoot.getChild("filename_index_v" + VERSION + ".blaze"); + // we can now use normal hash map as backing map, since dependency checker + // will manually purge records from the action cache. + Map<Integer, byte[]> backingMap = new HashMap<>(); + + try { + indexer = PersistentStringIndexer.newPersistentStringIndexer(indexFile, clock); + } catch (IOException e) { + renameCorruptedFiles(cacheRoot); + throw new IOException("Failed to load filename index data", e); + } + + try { + map = new ActionMap(backingMap, clock, cacheFile, journalFile); + } catch (IOException e) { + renameCorruptedFiles(cacheRoot); + throw new IOException("Failed to load action cache data", e); + } + + // Validate referential integrity between two collections. + if (!map.isEmpty()) { + String integrityError = validateIntegrity(indexer.size(), map.get(VALIDATION_KEY)); + if (integrityError != null) { + renameCorruptedFiles(cacheRoot); + throw new IOException("Failed action cache referential integrity check: " + integrityError); + } + } + } + + /** + * Rename corrupted files so they could be analyzed later. This would also ensure + * that next initialization attempt will create empty cache. + */ + private static void renameCorruptedFiles(Path cacheRoot) { + try { + for (Path path : UnixGlob.forPath(cacheRoot).addPattern("action_*_v" + VERSION + ".*") + .glob()) { + path.renameTo(path.getParentDirectory().getChild(path.getBaseName() + ".bad")); + } + for (Path path : UnixGlob.forPath(cacheRoot).addPattern("filename_*_v" + VERSION + ".*") + .glob()) { + path.renameTo(path.getParentDirectory().getChild(path.getBaseName() + ".bad")); + } + } catch (IOException e) { + // do nothing + } + } + + /** + * @return false iff indexer contains no data or integrity check has failed. + */ + private static String validateIntegrity(int indexerSize, byte[] validationRecord) { + if (indexerSize == 0) { + return "empty index"; + } + if (validationRecord == null) { + return "no validation record"; + } + try { + int validationSize = ByteBuffer.wrap(validationRecord).asIntBuffer().get(); + if (validationSize <= indexerSize) { + return null; + } else { + return String.format("Validation mismatch: validation entry %d is too large " + + "compared to index size %d", validationSize, indexerSize); + } + } catch (BufferUnderflowException e) { + return e.getMessage(); + } + + } + + public static Path cacheFile(Path cacheRoot) { + return cacheRoot.getChild("action_cache_v" + VERSION + ".blaze"); + } + + public static Path journalFile(Path cacheRoot) { + return cacheRoot.getChild("action_journal_v" + VERSION + ".blaze"); + } + + @Override + public ActionCache.Entry createEntry(String key) { + return new ActionCache.Entry(key); + } + + @Override + public ActionCache.Entry get(String key) { + int index = indexer.getIndex(key); + if (index < 0) { + return null; + } + byte[] data; + synchronized (this) { + data = map.get(index); + } + try { + return data != null ? CompactPersistentActionCache.decode(indexer, data) : null; + } catch (IOException e) { + // return entry marked as corrupted. + return CORRUPTED; + } + } + + @Override + public void put(String key, ActionCache.Entry entry) { + // Encode record. Note that both methods may create new mappings in the indexer. + int index = indexer.getOrCreateIndex(key); + byte[] content = encode(indexer, entry); + + // Update validation record. + ByteBuffer buffer = ByteBuffer.allocate(4); // size of int in bytes + int indexSize = indexer.size(); + buffer.asIntBuffer().put(indexSize); + + // Note the benign race condition here in which two threads might race on + // updating the VALIDATION_KEY. If the most recent update loses the race, + // a value lower than the indexer size will remain in the validation record. + // This will still pass the integrity check. + synchronized (this) { + map.put(VALIDATION_KEY, buffer.array()); + // Now update record itself. + map.put(index, content); + } + } + + @Override + public synchronized void remove(String key) { + map.remove(indexer.getIndex(key)); + } + + @Override + public synchronized long save() throws IOException { + long indexSize = indexer.save(); + long mapSize = map.save(); + return indexSize + mapSize; + } + + @Override + public synchronized String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Action cache (" + map.size() + " records):\n"); + for (Map.Entry<Integer, byte[]> entry: map.entrySet()) { + if (entry.getKey() == VALIDATION_KEY) { continue; } + String content; + try { + content = decode(indexer, entry.getValue()).toString(); + } catch (IOException e) { + content = e.toString() + "\n"; + } + builder.append("-> ").append(indexer.getStringForIndex(entry.getKey())).append("\n") + .append(content).append(" packed_len = ").append(entry.getValue().length).append("\n"); + } + return builder.toString(); + } + + /** + * Dumps action cache content. + */ + @Override + public synchronized void dump(PrintStream out) { + out.println("String indexer content:\n"); + out.println(indexer.toString()); + out.println("Action cache (" + map.size() + " records):\n"); + for (Map.Entry<Integer, byte[]> entry: map.entrySet()) { + if (entry.getKey() == VALIDATION_KEY) { continue; } + String content; + try { + content = CompactPersistentActionCache.decode(indexer, entry.getValue()).toString(); + } catch (IOException e) { + content = e.toString() + "\n"; + } + out.println(entry.getKey() + ", " + indexer.getStringForIndex(entry.getKey()) + ":\n" + + content + "\n packed_len = " + entry.getValue().length + "\n"); + } + } + + /** + * @return action data encoded as a byte[] array. + */ + private static byte[] encode(StringIndexer indexer, ActionCache.Entry entry) { + Preconditions.checkState(!entry.isCorrupted()); + + try { + byte[] actionKeyBytes = entry.getActionKey().getBytes(ISO_8859_1); + Collection<String> files = entry.getPaths(); + + // Estimate the size of the buffer: + // 5 bytes max for the actionKey length + // + the actionKey itself + // + 16 bytes for the digest + // + 5 bytes max for the file list length + // + 5 bytes max for each file id + int maxSize = VarInt.MAX_VARINT_SIZE + actionKeyBytes.length + Digest.MD5_SIZE + + VarInt.MAX_VARINT_SIZE + files.size() * VarInt.MAX_VARINT_SIZE; + ByteArrayOutputStream sink = new ByteArrayOutputStream(maxSize); + + VarInt.putVarInt(actionKeyBytes.length, sink); + sink.write(actionKeyBytes); + + entry.getFileDigest().write(sink); + + VarInt.putVarInt(files.size(), sink); + for (String file : files) { + VarInt.putVarInt(indexer.getOrCreateIndex(file), sink); + } + return sink.toByteArray(); + } catch (IOException e) { + // This Exception can never be thrown by ByteArrayOutputStream. + throw new AssertionError(e); + } + } + + /** + * Creates new action cache entry using given compressed entry data. Data + * will stay in the compressed format until entry is actually used by the + * dependency checker. + */ + private static ActionCache.Entry decode(StringIndexer indexer, byte[] data) throws IOException { + try { + ByteBuffer source = ByteBuffer.wrap(data); + + byte[] actionKeyBytes = new byte[VarInt.getVarInt(source)]; + source.get(actionKeyBytes); + String actionKey = new String(actionKeyBytes, ISO_8859_1); + + Digest digest = Digest.read(source); + + int count = VarInt.getVarInt(source); + ImmutableList.Builder<String> builder = new ImmutableList.Builder<>(); + for (int i = 0; i < count; i++) { + int id = VarInt.getVarInt(source); + String filename = (id >= 0 ? indexer.getStringForIndex(id) : null); + if (filename == null) { + throw new IOException("Corrupted file index"); + } + builder.add(filename); + } + if (source.remaining() > 0) { + throw new IOException("serialized entry data has not been fully decoded"); + } + return new Entry(actionKey, builder.build(), digest); + } catch (BufferUnderflowException e) { + throw new IOException("encoded entry data is incomplete", e); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/Digest.java b/src/main/java/com/google/devtools/build/lib/actions/cache/Digest.java new file mode 100644 index 0000000..f278507 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/Digest.java
@@ -0,0 +1,142 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.VarInt; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Map; + +/** + * A value class for capturing and comparing MD5-based digests. + * + * <p>Note that this class is responsible for digesting file metadata in an + * order-independent manner. Care must be taken to do this properly. The + * digest must be a function of the set of (path, metadata) tuples. While the + * order of these pairs must not matter, it would <b>not</b> be safe to make + * the digest be a function of the set of paths and the set of metadata. + * + * <p>Note that the (path, metadata) tuples must be unique, otherwise the + * XOR-based approach will fail. + */ +public class Digest { + + static final int MD5_SIZE = 16; + + private final byte[] digest; + + /** + * Construct the digest from the given bytes. + * @param digest an MD5 digest. Must be sized properly. + */ + @VisibleForTesting + Digest(byte[] digest) { + Preconditions.checkState(digest.length == MD5_SIZE); + this.digest = Arrays.copyOf(digest, digest.length); + } + + /** + * @param source the byte buffer source. + * @return the digest from the given buffer. + * @throws IOException if the byte buffer is incorrectly formatted. + */ + public static Digest read(ByteBuffer source) throws IOException { + int size = VarInt.getVarInt(source); + if (size != MD5_SIZE) { + throw new IOException("Unexpected digest length: " + size); + } + byte[] bytes = new byte[size]; + source.get(bytes); + return new Digest(bytes); + } + + /** + * Write the digest to the output stream. + */ + public void write(OutputStream sink) throws IOException { + VarInt.putVarInt(digest.length, sink); + sink.write(digest); + } + + /** + * @param mdMap A collection of (execPath, Metadata) pairs. + * Values may be null. + * @return an <b>order-independent</b> digest from the given "set" of + * (path, metadata) pairs. + */ + public static Digest fromMetadata(Map<String, Metadata> mdMap) { + byte[] result = new byte[MD5_SIZE]; + // Profiling showed that MD5 engine instantiation was a hotspot, so create one instance for + // this computation to amortize its cost. + Fingerprint fp = new Fingerprint(); + for (Map.Entry<String, Metadata> entry : mdMap.entrySet()) { + xorWith(result, getDigest(fp, entry.getKey(), entry.getValue())); + fp.reset(); + } + return new Digest(result); + } + + /** + * @return this Digest as a Metadata with no mtime. + */ + public Metadata asMetadata() { + return new Metadata(digest); + } + + @Override + public int hashCode() { + return Arrays.hashCode(digest); + } + + @Override + public boolean equals(Object obj) { + return (obj instanceof Digest) && Arrays.equals(digest, ((Digest) obj).digest); + } + + @Override + public String toString() { + return Fingerprint.hexDigest(digest); + } + + private static byte[] getDigest(Fingerprint fp, String execPath, Metadata md) { + fp.addString(execPath); + + if (md == null) { + // Move along, nothing to see here. + } else if (md.digest == null) { + // Use the timestamp if the digest is not present, but not both. + // Modifying a timestamp while keeping the contents of a file the + // same should not cause rebuilds. + fp.addLong(md.mtime); + } else { + fp.addBytes(md.digest); + } + return fp.digestAndReset(); + } + + /** + * Compute lhs ^= rhs bitwise operation of the arrays. + */ + private static void xorWith(byte[] lhs, byte[] rhs) { + for (int i = 0; i < lhs.length; i++) { + lhs[i] ^= rhs[i]; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java b/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java new file mode 100644 index 0000000..7295fb5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java
@@ -0,0 +1,130 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import com.google.common.base.Preconditions; +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; +import java.util.Objects; +import java.util.logging.Level; + +import javax.annotation.Nullable; + +/** + * Utility class for getting md5 digests of files. + */ +public class DigestUtils { + // Object to synchronize on when serializing large file reads. + private static final Object MD5_LOCK = new Object(); + + /** Private constructor to prevent instantiation of utility class. */ + private DigestUtils() {} + + /** + * Returns true iff using MD5 digests is appropriate for an artifact. + * + * @param artifact Artifact in question. + * @param isFile whether or not Artifact is a file versus a directory, isFile() on its stat. + * @param size size of Artifact on filesystem in bytes, getSize() on its stat. + */ + public static boolean useFileDigest(Artifact artifact, boolean isFile, long size) { + // Use timestamps for directories. Use digests for everything else. + return isFile && size != 0; + } + + /** + * Obtain file's MD5 metadata using synchronized method, ensuring that system + * is not overloaded in case when multiple threads are requesting MD5 + * calculations and underlying file system cannot provide it via extended + * attribute. + */ + private static byte[] getDigestInExclusiveMode(Path path) throws IOException { + long startTime = BlazeClock.nanoTime(); + synchronized (MD5_LOCK) { + Profiler.instance().logSimpleTask(startTime, ProfilerTask.WAIT, path.getPathString()); + return getDigestInternal(path); + } + } + + private static byte[] getDigestInternal(Path path) throws IOException { + long startTime = BlazeClock.nanoTime(); + byte[] md5bin = path.getMD5Digest(); + + long millis = (BlazeClock.nanoTime() - startTime) / 1000000; + if (millis > 5000L) { + System.err.println("Slow read: a " + path.getFileSize() + "-byte read from " + path + + " took " + millis + "ms."); + } + return md5bin; + } + + private static boolean binaryDigestWellFormed(byte[] digest) { + Preconditions.checkNotNull(digest); + return digest.length == 16; + } + + /** + * Returns the the fast md5 digest of the file, or null if not available. + */ + @Nullable + public static byte[] getFastDigest(Path path) throws IOException { + return path.getFastDigestFunctionType().equals("MD5") ? path.getFastDigest() : null; + } + + /** + * Get the md5 digest of {@code path}, using a constant-time xattr call if the filesystem supports + * it, and calculating the digest manually otherwise. + * + * @param path Path of the file. + * @param fileSize size of the file. Used to determine if digest calculation should be done + * serially or in parallel. Files larger than a certain threshold will be read serially, in order + * to avoid excessive disk seeks. + */ + public static byte[] getDigestOrFail(Path path, long fileSize) throws IOException { + // TODO(bazel-team): the action cache currently only works with md5 digests but it ought to + // work with any opaque digest. + byte[] md5bin = null; + if (Objects.equals(path.getFastDigestFunctionType(), "MD5")) { + md5bin = getFastDigest(path); + } + if (md5bin != null && !binaryDigestWellFormed(md5bin)) { + // Fail-soft in cases where md5bin is non-null, but not a valid digest. + String msg = String.format("Malformed digest '%s' for file %s", + BaseEncoding.base16().lowerCase().encode(md5bin), + path); + LoggingUtil.logToRemote(Level.SEVERE, msg, new IllegalStateException(msg)); + md5bin = null; + } + if (md5bin != null) { + return md5bin; + } else if (fileSize > 4096) { + // We'll have to read file content in order to calculate the digest. In that case + // it would be beneficial to serialize those calculations since there is a high + // probability that MD5 will be requested for multiple output files simultaneously. + // Exception is made for small (<=4K) files since they will not likely to introduce + // significant delays (at worst they will result in two extra disk seeks by + // interrupting other reads). + return getDigestInExclusiveMode(path); + } else { + return getDigestInternal(path); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/InjectedStat.java b/src/main/java/com/google/devtools/build/lib/actions/cache/InjectedStat.java new file mode 100644 index 0000000..9764cd8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/InjectedStat.java
@@ -0,0 +1,67 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import com.google.devtools.build.lib.vfs.FileStatus; + +/** + * A FileStatus corresponding to a file that is not determined by querying the file system. + */ +public class InjectedStat implements FileStatus { + + private final long mtime; + private final long size; + private final long nodeId; + + public InjectedStat(long mtime, long size, long nodeId) { + this.mtime = mtime; + this.size = size; + this.nodeId = nodeId; + } + + @Override + public boolean isFile() { + return true; + } + + @Override + public boolean isDirectory() { + return false; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public long getSize() { + return size; + } + + @Override + public long getLastModifiedTime() { + return mtime; + } + + @Override + public long getLastChangeTime() { + return getLastModifiedTime(); + } + + @Override + public long getNodeId() { + return nodeId; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java new file mode 100644 index 0000000..36c52b9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java
@@ -0,0 +1,92 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions.cache; + +import com.google.common.base.Preconditions; +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +import java.util.Arrays; +import java.util.Date; + +/** + * A class to represent file metadata. + * ActionCacheChecker may assume that, for a given file, equal + * metadata at different moments implies equal file-contents, + * where metadata equality is computed using Metadata.equals(). + * <p> + * NB! Several other parts of Blaze are relying on the fact that metadata + * uses mtime and not ctime. If metadata is ever changed + * to use ctime, all uses of Metadata must be carefully examined. + */ +@Immutable @ThreadSafe +public final class Metadata { + public final long mtime; + public final byte[] digest; + + // Convenience object for use with volatile files that we do not want checked + // (e.g. the build-changelist.txt) + public static final Metadata CONSTANT_METADATA = new Metadata(-1); + + public Metadata(long mtime) { + this.mtime = mtime; + this.digest = null; + } + + public Metadata(byte[] digest) { + this.mtime = 0L; + this.digest = Preconditions.checkNotNull(digest); + } + + @Override + public int hashCode() { + int hash = 0; + if (digest != null) { + // We are already dealing with the digest so we can just use portion of it + // as a hash code. + hash += digest[0] + (digest[1] << 8) + (digest[2] << 16) + (digest[3] << 24); + } else { + // Inlined hashCode for Long, so we don't + // have to construct an Object, just to compute + // a 32-bit hash out of a 64 bit value. + hash = (int) (mtime ^ (mtime >>> 32)); + } + return hash; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + if (!(that instanceof Metadata)) { + return false; + } + // Do a strict comparison - both digest and mtime should match + return Arrays.equals(this.digest, ((Metadata) that).digest) + && this.mtime == ((Metadata) that).mtime; + } + + @Override + public String toString() { + if (digest != null) { + return "MD5 " + BaseEncoding.base16().lowerCase().encode(digest); + } else if (mtime > 0) { + return "timestamp " + new Date(mtime); + } + return "no metadata"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java new file mode 100644 index 0000000..9d288db --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java
@@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.vfs.FileStatus; + +import java.io.IOException; +import java.util.Collection; + +/** Retrieves {@link Metadata} of {@link Artifact}s, and inserts virtual metadata as well. */ +public interface MetadataHandler { + /** + * Returns metadata for the given artifact or null if it does not exist. + * + * @param artifact artifact + * + * @return metadata instance or null if metadata cannot be obtained. + */ + Metadata getMetadataMaybe(Artifact artifact); + /** + * Returns metadata for the given artifact or throws an exception if the + * metadata could not be obtained. + * + * @return metadata instance + * + * @throws IOException if metadata could not be obtained. + */ + Metadata getMetadata(Artifact artifact) throws IOException; + + /** Sets digest for virtual artifacts (e.g. middlemen). {@code digest} must not be null. */ + void setDigestForVirtualArtifact(Artifact artifact, Digest digest); + + /** + * Injects provided digest into the metadata handler, simultaneously caching lstat() data as well. + */ + void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest); + + /** Returns true iff artifact exists. */ + boolean artifactExists(Artifact artifact); + /** Returns true iff artifact is a regular file. */ + boolean isRegularFile(Artifact artifact); + + /** + * @return Whether the artifact's data was injected. + * @throws IOException if implementation tried to stat artifact which threw an exception. + * Technically, this means that the artifact could not have been injected, but by throwing + * here we save the caller trying to stat this file on their own and throwing the same + * exception. Implementations are not guaranteed to throw in this case if they are able to + * determine that the artifact is not injected without statting it. + */ + boolean isInjected(Artifact artifact) throws IOException; + + /** Discards all metadata for the given artifacts, presumably because they will be modified. */ + void discardMetadata(Collection<Artifact> artifactList); + +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/NullActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/NullActionCache.java new file mode 100644 index 0000000..0975150 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/NullActionCache.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions.cache; + +import java.io.IOException; +import java.io.PrintStream; + +/** + * A no-op action cache that never caches anything. + */ +public final class NullActionCache implements ActionCache { + + @Override + public void put(String key, Entry entry) { + } + + @Override + public Entry get(String key) { + return null; + } + + @Override + public void remove(String key) { + } + + @Override + public Entry createEntry(String key) { + return new ActionCache.Entry(key); + } + + @Override + public long save() throws IOException { + return 0; + } + + @Override + public void dump(PrintStream out) { + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexer.java b/src/main/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexer.java new file mode 100644 index 0000000..bd98b2b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexer.java
@@ -0,0 +1,161 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import com.google.common.collect.MapMaker; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe; +import com.google.devtools.build.lib.util.CanonicalStringIndexer; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.PersistentMap; +import com.google.devtools.build.lib.util.StringCanonicalizer; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +/** + * Persistent version of the CanonicalStringIndexer. + * + * <p>This class is backed by a PersistentMap that holds one direction of the + * canonicalization mapping. The other direction is handled purely in memory + * and reconstituted at load-time. + * + * <p>Thread-safety is ensured by locking on all mutating operations from the + * superclass. Read-only operations are not locked, but rather backed by + * ConcurrentMaps. + */ +@ConditionallyThreadSafe // condition: each instance must instantiated with + // different dataFile. +final class PersistentStringIndexer extends CanonicalStringIndexer { + + /** + * Persistent metadata map. Used as a backing map to provide a persistent + * implementation of the metadata cache. + */ + private static final class PersistentIndexMap extends PersistentMap<String, Integer> { + private static final int VERSION = 0x01; + private static final long SAVE_INTERVAL_NS = 3L * 1000 * 1000 * 1000; + + private final Clock clock; + private long nextUpdate; + + public PersistentIndexMap(Path mapFile, Path journalFile, Clock clock) throws IOException { + super(VERSION, PersistentStringIndexer.<String, Integer>newConcurrentMap(INITIAL_ENTRIES), + mapFile, journalFile); + this.clock = clock; + nextUpdate = clock.nanoTime(); + load(/*throwOnLoadFailure=*/true); + } + + @Override + protected boolean updateJournal() { + long time = clock.nanoTime(); + if (SAVE_INTERVAL_NS == 0 || time > nextUpdate) { + nextUpdate = time + SAVE_INTERVAL_NS; + return true; + } + return false; + } + + @Override + public Integer remove(Object object) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + public void flush() { + super.forceFlush(); + } + + @Override + protected String readKey(DataInputStream in) throws IOException { + int length = in.readInt(); + if (length < 0) { + throw new IOException("corrupt key length: " + length); + } + byte[] content = new byte[length]; + in.readFully(content); + return StringCanonicalizer.intern(bytes2string(content)); + } + + @Override + protected Integer readValue(DataInputStream in) throws IOException { + return in.readInt(); + } + + @Override + protected void writeKey(String key, DataOutputStream out) throws IOException { + byte[] content = string2bytes(key); + out.writeInt(content.length); + out.write(content); + } + + @Override + protected void writeValue(Integer value, DataOutputStream out) throws IOException { + out.writeInt(value); + } + } + + private final PersistentIndexMap persistentIndexMap; + private static final int INITIAL_ENTRIES = 10000; + + /** + * Instantiates and loads instance of the persistent string indexer. + */ + static PersistentStringIndexer newPersistentStringIndexer(Path dataPath, + Clock clock) throws IOException { + PersistentIndexMap persistentIndexMap = new PersistentIndexMap(dataPath, + FileSystemUtils.replaceExtension(dataPath, ".journal"), clock); + Map<Integer, String> reverseMapping = newConcurrentMap(INITIAL_ENTRIES); + for (Map.Entry<String, Integer> entry : persistentIndexMap.entrySet()) { + if (reverseMapping.put(entry.getValue(), entry.getKey()) != null) { + throw new IOException("Corrupted filename index has duplicate entry: " + entry.getKey()); + } + } + return new PersistentStringIndexer(persistentIndexMap, reverseMapping); + } + + private PersistentStringIndexer(PersistentIndexMap stringToInt, + Map<Integer, String> intToString) { + super(stringToInt, intToString); + this.persistentIndexMap = stringToInt; + } + + /** + * Saves index data to the file. + */ + synchronized long save() throws IOException { + return persistentIndexMap.save(); + } + + /** + * Flushes the journal. + */ + synchronized void flush() { + persistentIndexMap.flush(); + } + + private static <K, V> ConcurrentMap<K, V> newConcurrentMap(int expectedCapacity) { + return new MapMaker().initialCapacity(expectedCapacity).makeMap(); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/VirtualActionInput.java b/src/main/java/com/google/devtools/build/lib/actions/cache/VirtualActionInput.java new file mode 100644 index 0000000..debb8ea --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/VirtualActionInput.java
@@ -0,0 +1,31 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import com.google.devtools.build.lib.actions.ActionInput; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * An ActionInput that does not actually exist on the filesystem, but can still be written to an + * OutputStream. + */ +public interface VirtualActionInput extends ActionInput { + /** + * Writes the the fake file to an OutputStream. MUST be deterministic, in that multiple calls + * to write the same VirtualActionInput must write identical bytes. + */ + void writeTo(OutputStream out) throws IOException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AbstractConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/AbstractConfiguredTarget.java new file mode 100644 index 0000000..e574978 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/AbstractConfiguredTarget.java
@@ -0,0 +1,111 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.PackageSpecification; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.ClassObject; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; + +/** + * An abstract implementation of ConfiguredTarget in which all properties are + * assigned trivial default values. + */ +public abstract class AbstractConfiguredTarget + implements ConfiguredTarget, VisibilityProvider, ClassObject { + private final Target target; + private final BuildConfiguration configuration; + + private final NestedSet<PackageSpecification> visibility; + + AbstractConfiguredTarget(Target target, + BuildConfiguration configuration) { + this.target = target; + this.configuration = configuration; + this.visibility = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + AbstractConfiguredTarget(TargetContext targetContext) { + this.target = targetContext.getTarget(); + this.configuration = targetContext.getConfiguration(); + this.visibility = targetContext.getVisibility(); + } + + @Override + public final NestedSet<PackageSpecification> getVisibility() { + return visibility; + } + + @Override + public Target getTarget() { + return target; + } + + @Override + public BuildConfiguration getConfiguration() { + return configuration; + } + + @Override + public Label getLabel() { + return getTarget().getLabel(); + } + + @Override + public String toString() { + return "ConfiguredTarget(" + getTarget().getLabel() + ", " + getConfiguration() + ")"; + } + + @Override + public <P extends TransitiveInfoProvider> P getProvider(Class<P> provider) { + AnalysisUtils.checkProvider(provider); + if (provider.isAssignableFrom(getClass())) { + return provider.cast(this); + } else { + return null; + } + } + + @Override + public Object getValue(String name) { + if (name.equals("label")) { + return getLabel(); + } else if (name.equals("files")) { + // A shortcut for files to build in Skylark. FileConfiguredTarget and RunleConfiguredTarget + // always has FileProvider and Error- and PackageGroupConfiguredTarget-s shouldn't be + // accessible in Skylark. + return SkylarkNestedSet.of(Artifact.class, getProvider(FileProvider.class).getFilesToBuild()); + } + return get(name); + } + + @Override + public String errorMessage(String name) { + return null; + } + + @Override + public ImmutableCollection<String> getKeys() { + return ImmutableList.<String>builder().add("label").add("files").build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AlwaysBuiltArtifactsProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/AlwaysBuiltArtifactsProvider.java new file mode 100644 index 0000000..e4d40fc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/AlwaysBuiltArtifactsProvider.java
@@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Artifacts that should be built when a target is mentioned in the command line, but are neither in + * the {@code filesToBuild} nor in the runfiles. + * + * <p> + * Link actions, may not run a link for their transitive dependencies, so it does not force the + * source files in the transitive closure to be built by default. However, users expect builds to + * fail when there is an error in a dependent library, so we use this mechanism to force their + * compilation. + */ +@Immutable +public final class AlwaysBuiltArtifactsProvider implements TransitiveInfoProvider { + + private final NestedSet<Artifact> artifactsToAlwaysBuild; + + public AlwaysBuiltArtifactsProvider(NestedSet<Artifact> artifactsToAlwaysBuild) { + this.artifactsToAlwaysBuild = artifactsToAlwaysBuild; + } + + /** + * Returns the collection of artifacts to be built. + */ + public NestedSet<Artifact> getArtifactsToAlwaysBuild() { + return artifactsToAlwaysBuild; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java new file mode 100644 index 0000000..0bccc72 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java
@@ -0,0 +1,128 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionRegistry; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.MiddlemanFactory; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; + +/** + * The set of services that are provided to {@link ConfiguredTarget} objects + * during initialization. + */ +public interface AnalysisEnvironment extends ActionRegistry { + /** + * Returns a callback to be used in this build for reporting analysis errors. + */ + EventHandler getEventHandler(); + + /** + * Returns whether any errors were reported to this instance. + */ + boolean hasErrors(); + + /** + * Returns the artifact for the derived file {@code rootRelativePath}. + * + * <p>Creates the artifact if necessary and sets the root of that artifact to {@code root}. + */ + Artifact getDerivedArtifact(PathFragment rootRelativePath, Root root); + + /** + * Returns an artifact for the derived file {@code rootRelativePath} whose changes do not cause + * a rebuild. + * + * <p>Creates the artifact if necessary and sets the root of that artifact to {@code root}. + * + * <p>This is useful for files that store data that changes very frequently (e.g. current time) + * but does not substantially affect the result of the build. + */ + Artifact getConstantMetadataArtifact(PathFragment rootRelativePath, + Root root); + + /** + * Returns the artifact for the derived file {@code rootRelativePath}, + * creating it if necessary, and setting the root of that artifact to + * {@code root}. The artifact will represent the output directory of a {@code Fileset}. + */ + Artifact getFilesetArtifact(PathFragment rootRelativePath, Root root); + + /** + * Returns the artifact for the specified tool. + */ + Artifact getEmbeddedToolArtifact(String embeddedPath); + + /** + * Returns the middleman factory associated with the build. + */ + // TODO(bazel-team): remove this method and replace it with delegate methods. + MiddlemanFactory getMiddlemanFactory(); + + /** + * Returns the generating action for the given local artifact. + * + * If the artifact was created in another analysis environment (e.g. by a different configured + * target instance) or the artifact is a source artifact, it returns null. + */ + Action getLocalGeneratingAction(Artifact artifact); + + /** + * Returns the actions that were registered so far with this analysis environment, that is, all + * the actions that were created by the current target being analyzed. + */ + Iterable<Action> getRegisteredActions(); + + /** + * Returns the Skyframe SkyFunction.Environment if available. Otherwise, null. + * + * <p>If you need to use this for something other than genquery, please think long and hard + * about that. + */ + SkyFunction.Environment getSkyframeEnv(); + + /** + * Returns the Artifact that is used to hold the non-volatile workspace status for the current + * build request. + */ + Artifact getStableWorkspaceStatusArtifact(); + + /** + * Returns the Artifact that is used to hold the volatile workspace status (e.g. build + * changelist) for the current build request. + */ + Artifact getVolatileWorkspaceStatusArtifact(); + + /** + * Returns the Artifacts that contain the workspace status for the current build request. + * + * @param ruleContext the rule to use for error reporting and to determine the + * configuration + */ + ImmutableList<Artifact> getBuildInfo(RuleContext ruleContext, BuildInfoKey key); + + /** + * Returns the set of orphan Artifacts (i.e. Artifacts without generating action). Should only be + * called after the ConfiguredTarget is created. + */ + ImmutableSet<Artifact> getOrphanArtifacts(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisFailureEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisFailureEvent.java new file mode 100644 index 0000000..5064163 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisFailureEvent.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.syntax.Label; + +/** + * This event is fired during the build, when it becomes known that the analysis + * of a target cannot be completed because of an error in one of its + * dependencies. + */ +public class AnalysisFailureEvent { + private final LabelAndConfiguration failedTarget; + private final Label failureReason; + + public AnalysisFailureEvent(LabelAndConfiguration failedTarget, Label failureReason) { + this.failedTarget = failedTarget; + this.failureReason = failureReason; + } + + public LabelAndConfiguration getFailedTarget() { + return failedTarget; + } + + public Label getFailureReason() { + return failureReason; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisHooks.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisHooks.java new file mode 100644 index 0000000..82c4485 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisHooks.java
@@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.PackageManager; + +/** + * This interface resolves target - configuration pairs to {@link ConfiguredTarget} instances. + * + * <p>This interface is used to provide analysis phase functionality to actions that need it in + * the execution phase. + */ +public interface AnalysisHooks { + /** + * Returns the package manager used during the analysis phase. + */ + PackageManager getPackageManager(); + + /** + * Resolves an existing configured target. Returns null if it is not in the cache. + */ + ConfiguredTarget getExistingConfiguredTarget(Target target, BuildConfiguration configuration); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseCompleteEvent.java new file mode 100644 index 0000000..0d1e565 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseCompleteEvent.java
@@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableList; + +import java.util.Collection; + +/** + * This event is fired after the analysis phase is complete. + */ +public class AnalysisPhaseCompleteEvent { + + private final Collection<ConfiguredTarget> targets; + private final long timeInMs; + private int targetsVisited; + + /** + * Construct the event. + * @param targets The set of active targets that remain. + */ + public AnalysisPhaseCompleteEvent(Collection<? extends ConfiguredTarget> targets, + int targetsVisited, long timeInMs) { + this.timeInMs = timeInMs; + this.targets = ImmutableList.copyOf(targets); + this.targetsVisited = targetsVisited; + } + + /** + * @return The set of active targets remaining, which is a subset + * of the targets we attempted to analyze. + */ + public Collection<ConfiguredTarget> getTargets() { + return targets; + } + + /** + * @return The number of targets freshly visited during analysis + */ + public int getTargetsVisited() { + return targetsVisited; + } + + public long getTimeInMs() { + return timeInMs; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseStartedEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseStartedEvent.java new file mode 100644 index 0000000..fc97c60 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseStartedEvent.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Collection; + +/** + * This event is fired before the analysis phase is started. + */ +public class AnalysisPhaseStartedEvent { + + private final Iterable<Label> labels; + + /** + * Construct the event. + * @param targets The set of active targets that remain. + */ + public AnalysisPhaseStartedEvent(Collection<Target> targets) { + this.labels = Iterables.transform(targets, new Function<Target, Label>() { + @Override + public Label apply(Target input) { + return input.getLabel(); + } + }); + } + + /** + * @return The set of active targets remaining, which is a subset + * of the targets we attempted to load. + */ + public Iterable<Label> getLabels() { + return labels; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisUtils.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisUtils.java new file mode 100644 index 0000000..2e4c251 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisUtils.java
@@ -0,0 +1,146 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.TriState; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Collection; + +/** + * Utility functions for use during analysis. + */ +public final class AnalysisUtils { + + private AnalysisUtils() { + throw new IllegalStateException(); // utility class + } + + /** + * Returns whether link stamping is enabled for a rule. + * + * <p>This returns false for unstampable rule classes and for rules in the + * host configuration. Otherwise it returns the value of the stamp attribute, + * or of the stamp option if the attribute value is -1. + */ + public static boolean isStampingEnabled(RuleContext ruleContext) { + BuildConfiguration config = ruleContext.getConfiguration(); + Rule rule = ruleContext.getRule(); + if (config.isHostConfiguration() + || !rule.getRuleClassObject().hasAttr("stamp", Type.TRISTATE)) { + return false; + } + TriState stamp = ruleContext.attributes().get("stamp", Type.TRISTATE); + return stamp == TriState.YES || (stamp == TriState.AUTO && config.stampBinaries()); + } + + // TODO(bazel-team): These need Iterable<? extends TransitiveInfoCollection> because they need to + // be called with Iterable<ConfiguredTarget>. Once the configured target lockdown is complete, we + // can eliminate the "extends" clauses. + /** + * Returns the list of providers of the specified type from a set of transitive info + * collections. + */ + public static <C extends TransitiveInfoProvider> Iterable<C> getProviders( + Iterable<? extends TransitiveInfoCollection> prerequisites, Class<C> provider) { + Collection<C> result = new ArrayList<>(); + for (TransitiveInfoCollection prerequisite : prerequisites) { + C prerequisiteProvider = prerequisite.getProvider(provider); + if (prerequisiteProvider != null) { + result.add(prerequisiteProvider); + } + } + return ImmutableList.copyOf(result); + } + + /** + * Returns the iterable of collections that have the specified provider. + */ + public static <S extends TransitiveInfoCollection, C extends TransitiveInfoProvider> Iterable<S> + filterByProvider(Iterable<S> prerequisites, final Class<C> provider) { + return Iterables.filter(prerequisites, new Predicate<S>() { + @Override + public boolean apply(S target) { + return target.getProvider(provider) != null; + } + }); + } + + /** + * Returns the path of the associated manifest file for the path of a Fileset. Works for both + * exec paths and root relative paths. + */ + public static PathFragment getManifestPathFromFilesetPath(PathFragment filesetDir) { + PathFragment manifestDir = filesetDir.replaceName("_" + filesetDir.getBaseName()); + PathFragment outputManifestFrag = manifestDir.getRelative("MANIFEST"); + return outputManifestFrag; + } + + /** + * Returns the middleman artifact on the specified attribute of the specified rule, or an empty + * set if it does not exist. + */ + public static NestedSet<Artifact> getMiddlemanFor(RuleContext rule, String attribute) { + TransitiveInfoCollection prereq = rule.getPrerequisite(attribute, Mode.HOST); + if (prereq == null) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + MiddlemanProvider provider = prereq.getProvider(MiddlemanProvider.class); + if (provider == null) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + return provider.getMiddlemanArtifact(); + } + + /** + * Returns a path fragment qualified by the rule name and unique fragment to + * disambiguate artifacts produced from the source file appearing in + * multiple rules. + * + * <p>For example "//pkg:target" -> "pkg/<fragment>/target. + */ + public static PathFragment getUniqueDirectory(Label label, PathFragment fragment) { + return label.getPackageFragment().getRelative(fragment) + .getRelative(label.getName()); + } + + /** + * Checks that the given provider class either refers to an interface or to a value class. + */ + public static <T extends TransitiveInfoProvider> void checkProvider(Class<T> clazz) { + if (!clazz.isInterface()) { + Preconditions.checkArgument(Modifier.isFinal(clazz.getModifiers()), + clazz.getName() + " has to be final"); + Preconditions.checkArgument(clazz.isAnnotationPresent(Immutable.class), + clazz.getName() + " has to be tagged with @Immutable"); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/Aspect.java b/src/main/java/com/google/devtools/build/lib/analysis/Aspect.java new file mode 100644 index 0000000..3f4a06e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/Aspect.java
@@ -0,0 +1,82 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.UnmodifiableIterator; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Extra information about a configured target computed on request of a dependent. + * + * <p>Analogous to {@link ConfiguredTarget}: contains a bunch of transitive info providers, which + * are merged with the providers of the associated configured target before they are passed to + * the configured target factories that depend on the configured target to which this aspect is + * added. + * + * <p>Aspects are created alongside configured targets on request from dependents. + */ +@Immutable +public final class Aspect implements Iterable<TransitiveInfoProvider> { + private final + ImmutableMap<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers; + + private Aspect( + ImmutableMap<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers) { + this.providers = providers; + } + + /** + * Returns the providers created by the aspect. + */ + public ImmutableMap<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> + getProviders() { + return providers; + } + + @Override + public UnmodifiableIterator<TransitiveInfoProvider> iterator() { + return providers.values().iterator(); + } + + /** + * Builder for {@link Aspect}. + */ + public static class Builder { + private final Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> + providers = new LinkedHashMap<>(); + + /** + * Adds a provider to the aspect. + */ + public Builder addProvider( + Class<? extends TransitiveInfoProvider> key, TransitiveInfoProvider value) { + Preconditions.checkNotNull(key); + Preconditions.checkNotNull(value); + AnalysisUtils.checkProvider(key); + Preconditions.checkState(!providers.containsKey(key)); + providers.put(key, value); + return this; + } + + public Aspect build() { + return new Aspect(ImmutableMap.copyOf(providers)); + } + } +} \ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BaseRuleClasses.java b/src/main/java/com/google/devtools/build/lib/analysis/BaseRuleClasses.java new file mode 100644 index 0000000..ad5756e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/BaseRuleClasses.java
@@ -0,0 +1,265 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.DATA; +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.DISTRIBUTIONS; +import static com.google.devtools.build.lib.packages.Type.INTEGER; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.LICENSE; +import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.RunUnder; +import com.google.devtools.build.lib.analysis.constraints.EnvironmentRule; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel; +import com.google.devtools.build.lib.packages.Attribute.LateBoundLabelList; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.packages.TestSize; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileTypeSet; + +import java.util.List; + +/** + * Rule class definitions used by (almost) every rule. + */ +public class BaseRuleClasses { + /** + * Label of the pseudo-filegroup that contains all the targets that are needed + * for running tests in coverage mode. + */ + private static final Label COVERAGE_SUPPORT_LABEL = + Label.parseAbsoluteUnchecked("//tools/defaults:coverage"); + + private static final Attribute.ComputedDefault obsoleteDefault = + new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + return rule.getPackageDefaultObsolete(); + } + }; + + private static final Attribute.ComputedDefault testonlyDefault = + new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + return rule.getPackageDefaultTestOnly(); + } + }; + + private static final Attribute.ComputedDefault deprecationDefault = + new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + return rule.getPackageDefaultDeprecation(); + } + }; + + /** + * Implementation for the :action_listener attribute. + */ + private static final LateBoundLabelList<BuildConfiguration> ACTION_LISTENER = + new LateBoundLabelList<BuildConfiguration>() { + @Override + public List<Label> getDefault(Rule rule, BuildConfiguration configuration) { + // action_listeners are special rules; they tell the build system to add extra_actions to + // existing rules. As such they need an edge to every ConfiguredTarget with the limitation + // that they only run on the target configuration and should not operate on action_listeners + // and extra_actions themselves (to avoid cycles). + return configuration.getActionListeners(); + } + }; + + private static final LateBoundLabelList<BuildConfiguration> COVERAGE_SUPPORT = + new LateBoundLabelList<BuildConfiguration>(ImmutableList.of(COVERAGE_SUPPORT_LABEL)) { + @Override + public List<Label> getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.isCodeCoverageEnabled() + ? ImmutableList.<Label>copyOf(configuration.getCoverageLabels()) + : ImmutableList.<Label>of(); + } + }; + + private static final LateBoundLabelList<BuildConfiguration> COVERAGE_REPORT_GENERATOR = + new LateBoundLabelList<BuildConfiguration>(ImmutableList.of(COVERAGE_SUPPORT_LABEL)) { + @Override + public List<Label> getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.isCodeCoverageEnabled() + ? ImmutableList.<Label>copyOf(configuration.getCoverageReportGeneratorLabels()) + : ImmutableList.<Label>of(); + } + }; + + /** + * Implementation for the :run_under attribute. + */ + private static final LateBoundLabel<BuildConfiguration> RUN_UNDER = + new LateBoundLabel<BuildConfiguration>() { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + RunUnder runUnder = configuration.getRunUnder(); + return runUnder == null ? null : runUnder.getLabel(); + } + }; + + /** + * A base rule for all test rules. + */ + @BlazeRule(name = "$test_base_rule", + type = RuleClassType.ABSTRACT) + public static final class TestBaseRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .add(attr("size", STRING).value("medium").taggable() + .nonconfigurable("policy decision: should be consistent across configurations")) + .add(attr("timeout", STRING).taggable() + .nonconfigurable("policy decision: should be consistent across configurations") + .value(new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + TestSize size = TestSize.getTestSize(rule.get("size", Type.STRING)); + if (size != null) { + String timeout = size.getDefaultTimeout().toString(); + if (timeout != null) { + return timeout; + } + } + return "illegal"; + } + })) + .add(attr("flaky", BOOLEAN).value(false).taggable() + .nonconfigurable("policy decision: should be consistent across configurations")) + .add(attr("shard_count", INTEGER).value(-1)) + .add(attr("local", BOOLEAN).value(false).taggable() + .nonconfigurable("policy decision: should be consistent across configurations")) + .add(attr("args", STRING_LIST) + .nonconfigurable("policy decision: should be consistent across configurations")) + .add(attr("$test_runtime", LABEL_LIST).cfg(HOST).value(ImmutableList.of( + env.getLabel("//tools/test:runtime")))) + + // TODO(bazel-team): TestActions may need to be run with coverage, so all tests + // implicitly depend on crosstool, which provides gcov. We could add gcov to + // InstrumentedFilesProvider.getInstrumentationMetadataFiles() (or a new method) for + // all the test rules that have C++ in their transitive closure. Then this could go. + .add(attr(":coverage_support", LABEL_LIST).cfg(HOST).value(COVERAGE_SUPPORT)) + .add(attr(":coverage_report_generator", LABEL_LIST).cfg(HOST) + .value(COVERAGE_REPORT_GENERATOR)) + + // The target itself and run_under both run on the same machine. We use the DATA config + // here because the run_under acts like a data dependency (e.g. no LIPO optimization). + .add(attr(":run_under", LABEL).cfg(DATA).value(RUN_UNDER)) + .build(); + } + } + + /** + * Share common attributes across both base and Skylark base rules. + */ + public static RuleClass.Builder commonCoreAndSkylarkAttributes(RuleClass.Builder builder) { + return builder + // The visibility attribute is special: it is a nodep label, and loading the + // necessary package groups is handled by {@link LabelVisitor#visitTargetVisibility}. + // Package groups always have the null configuration so that they are not duplicated + // needlessly. + .add(attr("visibility", NODEP_LABEL_LIST).orderIndependent().cfg(HOST) + .nonconfigurable("special attribute integrated more deeply into Bazel's core logic")) + .add(attr("deprecation", STRING).value(deprecationDefault) + .nonconfigurable("Used in core loading phase logic with no access to configs")) + .add(attr("tags", STRING_LIST).orderIndependent().taggable() + .nonconfigurable("low-level attribute, used in TargetUtils without configurations")) + .add(attr("generator_name", STRING).undocumented("internal")) + .add(attr("generator_function", STRING).undocumented("internal")) + .add(attr("testonly", BOOLEAN).value(testonlyDefault) + .nonconfigurable("policy decision: rules testability should be consistent")) + .add(attr(RuleClass.COMPATIBLE_ENVIRONMENT_ATTR, LABEL_LIST) + .allowedRuleClasses(EnvironmentRule.RULE_NAME) + .cfg(Attribute.ConfigurationTransition.HOST) + .allowedFileTypes(FileTypeSet.NO_FILE) + .undocumented("not yet released")) + .add(attr(RuleClass.RESTRICTED_ENVIRONMENT_ATTR, LABEL_LIST) + .allowedRuleClasses(EnvironmentRule.RULE_NAME) + .cfg(Attribute.ConfigurationTransition.HOST) + .allowedFileTypes(FileTypeSet.NO_FILE) + .undocumented("not yet released")); + } + + /** + * Common parts of rules. + */ + @BlazeRule(name = "$base_rule", + type = RuleClassType.ABSTRACT) + public static final class BaseRule implements RuleDefinition { + @Override + public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) { + return commonCoreAndSkylarkAttributes(builder) + // The name attribute is handled specially, so it does not appear here. + // + // Aggregates the labels of all {@link ConfigRuleClasses} rules this rule uses (e.g. + // keys for configurable attributes). This is specially populated in + // {@RuleClass#populateRuleAttributeValues}. + // + // This attribute is not needed for actual builds. Its main purpose is so query's + // proto/XML output includes the labels of config dependencies, so, e.g., depserver + // reverse dependency lookups remain accurate. These can't just be added to the + // attribute definitions proto/XML queries already output because not all attributes + // contain labels. + // + // Builds and Blaze-interactive queries don't need this because they find dependencies + // through direct Rule label visitation, which already factors these in. + .add(attr("$config_dependencies", LABEL_LIST) + .nonconfigurable("not intended for actual builds")) + .add(attr("licenses", LICENSE) + .nonconfigurable("Used in core loading phase logic with no access to configs")) + .add(attr("distribs", DISTRIBUTIONS) + .nonconfigurable("Used in core loading phase logic with no access to configs")) + .add(attr("obsolete", BOOLEAN).value(obsoleteDefault) + .nonconfigurable("Used in core loading phase logic with no access to configs")) + .add(attr(":action_listener", LABEL_LIST).cfg(HOST).value(ACTION_LISTENER)) + .build(); + } + } + + /** + * Common ancestor class for all rules. + */ + @BlazeRule(name = "$rule", + type = RuleClassType.ABSTRACT, + ancestors = { BaseRule.class }) + public static final class RuleBase implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .add(attr("deps", LABEL_LIST).legacyAllowAnyFileType()) + .add(attr("data", LABEL_LIST).cfg(DATA).allowedFileTypes(FileTypeSet.ANY_FILE)) + .build(); + } + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BaselineCoverageArtifactsProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/BaselineCoverageArtifactsProvider.java new file mode 100644 index 0000000..ab74581 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/BaselineCoverageArtifactsProvider.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A {@link TransitiveInfoProvider} that has baseline coverage artifacts. + */ +@Immutable +public final class BaselineCoverageArtifactsProvider implements TransitiveInfoProvider { + private final ImmutableList<Artifact> baselineCoverageArtifacts; + + public BaselineCoverageArtifactsProvider(ImmutableList<Artifact> baselineCoverageArtifacts) { + this.baselineCoverageArtifacts = baselineCoverageArtifacts; + } + + /** + * Returns a set of baseline coverage artifacts for a given set of configured targets. + * + * <p>These artifacts represent "empty" code coverage data for non-test libraries and binaries and + * used to establish correct baseline when calculating code coverage ratios since they would cover + * completely non-tested code as well. + */ + public ImmutableList<Artifact> getBaselineCoverageArtifacts() { + return baselineCoverageArtifacts; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BlazeDirectories.java b/src/main/java/com/google/devtools/build/lib/analysis/BlazeDirectories.java new file mode 100644 index 0000000..38a8d41 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/BlazeDirectories.java
@@ -0,0 +1,183 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.util.StringCanonicalizer; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.Serializable; + +import javax.annotation.Nullable; + +/** + * Encapsulation of all of the interesting top-level directories in any Blaze application. + * + * <p>The <code>installBase</code> is the directory where the Blaze binary has been installed.The + * <code>workspace</code> is the top-level directory in the user's client (possibly read-only).The + * <code>outputBase</code> is the directory below which Blaze puts all its state. The + * <code>execRoot</code> is the working directory for all spawned tools, which is generally below + * <code>outputBase</code>. + * + * <p>There is a 1:1 correspondence between a running Blaze instance and an output base directory; + * however, multiple Blaze instances may compile code that's in the same workspace, even on the same + * machine. If the user does not qualify an output base directory, the startup code will derive it + * deterministically from the workspace. Note also that while the Blaze server process runs with the + * workspace directory as its working directory, the client process may have a different working + * directory, typically a subdirectory. + * + * <p>Do not put shortcuts to specific files here! + */ +@Immutable +public final class BlazeDirectories implements Serializable { + + // Output directory name, relative to the execRoot. + // TODO(bazel-team): (2011) make this private? + public static final String RELATIVE_OUTPUT_PATH = StringCanonicalizer.intern( + Constants.PRODUCT_NAME + "-out"); + + // Include directory name, relative to execRoot/blaze-out/configuration. + public static final String RELATIVE_INCLUDE_DIR = StringCanonicalizer.intern("include"); + + private final Path installBase; // Where Blaze gets unpacked + private final Path workspace; // Workspace root and server CWD + private final Path outputBase; // The root of the temp and output trees + private final Path execRoot; // the root of all build actions + + // These two are kept to avoid creating new objects every time they are accessed. This showed up + // in a profiler. + private final Path outputPath; + private final Path localOutputPath; + + public BlazeDirectories(Path installBase, Path outputBase, @Nullable Path workspace) { + this.installBase = installBase; + this.workspace = workspace; + this.outputBase = outputBase; + if (this.workspace == null) { + // TODO(bazel-team): this should be null, but at the moment there is a lot of code that + // depends on it being non-null. + this.execRoot = outputBase.getChild("default-exec-root"); + } else { + this.execRoot = outputBase.getChild(workspace.getBaseName()); + } + this.outputPath = execRoot.getRelative(RELATIVE_OUTPUT_PATH); + Preconditions.checkState(this.workspace == null || outputPath.asFragment().equals( + outputPathFromOutputBase(outputBase.asFragment(), workspace.asFragment()))); + this.localOutputPath = outputBase.getRelative(BlazeDirectories.RELATIVE_OUTPUT_PATH); + } + + /** + * Returns the Filesystem that all of our directories belong to. Handy for + * resolving absolute paths. + */ + public FileSystem getFileSystem() { + return installBase.getFileSystem(); + } + + /** + * Returns the installation base directory. Currently used by info command only. + */ + public Path getInstallBase() { + return installBase; + } + + /** + * Returns the workspace directory, which is also the working dir of the server. + */ + public Path getWorkspace() { + return workspace; + } + + /** + * Returns if the workspace directory is a valid workspace. + */ + public boolean inWorkspace() { + return this.workspace != null; + } + + /** + * Returns the base of the output tree, which hosts all build and scratch + * output for a user and workspace. + */ + public Path getOutputBase() { + return outputBase; + } + + /** + * Returns the execution root. This is the directory underneath which Blaze builds the source + * symlink forest, to represent the merged view of different workspaces specified + * with --package_path. + */ + public Path getExecRoot() { + return execRoot; + } + + /** + * Returns the output path used by this Blaze instance. + */ + public Path getOutputPath() { + return outputPath; + } + + /** + * @param outputBase the outputBase as a path fragment. + * @param workspace the workspace as a path fragment. + * @return the outputPath as a path fragment, given the outputBase. + */ + public static PathFragment outputPathFromOutputBase( + PathFragment outputBase, PathFragment workspace) { + if (workspace.equals(PathFragment.EMPTY_FRAGMENT)) { + return outputBase; + } + return outputBase.getRelative(workspace.getBaseName() + "/" + RELATIVE_OUTPUT_PATH); + } + + /** + * Returns the local output path used by this Blaze instance. + */ + public Path getLocalOutputPath() { + return localOutputPath; + } + + /** + * Returns the directory where the stdout/stderr for actions can be stored + * temporarily for a build. If the directory already exists, the directory + * is cleaned. + */ + public Path getActionConsoleOutputDirectory() { + return getOutputBase().getRelative("action_outs"); + } + + /** + * Returns the installed embedded binaries directory, under the shared + * installBase location. + */ + public Path getEmbeddedBinariesRoot() { + return installBase.getChild("_embedded_binaries"); + } + + /** + * Returns the configuration-independent root where the build-data should be placed, given the + * {@link BlazeDirectories} of this server instance. Nothing else should be placed here. + */ + public Root getBuildDataDirectory() { + return Root.asDerivedRoot(getExecRoot(), getOutputPath()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BlazeRule.java b/src/main/java/com/google/devtools/build/lib/analysis/BlazeRule.java new file mode 100644 index 0000000..c349e65 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/BlazeRule.java
@@ -0,0 +1,55 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * An annotation for rule classes. + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface BlazeRule { + /** + * The name of the rule, as it appears in the BUILD file. If it starts with + * '$', the rule will be hidden from users and will only be usable from + * inside Blaze. + */ + String name(); + + /** + * The type of the rule. It can be an abstract rule, a normal rule or a test + * rule. If the rule type is abstract, the configured class must not be set. + */ + RuleClassType type() default RuleClassType.NORMAL; + + /** + * The {@link RuleConfiguredTargetFactory} class that implements this rule. If the rule is + * abstract, this must not be set. + */ + Class<? extends RuleConfiguredTargetFactory> factoryClass() + default RuleConfiguredTargetFactory.class; + + /** + * The list of other rule classes this rule inherits from. + */ + Class<? extends RuleDefinition>[] ancestors() default {}; +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BlazeVersionInfo.java b/src/main/java/com/google/devtools/build/lib/analysis/BlazeVersionInfo.java new file mode 100644 index 0000000..d5b5f94 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/BlazeVersionInfo.java
@@ -0,0 +1,113 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.util.StringUtilities; + +import java.util.Map; +import java.util.logging.Logger; + +/** + * Determines the version information of the current process. + * + * <p>The version information is a dictionary mapping from string keys to string values. For + * build stamping, it should have the key "Build label", which contains among others a + * XXXXXXX-YYYY.MM.DD string to indicate the version of the release. If no data is available + * (eg. when running non-released version), {@link #isAvailable()} returns false. + */ +public class BlazeVersionInfo { + private final Map<String, String> buildData = Maps.newTreeMap(); + private static BlazeVersionInfo instance = null; + private static final String BUILD_LABEL = "Build label"; + + private static final Logger LOG = Logger.getLogger(BlazeVersionInfo.class.getName()); + + public BlazeVersionInfo(Map<String, String> info) { + buildData.putAll(info); + } + + /** + * Accessor method for BlazeVersionInfo singleton. + * + * <p>If setBuildInfo was not called, returns an empty BlazeVersionInfo instance, which should + * not be persisted. + */ + public static synchronized BlazeVersionInfo instance() { + if (instance == null) { + return new BlazeVersionInfo(ImmutableMap.<String, String>of()); + } + return instance; + } + + private static void logVersionInfo(BlazeVersionInfo info) { + if (info.getSummary() == null) { + LOG.warning("Blaze release version information not available"); + } else { + LOG.info("Blaze version info: " + info.getSummary()); + } + } + + /** + * Sets build info. + * + * <p>This should be called once in the program execution, as early soon as possible, so we + * can have the version information even before modules are initialized. + */ + public static synchronized void setBuildInfo(Map<String, String> info) { + if (instance != null) { + throw new IllegalStateException("setBuildInfo called twice."); + } + instance = new BlazeVersionInfo(info); + logVersionInfo(instance); + } + + /** + * Indicates whether version information is available. + */ + public boolean isAvailable() { + return !buildData.isEmpty(); + } + + /** + * Returns the summary which gets displayed in the 'version' command. + * The summary is a list of formatted key / value pairs. + */ + public String getSummary() { + if (buildData.isEmpty()) { + return null; + } + return StringUtilities.layoutTable(buildData); + } + + /** + * Returns true iff this binary is released--that is, a + * binary built with a release label. + */ + public boolean isReleasedBlaze() { + String buildLabel = buildData.get(BUILD_LABEL); + return buildLabel != null && buildLabel.length() > 0; + } + + /** + * Returns the release label, if any, or "development version". + */ + public String getReleaseName() { + String buildLabel = buildData.get(BUILD_LABEL); + return (buildLabel != null && buildLabel.length() > 0) + ? "release " + buildLabel + : "development version"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoEvent.java new file mode 100644 index 0000000..59d1514 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoEvent.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +/** + * This event is fired once build info data is available. + */ +public final class BuildInfoEvent { + private final Map<String, String> buildInfoMap; + + /** + * Construct the event from a map. + */ + public BuildInfoEvent(Map<String, String> buildInfo) { + buildInfoMap = ImmutableMap.copyOf(buildInfo); + } + + /** + * Return immutable map populated with build info key/value pairs. + */ + public Map<String, String> getBuildInfoMap() { + return buildInfoMap; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoHelper.java b/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoHelper.java new file mode 100644 index 0000000..755df9f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/BuildInfoHelper.java
@@ -0,0 +1,31 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.actions.AbstractActionOwner; +import com.google.devtools.build.lib.actions.ActionOwner; + +// TODO(bazel-team): move BUILD_INFO_ACTION_OWNER somewhere else and remove this class. +/** + * Helper class for the CompatibleWriteBuildInfoAction, which holds the + * methods for generating build information. + * Abstracted away to allow non-action code to also generate build info under + * --nobuild or --check_up_to_date. + */ +public abstract class BuildInfoHelper { + /** ActionOwner for BuildInfoActions. */ + public static final ActionOwner BUILD_INFO_ACTION_OWNER = + AbstractActionOwner.SYSTEM_ACTION_OWNER; +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java new file mode 100644 index 0000000..052d0a2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java
@@ -0,0 +1,1056 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimaps; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.PackageRootResolver; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency; +import com.google.devtools.build.lib.analysis.ExtraActionArtifactsProvider.ExtraArtifactSet; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; +import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.DelegatingEventHandler; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.events.WarningsAsErrorsEventHandler; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.PackageSpecification; +import com.google.devtools.build.lib.packages.RawAttributeMapper; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner.LoadingResult; +import com.google.devtools.build.lib.pkgcache.PackageManager; +import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory; +import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; +import com.google.devtools.build.lib.skyframe.CoverageReportValue; +import com.google.devtools.build.lib.skyframe.SkyframeBuildView; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.RegexFilter; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +/** + * <p>The BuildView presents a semantically-consistent and transitively-closed + * dependency graph for some set of packages. + * + * <h2>Package design</h2> + * + * <p>This package contains the Blaze dependency analysis framework (aka + * "analysis phase"). The goal of this code is to perform semantic analysis of + * all of the build targets required for a given build, to report + * errors/warnings for any problems in the input, and to construct an "action + * graph" (see {@code lib.actions} package) correctly representing the work to + * be done during the execution phase of the build. + * + * <p><b>Configurations</b> the inputs to a build come from two sources: the + * intrinsic inputs, specified in the BUILD file, are called <em>targets</em>. + * The environmental inputs, coming from the build tool, the command-line, or + * configuration files, are called the <em>configuration</em>. Only when a + * target and a configuration are combined is there sufficient information to + * perform a build. </p> + * + * <p>Targets are implemented by the {@link Target} hierarchy in the {@code + * lib.packages} code. Configurations are implemented by {@link + * BuildConfiguration}. The pair of these together is represented by an + * instance of class {@link ConfiguredTarget}; this is the root of a hierarchy + * with different implementations for each kind of target: source file, derived + * file, rules, etc. + * + * <p>The framework code in this package (as opposed to its subpackages) is + * responsible for constructing the {@code ConfiguredTarget} graph for a given + * target and configuration, taking care of such issues as: + * <ul> + * <li>caching common subgraphs. + * <li>detecting and reporting cycles. + * <li>correct propagation of errors through the graph. + * <li>reporting universal errors, such as dependencies from production code + * to tests, or to experimental branches. + * <li>capturing and replaying errors. + * <li>maintaining the graph from one build to the next to + * avoid unnecessary recomputation. + * <li>checking software licenses. + * </ul> + * + * <p>See also {@link ConfiguredTarget} which documents some important + * invariants. + */ +public class BuildView { + + /** + * Options that affect the <i>mechanism</i> of analysis. These are distinct from {@link + * com.google.devtools.build.lib.analysis.config.BuildOptions}, which affect the <i>value</i> + * of a BuildConfiguration. + */ + public static class Options extends OptionsBase { + + @Option(name = "keep_going", + abbrev = 'k', + defaultValue = "false", + category = "strategy", + help = "Continue as much as possible after an error. While the " + + "target that failed, and those that depend on it, cannot be " + + "analyzed (or built), the other prerequisites of these " + + "targets can be analyzed (or built) all the same.") + public boolean keepGoing; + + @Option(name = "analysis_warnings_as_errors", + defaultValue = "false", + category = "strategy", + help = "Treat visible analysis warnings as errors.") + public boolean analysisWarningsAsErrors; + + @Option(name = "discard_analysis_cache", + defaultValue = "false", + category = "strategy", + help = "Discard the analysis cache immediately after the analysis phase completes. " + + "Reduces memory usage by ~10%, but makes further incremental builds slower.") + public boolean discardAnalysisCache; + + @Option(name = "keep_forward_graph", + deprecationWarning = "keep_forward_graph is now a no-op and will be removed in an " + + "upcoming Blaze release", + defaultValue = "false", + category = "undocumented", + help = "Cache the forward action graph across builds for faster " + + "incremental rebuilds. May slightly increase memory while Blaze " + + "server is idle." + ) + public boolean keepForwardGraph; + + @Option(name = "experimental_extra_action_filter", + defaultValue = "", + category = "experimental", + converter = RegexFilter.RegexFilterConverter.class, + help = "Filters set of targets to schedule extra_actions for.") + public RegexFilter extraActionFilter; + + @Option(name = "experimental_extra_action_top_level_only", + defaultValue = "false", + category = "experimental", + help = "Only schedules extra_actions for top level targets.") + public boolean extraActionTopLevelOnly; + + @Option(name = "version_window_for_dirty_node_gc", + defaultValue = "0", + category = "undocumented", + help = "Nodes that have been dirty for more than this many versions will be deleted" + + " from the graph upon the next update. Values must be non-negative long integers," + + " or -1 indicating the maximum possible window.") + public long versionWindowForDirtyNodeGc; + } + + private static Logger LOG = Logger.getLogger(BuildView.class.getName()); + + private final BlazeDirectories directories; + + private final SkyframeExecutor skyframeExecutor; + private final SkyframeBuildView skyframeBuildView; + + private final PackageManager packageManager; + + private final BinTools binTools; + + private BuildConfigurationCollection configurations; + + private final ConfiguredRuleClassProvider ruleClassProvider; + + private final ArtifactFactory artifactFactory; + + /** + * A factory class to create the coverage report action. May be null. + */ + @Nullable private final CoverageReportActionFactory coverageReportActionFactory; + + /** + * A union of package roots of all previous incremental analysis results. This is used to detect + * changes of package roots between incremental analysis instances. + */ + private final Map<PackageIdentifier, Path> cumulativePackageRoots = new HashMap<>(); + + /** + * Used only for testing that we clear Skyframe caches correctly. + * TODO(bazel-team): Remove this once we get rid of legacy Skyframe synchronization. + */ + private boolean skyframeCacheWasInvalidated = false; + + /** + * If the last build was executed with {@code Options#discard_analysis_cache} and we are not + * running Skyframe full, we should clear the legacy data since it is out-of-sync. + */ + private boolean skyframeAnalysisWasDiscarded = false; + + @VisibleForTesting + public Set<SkyKey> getSkyframeEvaluatedTargetKeysForTesting() { + return skyframeBuildView.getEvaluatedTargetKeys(); + } + + /** The number of targets freshly evaluated in the last analysis run. */ + public int getTargetsVisited() { + return skyframeBuildView.getEvaluatedTargetKeys().size(); + } + + /** + * Returns true iff Skyframe was invalidated during the analysis phase. + * TODO(bazel-team): Remove this once we do not need to keep legacy in sync with Skyframe. + */ + @VisibleForTesting + boolean wasSkyframeCacheInvalidatedDuringAnalysis() { + return skyframeCacheWasInvalidated; + } + + public BuildView(BlazeDirectories directories, PackageManager packageManager, + ConfiguredRuleClassProvider ruleClassProvider, + SkyframeExecutor skyframeExecutor, + BinTools binTools, CoverageReportActionFactory coverageReportActionFactory) { + this.directories = directories; + this.packageManager = packageManager; + this.binTools = binTools; + this.coverageReportActionFactory = coverageReportActionFactory; + this.artifactFactory = new ArtifactFactory(directories.getExecRoot()); + this.ruleClassProvider = ruleClassProvider; + this.skyframeExecutor = Preconditions.checkNotNull(skyframeExecutor); + this.skyframeBuildView = + new SkyframeBuildView( + new ConfiguredTargetFactory(ruleClassProvider), + artifactFactory, + skyframeExecutor, + new Runnable() { + @Override + public void run() { + clear(); + } + }, + binTools); + skyframeExecutor.setSkyframeBuildView(skyframeBuildView); + } + + /** Returns the action graph. */ + public ActionGraph getActionGraph() { + return new ActionGraph() { + @Override + public Action getGeneratingAction(Artifact artifact) { + return skyframeExecutor.getGeneratingAction(artifact); + } + }; + } + + /** + * Returns whether the given configured target has errors. + */ + @VisibleForTesting + public boolean hasErrors(ConfiguredTarget configuredTarget) { + return configuredTarget == null; + } + + /** + * Sets the configurations. Not thread-safe. DO NOT CALL except from tests! + */ + @VisibleForTesting + void setConfigurationsForTesting(BuildConfigurationCollection configurations) { + this.configurations = configurations; + } + + public BuildConfigurationCollection getConfigurationCollection() { + return configurations; + } + + /** + * Clear the graphs of ConfiguredTargets and Artifacts. + */ + @VisibleForTesting + public void clear() { + cumulativePackageRoots.clear(); + artifactFactory.clear(); + } + + public ArtifactFactory getArtifactFactory() { + return artifactFactory; + } + + @VisibleForTesting + WorkspaceStatusAction getLastWorkspaceBuildInfoActionForTesting() { + return skyframeExecutor.getLastWorkspaceStatusActionForTesting(); + } + + /** + * Returns a corresponding ConfiguredTarget, if one exists; otherwise throws an {@link + * NoSuchConfiguredTargetException}. + */ + @ThreadSafe + private ConfiguredTarget getConfiguredTarget(Target target, BuildConfiguration config) + throws NoSuchConfiguredTargetException { + ConfiguredTarget result = + getExistingConfiguredTarget(target.getLabel(), config); + if (result == null) { + throw new NoSuchConfiguredTargetException(target.getLabel(), config); + } + return result; + } + + /** + * Obtains a {@link ConfiguredTarget} given a {@code label}, by delegating + * to the package cache and + * {@link #getConfiguredTarget(Target, BuildConfiguration)}. + */ + public ConfiguredTarget getConfiguredTarget(Label label, BuildConfiguration config) + throws NoSuchPackageException, NoSuchTargetException, NoSuchConfiguredTargetException { + return getConfiguredTarget(packageManager.getLoadedTarget(label), config); + } + + public Iterable<ConfiguredTarget> getDirectPrerequisites(ConfiguredTarget ct) { + return getDirectPrerequisites(ct, null); + } + + public Iterable<ConfiguredTarget> getDirectPrerequisites(ConfiguredTarget ct, + @Nullable final LoadingCache<Label, Target> targetCache) { + if (!(ct.getTarget() instanceof Rule)) { + return ImmutableList.of(); + } + + class SilentDependencyResolver extends DependencyResolver { + @Override + protected void invalidVisibilityReferenceHook(TargetAndConfiguration node, Label label) { + // The error must have been reported already during analysis. + } + + @Override + protected void invalidPackageGroupReferenceHook(TargetAndConfiguration node, Label label) { + // The error must have been reported already during analysis. + } + + @Override + protected Target getTarget(Label label) throws NoSuchThingException { + if (targetCache == null) { + return packageManager.getLoadedTarget(label); + } + + try { + return targetCache.get(label); + } catch (ExecutionException e) { + // All lookups should succeed because we should not be looking up any targets in error. + throw new IllegalStateException(e); + } + } + } + + DependencyResolver dependencyResolver = new SilentDependencyResolver(); + TargetAndConfiguration ctgNode = + new TargetAndConfiguration(ct.getTarget(), ct.getConfiguration()); + return skyframeExecutor.getConfiguredTargets( + dependencyResolver.dependentNodes(ctgNode, getConfigurableAttributeKeys(ctgNode))); + } + + /** + * Returns ConfigMatchingProvider instances corresponding to the configurable attribute keys + * present in this rule's attributes. + */ + private Set<ConfigMatchingProvider> getConfigurableAttributeKeys(TargetAndConfiguration ctg) { + if (!(ctg.getTarget() instanceof Rule)) { + return ImmutableSet.of(); + } + Rule rule = (Rule) ctg.getTarget(); + ImmutableSet.Builder<ConfigMatchingProvider> keys = ImmutableSet.builder(); + RawAttributeMapper mapper = RawAttributeMapper.of(rule); + for (Attribute attribute : rule.getAttributes()) { + for (Label label : mapper.getConfigurabilityKeys(attribute.getName(), attribute.getType())) { + if (Type.Selector.isReservedLabel(label)) { + continue; + } + try { + ConfiguredTarget ct = getConfiguredTarget(label, ctg.getConfiguration()); + keys.add(Preconditions.checkNotNull(ct.getProvider(ConfigMatchingProvider.class))); + } catch (NoSuchPackageException e) { + // All lookups should succeed because we should not be looking up any targets in error. + throw new IllegalStateException(e); + } catch (NoSuchTargetException e) { + // All lookups should succeed because we should not be looking up any targets in error. + throw new IllegalStateException(e); + } catch (NoSuchConfiguredTargetException e) { + // All lookups should succeed because we should not be looking up any targets in error. + throw new IllegalStateException(e); + } + } + } + return keys.build(); + } + + public TransitiveInfoCollection getGeneratingRule(OutputFileConfiguredTarget target) { + return target.getGeneratingRule(); + } + + @Override + public int hashCode() { + throw new UnsupportedOperationException(); // avoid nondeterminism + } + + /** + * Return value for {@link BuildView#update} and {@code BuildTool.prepareToBuild}. + */ + public static final class AnalysisResult { + + public static final AnalysisResult EMPTY = new AnalysisResult( + ImmutableList.<ConfiguredTarget>of(), null, null, null, + ImmutableList.<Artifact>of(), + ImmutableList.<ConfiguredTarget>of(), + ImmutableList.<ConfiguredTarget>of(), + null); + + private final ImmutableList<ConfiguredTarget> targetsToBuild; + @Nullable private final ImmutableList<ConfiguredTarget> targetsToTest; + @Nullable private final String error; + private final ActionGraph actionGraph; + private final ImmutableSet<Artifact> artifactsToBuild; + private final ImmutableSet<ConfiguredTarget> parallelTests; + private final ImmutableSet<ConfiguredTarget> exclusiveTests; + @Nullable private final TopLevelArtifactContext topLevelContext; + + private AnalysisResult( + Collection<ConfiguredTarget> targetsToBuild, Collection<ConfiguredTarget> targetsToTest, + @Nullable String error, ActionGraph actionGraph, + Collection<Artifact> artifactsToBuild, Collection<ConfiguredTarget> parallelTests, + Collection<ConfiguredTarget> exclusiveTests, TopLevelArtifactContext topLevelContext) { + this.targetsToBuild = ImmutableList.copyOf(targetsToBuild); + this.targetsToTest = targetsToTest == null ? null : ImmutableList.copyOf(targetsToTest); + this.error = error; + this.actionGraph = actionGraph; + this.artifactsToBuild = ImmutableSet.copyOf(artifactsToBuild); + this.parallelTests = ImmutableSet.copyOf(parallelTests); + this.exclusiveTests = ImmutableSet.copyOf(exclusiveTests); + this.topLevelContext = topLevelContext; + } + + /** + * Returns configured targets to build. + */ + public Collection<ConfiguredTarget> getTargetsToBuild() { + return targetsToBuild; + } + + /** + * Returns the configured targets to run as tests, or {@code null} if testing was not + * requested (e.g. "build" command rather than "test" command). + */ + @Nullable + public Collection<ConfiguredTarget> getTargetsToTest() { + return targetsToTest; + } + + public ImmutableSet<Artifact> getAdditionalArtifactsToBuild() { + return artifactsToBuild; + } + + public ImmutableSet<ConfiguredTarget> getExclusiveTests() { + return exclusiveTests; + } + + public ImmutableSet<ConfiguredTarget> getParallelTests() { + return parallelTests; + } + + /** + * Returns an error description (if any). + */ + @Nullable public String getError() { + return error; + } + + /** + * Returns the action graph. + */ + public ActionGraph getActionGraph() { + return actionGraph; + } + + public TopLevelArtifactContext getTopLevelContext() { + return topLevelContext; + } + } + + + /** + * Returns the collection of configured targets corresponding to any of the provided targets. + */ + @VisibleForTesting + static Iterable<? extends ConfiguredTarget> filterTestsByTargets( + Collection<? extends ConfiguredTarget> targets, + final Set<? extends Target> allowedTargets) { + return Iterables.filter(targets, + new Predicate<ConfiguredTarget>() { + @Override + public boolean apply(ConfiguredTarget rule) { + return allowedTargets.contains(rule.getTarget()); + } + }); + } + + private void prepareToBuild(PackageRootResolver resolver) throws ViewCreationFailedException { + for (BuildConfiguration config : configurations.getTargetConfigurations()) { + config.prepareToBuild(directories.getExecRoot(), getArtifactFactory(), resolver); + } + } + + @ThreadCompatible + public AnalysisResult update(LoadingResult loadingResult, + BuildConfigurationCollection configurations, Options viewOptions, + TopLevelArtifactContext topLevelOptions, EventHandler eventHandler, EventBus eventBus) + throws ViewCreationFailedException, InterruptedException { + + // Detect errors during analysis and don't attempt a build. + // + // (Errors reported during the previous step, package loading, that do + // not cause the visitation of the transitive closure to abort, are + // recoverable. For example, an error encountered while evaluating an + // irrelevant rule in a visited package causes an error to be reported, + // but visitation still succeeds.) + ErrorCollector errorCollector = null; + if (!viewOptions.keepGoing) { + eventHandler = errorCollector = new ErrorCollector(eventHandler); + } + + // Treat analysis warnings as errors, to enable strict builds. + // + // Warnings reported during analysis are converted to errors, ultimately + // triggering failure. This check needs to be added after the keep-going check + // above so that it is invoked first (FIFO eventHandler chain). This way, detected + // warnings are converted to errors first, and then the proper error handling + // logic is invoked. + WarningsAsErrorsEventHandler warningsHandler = null; + if (viewOptions.analysisWarningsAsErrors) { + eventHandler = warningsHandler = new WarningsAsErrorsEventHandler(eventHandler); + } + + skyframeBuildView.setWarningListener(eventHandler); + skyframeExecutor.setErrorEventListener(eventHandler); + + LOG.info("Starting analysis"); + pollInterruptedStatus(); + + skyframeBuildView.resetEvaluatedConfiguredTargetKeysSet(); + + Collection<Target> targets = loadingResult.getTargets(); + eventBus.post(new AnalysisPhaseStartedEvent(targets)); + + skyframeCacheWasInvalidated = false; + // Clear all cached ConfiguredTargets on configuration change. We need to do this explicitly + // because we need to make sure that the legacy action graph does not contain multiple actions + // with different versions of the same (target/host/etc.) configuration. + // In the future the action graph will be probably be keyed by configurations, which should + // obviate the need for this workaround. + // + // Also if --discard_analysis_cache was used in the last build we want to clear the legacy + // data. + if ((this.configurations != null && !configurations.equals(this.configurations)) + || skyframeAnalysisWasDiscarded) { + skyframeExecutor.dropConfiguredTargets(); + skyframeCacheWasInvalidated = true; + clear(); + } + skyframeAnalysisWasDiscarded = false; + ImmutableMap<PackageIdentifier, Path> packageRoots = loadingResult.getPackageRoots(); + + if (buildHasIncompatiblePackageRoots(packageRoots)) { + // When a package root changes source artifacts with the new root will be created, but we + // cannot be sure that there are no references remaining to the corresponding artifacts + // with the old root. To avoid that scenario, the analysis cache is simply dropped when + // a package root change is detected. + LOG.info("Discarding analysis cache: package roots have changed."); + + skyframeExecutor.dropConfiguredTargets(); + skyframeCacheWasInvalidated = true; + clear(); + } + cumulativePackageRoots.putAll(packageRoots); + this.configurations = configurations; + setArtifactRoots(packageRoots); + + // Determine the configurations. + List<TargetAndConfiguration> nodes = nodesForTargets(targets); + + List<ConfiguredTargetKey> targetSpecs = + Lists.transform(nodes, new Function<TargetAndConfiguration, ConfiguredTargetKey>() { + @Override + public ConfiguredTargetKey apply(TargetAndConfiguration node) { + return new ConfiguredTargetKey(node.getLabel(), node.getConfiguration()); + } + }); + + prepareToBuild(new SkyframePackageRootResolver(skyframeExecutor)); + skyframeBuildView.setWarningListener(warningsHandler); + skyframeExecutor.injectWorkspaceStatusData(); + Collection<ConfiguredTarget> configuredTargets; + try { + configuredTargets = skyframeBuildView.configureTargets( + targetSpecs, eventBus, viewOptions.keepGoing); + } finally { + skyframeBuildView.clearInvalidatedConfiguredTargets(); + } + + int numTargetsToAnalyze = nodes.size(); + int numSuccessful = configuredTargets.size(); + boolean analysisSuccessful = (numSuccessful == numTargetsToAnalyze); + if (0 < numSuccessful && numSuccessful < numTargetsToAnalyze) { + String msg = String.format("Analysis succeeded for only %d of %d top-level targets", + numSuccessful, numTargetsToAnalyze); + eventHandler.handle(Event.info(msg)); + LOG.info(msg); + } + + postUpdateValidation(errorCollector, warningsHandler); + + AnalysisResult result = createResult(loadingResult, topLevelOptions, + viewOptions, configuredTargets, analysisSuccessful); + LOG.info("Finished analysis"); + return result; + } + + // Validates that the update has been done correctly + private void postUpdateValidation(ErrorCollector errorCollector, + WarningsAsErrorsEventHandler warningsHandler) throws ViewCreationFailedException { + if (warningsHandler != null && warningsHandler.warningsEncountered()) { + throw new ViewCreationFailedException("Warnings being treated as errors"); + } + + if (errorCollector != null && !errorCollector.getEvents().isEmpty()) { + // This assertion ensures that if any errors were reported during the + // initialization phase, the call to configureTargets will fail with a + // ViewCreationFailedException. Violation of this invariant leads to + // incorrect builds, because the fact that errors were encountered is not + // properly recorded in the view (i.e. the graph of configured targets). + // Rule errors must be reported via RuleConfiguredTarget.reportError, + // which causes the rule's hasErrors() flag to be set, and thus the + // hasErrors() flag of anything that depends on it transitively. If the + // toplevel rule hasErrors, then analysis is aborted and we do not + // proceed to the execution phase of a build. + // + // Reporting errors directly through the Reporter does not set the error + // flag, so analysis may succeed spuriously, allowing the execution + // phase to begin with unpredictable consequences. + // + // The use of errorCollector (rather than an ErrorSensor) makes the + // assertion failure messages more informative. + // Note we tolerate errors iff --keep-going, because some of the + // requested targets may have had problems during analysis, but that's ok. + StringBuilder message = new StringBuilder("Unexpected errors reported during analysis:"); + for (Event event : errorCollector.getEvents()) { + message.append('\n').append(event); + } + throw new IllegalStateException(message.toString()); + } + } + + /** + * Skyframe implementation of {@link PackageRootResolver}. + * + * <p> Note: you should not use this class inside of any SkyFunction. + */ + @VisibleForTesting + public static final class SkyframePackageRootResolver implements PackageRootResolver { + private final SkyframeExecutor executor; + + public SkyframePackageRootResolver(SkyframeExecutor executor) { + this.executor = executor; + } + + @Override + public Map<PathFragment, Root> findPackageRoots(Iterable<PathFragment> execPaths) { + return executor.getArtifactRoots(execPaths); + } + } + + private AnalysisResult createResult(LoadingResult loadingResult, + TopLevelArtifactContext topLevelOptions, BuildView.Options viewOptions, + Collection<ConfiguredTarget> configuredTargets, boolean analysisSuccessful) + throws InterruptedException { + Collection<Target> testsToRun = loadingResult.getTestsToRun(); + Collection<ConfiguredTarget> allTargetsToTest = null; + if (testsToRun != null) { + // Determine the subset of configured targets that are meant to be run as tests. + allTargetsToTest = Lists.newArrayList( + filterTestsByTargets(configuredTargets, Sets.newHashSet(testsToRun))); + } + + skyframeExecutor.injectTopLevelContext(topLevelOptions); + + Set<Artifact> artifactsToBuild = new HashSet<>(); + Set<ConfiguredTarget> parallelTests = new HashSet<>(); + Set<ConfiguredTarget> exclusiveTests = new HashSet<>(); + Collection<Artifact> buildInfoArtifacts; + buildInfoArtifacts = skyframeExecutor.getWorkspaceStatusArtifacts(); + // build-info and build-changelist. + Preconditions.checkState(buildInfoArtifacts.size() == 2, buildInfoArtifacts); + artifactsToBuild.addAll(buildInfoArtifacts); + addExtraActionsIfRequested(viewOptions, artifactsToBuild, configuredTargets); + if (coverageReportActionFactory != null) { + Action action = coverageReportActionFactory.createCoverageReportAction( + allTargetsToTest, + getBaselineCoverageArtifacts(configuredTargets), + artifactFactory, + CoverageReportValue.ARTIFACT_OWNER); + if (action != null) { + skyframeExecutor.injectCoverageReportData(action); + artifactsToBuild.addAll(action.getOutputs()); + } + } + + // Note that this must come last, so that the tests are scheduled after all artifacts are built. + scheduleTestsIfRequested(parallelTests, exclusiveTests, topLevelOptions, allTargetsToTest); + + String error = !loadingResult.hasLoadingError() + ? (analysisSuccessful + ? null + : "execution phase succeeded, but not all targets were analyzed") + : "execution phase succeeded, but there were loading phase errors"; + return new AnalysisResult(configuredTargets, allTargetsToTest, error, getActionGraph(), + artifactsToBuild, parallelTests, exclusiveTests, topLevelOptions); + } + + private static ImmutableSet<Artifact> getBaselineCoverageArtifacts( + Collection<ConfiguredTarget> configuredTargets) { + Set<Artifact> baselineCoverageArtifacts = Sets.newHashSet(); + for (ConfiguredTarget target : configuredTargets) { + BaselineCoverageArtifactsProvider provider = + target.getProvider(BaselineCoverageArtifactsProvider.class); + if (provider != null) { + baselineCoverageArtifacts.addAll(provider.getBaselineCoverageArtifacts()); + } + } + return ImmutableSet.copyOf(baselineCoverageArtifacts); + } + + private void addExtraActionsIfRequested(BuildView.Options viewOptions, + Set<Artifact> artifactsToBuild, Iterable<ConfiguredTarget> topLevelTargets) { + NestedSetBuilder<ExtraArtifactSet> builder = NestedSetBuilder.stableOrder(); + for (ConfiguredTarget topLevel : topLevelTargets) { + ExtraActionArtifactsProvider provider = topLevel.getProvider( + ExtraActionArtifactsProvider.class); + if (provider != null) { + if (viewOptions.extraActionTopLevelOnly) { + builder.add(ExtraArtifactSet.of(topLevel.getLabel(), provider.getExtraActionArtifacts())); + } else { + builder.addTransitive(provider.getTransitiveExtraActionArtifacts()); + } + } + } + + RegexFilter filter = viewOptions.extraActionFilter; + for (ExtraArtifactSet set : builder.build()) { + boolean filterMatches = filter == null || filter.isIncluded(set.getLabel().toString()); + if (filterMatches) { + Iterables.addAll(artifactsToBuild, set.getArtifacts()); + } + } + } + + private static void scheduleTestsIfRequested(Collection<ConfiguredTarget> targetsToTest, + Collection<ConfiguredTarget> targetsToTestExclusive, TopLevelArtifactContext topLevelOptions, + Collection<ConfiguredTarget> allTestTargets) { + if (!topLevelOptions.compileOnly() && !topLevelOptions.compilationPrerequisitesOnly() + && allTestTargets != null) { + scheduleTests(targetsToTest, targetsToTestExclusive, allTestTargets, + topLevelOptions.runTestsExclusively()); + } + } + + + /** + * Returns set of artifacts representing test results, writing into targetsToTest and + * targetsToTestExclusive. + */ + private static void scheduleTests(Collection<ConfiguredTarget> targetsToTest, + Collection<ConfiguredTarget> targetsToTestExclusive, + Collection<ConfiguredTarget> allTestTargets, + boolean isExclusive) { + for (ConfiguredTarget target : allTestTargets) { + if (target.getTarget() instanceof Rule) { + boolean exclusive = + isExclusive || TargetUtils.isExclusiveTestRule((Rule) target.getTarget()); + Collection<ConfiguredTarget> testCollection = exclusive + ? targetsToTestExclusive + : targetsToTest; + testCollection.add(target); + } + } + } + + @VisibleForTesting + List<TargetAndConfiguration> nodesForTargets(Collection<Target> targets) { + // We use a hash set here to remove duplicate nodes; this can happen for input files and package + // groups. + LinkedHashSet<TargetAndConfiguration> nodes = new LinkedHashSet<>(targets.size()); + for (BuildConfiguration config : configurations.getTargetConfigurations()) { + for (Target target : targets) { + nodes.add(new TargetAndConfiguration(target, + BuildConfigurationCollection.configureTopLevelTarget(config, target))); + } + } + return ImmutableList.copyOf(nodes); + } + + /** + * Detects when a package root changes between instances of incremental analysis. + * + * <p>This case is currently problematic for incremental analysis because when a package root + * changes, source artifacts with the new root will be created, but we can not be sure that there + * are no references remaining to the corresponding artifacts with the old root. + */ + private boolean buildHasIncompatiblePackageRoots(Map<PackageIdentifier, Path> packageRoots) { + for (Map.Entry<PackageIdentifier, Path> entry : packageRoots.entrySet()) { + Path prevRoot = cumulativePackageRoots.get(entry.getKey()); + if (prevRoot != null && !entry.getValue().equals(prevRoot)) { + return true; + } + } + return false; + } + + /** + * Returns an existing ConfiguredTarget for the specified target and + * configuration, or null if none exists. No validity check is done. + */ + @ThreadSafe + public ConfiguredTarget getExistingConfiguredTarget(Target target, BuildConfiguration config) { + return getExistingConfiguredTarget(target.getLabel(), config); + } + + /** + * Returns an existing ConfiguredTarget for the specified node, or null if none exists. No + * validity check is done. + */ + @ThreadSafe + private ConfiguredTarget getExistingConfiguredTarget( + Label label, BuildConfiguration configuration) { + return Iterables.getFirst( + skyframeExecutor.getConfiguredTargets( + ImmutableList.of(new Dependency(label, configuration))), + null); + } + + @VisibleForTesting + ListMultimap<Attribute, ConfiguredTarget> getPrerequisiteMapForTesting(ConfiguredTarget target) { + DependencyResolver resolver = new DependencyResolver() { + @Override + protected void invalidVisibilityReferenceHook(TargetAndConfiguration node, Label label) { + throw new RuntimeException("bad visibility on " + label + " during testing unexpected"); + } + + @Override + protected void invalidPackageGroupReferenceHook(TargetAndConfiguration node, Label label) { + throw new RuntimeException("bad package group on " + label + " during testing unexpected"); + } + + @Override + protected Target getTarget(Label label) throws NoSuchThingException { + return packageManager.getLoadedTarget(label); + } + }; + TargetAndConfiguration ctNode = new TargetAndConfiguration(target); + ListMultimap<Attribute, Dependency> depNodeNames; + try { + depNodeNames = resolver.dependentNodeMap(ctNode, null, getConfigurableAttributeKeys(ctNode)); + } catch (EvalException e) { + throw new IllegalStateException(e); + } + + final Map<LabelAndConfiguration, ConfiguredTarget> depMap = new HashMap<>(); + for (ConfiguredTarget dep : skyframeExecutor.getConfiguredTargets(depNodeNames.values())) { + depMap.put(LabelAndConfiguration.of(dep.getLabel(), dep.getConfiguration()), dep); + } + + return Multimaps.transformValues(depNodeNames, new Function<Dependency, ConfiguredTarget>() { + @Override + public ConfiguredTarget apply(Dependency depName) { + return depMap.get(LabelAndConfiguration.of(depName.getLabel(), + depName.getConfiguration())); + } + }); + } + + /** + * Sets the possible artifact roots in the artifact factory. This allows the + * factory to resolve paths with unknown roots to artifacts. + * <p> + * <em>Note: This must be called before any call to + * {@link #getConfiguredTarget(Label, BuildConfiguration)} + * </em> + */ + @VisibleForTesting // for BuildViewTestCase + void setArtifactRoots(ImmutableMap<PackageIdentifier, Path> packageRoots) { + Map<Path, Root> rootMap = new HashMap<>(); + Map<PackageIdentifier, Root> realPackageRoots = new HashMap<>(); + for (Map.Entry<PackageIdentifier, Path> entry : packageRoots.entrySet()) { + Root root = rootMap.get(entry.getValue()); + if (root == null) { + root = Root.asSourceRoot(entry.getValue()); + rootMap.put(entry.getValue(), root); + } + realPackageRoots.put(entry.getKey(), root); + } + // Source Artifact roots: + artifactFactory.setPackageRoots(realPackageRoots); + + // Derived Artifact roots: + ImmutableList.Builder<Root> roots = ImmutableList.builder(); + + // build-info.txt and friends; this root is not configuration specific. + roots.add(directories.getBuildDataDirectory()); + + // The roots for each configuration - duplicates are automatically removed in the call below. + for (BuildConfiguration cfg : configurations.getAllConfigurations()) { + roots.addAll(cfg.getRoots()); + } + + artifactFactory.setDerivedArtifactRoots(roots.build()); + } + + /** + * Returns a configured target for the specified target and configuration. + * This should only be called from test cases, and is needed, because + * plain {@link #getConfiguredTarget(Target, BuildConfiguration)} does not + * construct the configured target graph, and would thus fail if called from + * outside an update. + */ + @VisibleForTesting + public ConfiguredTarget getConfiguredTargetForTesting(Label label, BuildConfiguration config) + throws NoSuchPackageException, NoSuchTargetException { + return getConfiguredTargetForTesting(packageManager.getLoadedTarget(label), config); + } + + @VisibleForTesting + public ConfiguredTarget getConfiguredTargetForTesting(Target target, BuildConfiguration config) { + return skyframeExecutor.getConfiguredTargetForTesting(target.getLabel(), config); + } + + /** + * Returns a RuleContext which is the same as the original RuleContext of the target parameter. + */ + @VisibleForTesting + public RuleContext getRuleContextForTesting(ConfiguredTarget target, + StoredEventHandler eventHandler) { + BuildConfiguration config = target.getConfiguration(); + CachingAnalysisEnvironment analysisEnvironment = + new CachingAnalysisEnvironment(artifactFactory, + new ConfiguredTargetKey(target.getLabel(), config), + /*isSystemEnv=*/false, config.extendedSanityChecks(), eventHandler, + /*skyframeEnv=*/null, config.isActionsEnabled(), binTools); + RuleContext ruleContext = new RuleContext.Builder(analysisEnvironment, + (Rule) target.getTarget(), config, ruleClassProvider.getPrerequisiteValidator()) + .setVisibility(NestedSetBuilder.<PackageSpecification>create( + Order.STABLE_ORDER, PackageSpecification.EVERYTHING)) + .setPrerequisites(getPrerequisiteMapForTesting(target)) + .setConfigConditions(ImmutableSet.<ConfigMatchingProvider>of()) + .build(); + return ruleContext; + } + + /** + * Tests and clears the current thread's pending "interrupted" status, and + * throws InterruptedException iff it was set. + */ + protected final void pollInterruptedStatus() throws InterruptedException { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + } + + /** + * Drops the analysis cache. If building with Skyframe, targets in {@code topLevelTargets} may + * remain in the cache for use during the execution phase. + * + * @see BuildView.Options#discardAnalysisCache + */ + public void clearAnalysisCache(Collection<ConfiguredTarget> topLevelTargets) { + // TODO(bazel-team): Consider clearing packages too to save more memory. + skyframeAnalysisWasDiscarded = true; + skyframeExecutor.clearAnalysisCache(topLevelTargets); + } + + /******************************************************************** + * * + * 'blaze dump' related functions * + * * + ********************************************************************/ + + /** + * Collects and stores error events while also forwarding them to another eventHandler. + */ + public static class ErrorCollector extends DelegatingEventHandler { + private final List<Event> events; + + public ErrorCollector(EventHandler delegate) { + super(delegate); + this.events = Lists.newArrayList(); + } + + public List<Event> getEvents() { + return events; + } + + @Override + public void handle(Event e) { + super.handle(e); + if (e.getKind() == EventKind.ERROR) { + events.add(e); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java new file mode 100644 index 0000000..bc45ba3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java
@@ -0,0 +1,303 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.actions.MiddlemanFactory; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoCollection; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.skyframe.BuildInfoCollectionValue; +import com.google.devtools.build.lib.skyframe.WorkspaceStatusValue; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import javax.annotation.Nullable; + +/** + * The implementation of AnalysisEnvironment used for analysis. It tracks metadata for each + * configured target, such as the errors and warnings emitted by that target. It is intended that + * a separate instance is used for each configured target, so that these don't mix up. + */ +public class CachingAnalysisEnvironment implements AnalysisEnvironment { + private final ArtifactFactory artifactFactory; + + private final ArtifactOwner owner; + /** + * If this is the system analysis environment, then errors and warnings are directly reported + * to the global reporter, rather than stored, i.e., we don't track here whether there are any + * errors. + */ + private final boolean isSystemEnv; + private final boolean extendedSanityChecks; + + /** + * If false, no actions will be registered, they'll all be just dropped. + * + * <p>Usually, an analysis environment should register all actions. However, in some scenarios we + * analyze some targets twice, but the first one only serves the purpose of collecting information + * for the second analysis. In this case we don't register actions created by the first pass in + * order to avoid action conflicts. + */ + private final boolean allowRegisteringActions; + + private boolean enabled = true; + private MiddlemanFactory middlemanFactory; + private EventHandler errorEventListener; + private SkyFunction.Environment skyframeEnv; + private Map<Artifact, String> artifacts; + private final BinTools binTools; + + /** + * The list of actions registered by the configured target this analysis environment is + * responsible for. May get cleared out at the end of the analysis of said target. + */ + final List<Action> actions = new ArrayList<>(); + + public CachingAnalysisEnvironment(ArtifactFactory artifactFactory, + ArtifactOwner owner, boolean isSystemEnv, boolean extendedSanityChecks, + EventHandler errorEventListener, SkyFunction.Environment env, boolean allowRegisteringActions, + BinTools binTools) { + this.artifactFactory = artifactFactory; + this.owner = Preconditions.checkNotNull(owner); + this.isSystemEnv = isSystemEnv; + this.extendedSanityChecks = extendedSanityChecks; + this.errorEventListener = errorEventListener; + this.skyframeEnv = env; + this.allowRegisteringActions = allowRegisteringActions; + this.binTools = binTools; + middlemanFactory = new MiddlemanFactory(artifactFactory, this); + artifacts = new HashMap<>(); + } + + public void disable(Target target) { + if (!hasErrors() && allowRegisteringActions) { + verifyGeneratedArtifactHaveActions(target); + } + artifacts = null; + middlemanFactory = null; + enabled = false; + errorEventListener = null; + skyframeEnv = null; + } + + private static StringBuilder shortDescription(Action action) { + if (action == null) { + return new StringBuilder("null Action"); + } + return new StringBuilder() + .append(action.getClass().getName()) + .append(' ') + .append(action.getMnemonic()); + } + + /** + * Sanity checks that all generated artifacts have a generating action. + * @param target for error reporting + */ + public void verifyGeneratedArtifactHaveActions(Target target) { + Collection<String> orphanArtifacts = getOrphanArtifactMap().values(); + List<String> checkedActions = null; + if (!orphanArtifacts.isEmpty()) { + checkedActions = Lists.newArrayListWithCapacity(actions.size()); + for (Action action : actions) { + StringBuilder sb = shortDescription(action); + for (Artifact o : action.getOutputs()) { + sb.append("\n "); + sb.append(o.getExecPathString()); + } + checkedActions.add(sb.toString()); + } + throw new IllegalStateException( + String.format( + "%s %s : These artifacts miss a generating action:\n%s\n" + + "These actions we checked:\n%s\n", + target.getTargetKind(), target.getLabel(), + Joiner.on('\n').join(orphanArtifacts), Joiner.on('\n').join(checkedActions))); + } + } + + @Override + public ImmutableSet<Artifact> getOrphanArtifacts() { + return ImmutableSet.copyOf(getOrphanArtifactMap().keySet()); + } + + private Map<Artifact, String> getOrphanArtifactMap() { + // Construct this set to avoid poor performance under large --runs_per_test. + Set<Artifact> artifactsWithActions = new HashSet<>(); + for (Action action : actions) { + // Don't bother checking that every Artifact only appears once; that test is performed + // elsewhere (see #testNonUniqueOutputs in ActionListenerIntegrationTest). + artifactsWithActions.addAll(action.getOutputs()); + } + // The order of the artifacts.entrySet iteration is unspecified - we use a TreeMap here to + // guarantee that the return value of this method is deterministic. + Map<Artifact, String> orphanArtifacts = new TreeMap<>(); + for (Map.Entry<Artifact, String> entry : artifacts.entrySet()) { + Artifact a = entry.getKey(); + if (!a.isSourceArtifact() && !artifactsWithActions.contains(a)) { + orphanArtifacts.put(a, String.format("%s\n%s", + a.getExecPathString(), // uncovered artifact + entry.getValue())); // origin of creation + } + } + return orphanArtifacts; + } + + @Override + public EventHandler getEventHandler() { + return errorEventListener; + } + + @Override + public boolean hasErrors() { + // The system analysis environment never has errors. + if (isSystemEnv) { + return false; + } + Preconditions.checkState(enabled); + return ((StoredEventHandler) errorEventListener).hasErrors(); + } + + @Override + public MiddlemanFactory getMiddlemanFactory() { + Preconditions.checkState(enabled); + return middlemanFactory; + } + + /** + * Keeps track of artifacts. We check that all of them have an owner when the environment is + * sealed (disable()). For performance reasons we only track the originating stacktrace when + * running with --experimental_extended_sanity_checks. + */ + private Artifact trackArtifactAndOrigin(Artifact a, @Nullable Throwable e) { + if ((e != null) && !artifacts.containsKey(a)) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + artifacts.put(a, sw.toString()); + } else { + artifacts.put(a, "No origin, run with --experimental_extended_sanity_checks"); + } + return a; + } + + @Override + public Artifact getDerivedArtifact(PathFragment rootRelativePath, Root root) { + Preconditions.checkState(enabled); + return trackArtifactAndOrigin( + artifactFactory.getDerivedArtifact(rootRelativePath, root, getOwner()), + extendedSanityChecks ? new Throwable() : null); + } + + @Override + public Artifact getFilesetArtifact(PathFragment rootRelativePath, Root root) { + Preconditions.checkState(enabled); + return trackArtifactAndOrigin( + artifactFactory.getFilesetArtifact(rootRelativePath, root, getOwner()), + extendedSanityChecks ? new Throwable() : null); + } + + @Override + public Artifact getConstantMetadataArtifact(PathFragment rootRelativePath, Root root) { + return artifactFactory.getConstantMetadataArtifact(rootRelativePath, root, getOwner()); + } + + @Override + public Artifact getEmbeddedToolArtifact(String embeddedPath) { + Preconditions.checkState(enabled); + return binTools.getEmbeddedArtifact(embeddedPath, artifactFactory); + } + + @Override + public void registerAction(Action... actions) { + Preconditions.checkState(enabled); + if (allowRegisteringActions) { + for (Action action : actions) { + this.actions.add(action); + } + } + } + + @Override + public Action getLocalGeneratingAction(Artifact artifact) { + Preconditions.checkState(allowRegisteringActions); + for (Action action : actions) { + if (action.getOutputs().contains(artifact)) { + return action; + } + } + return null; + } + + @Override + public Collection<Action> getRegisteredActions() { + return Collections.unmodifiableCollection(actions); + } + + @Override + public SkyFunction.Environment getSkyframeEnv() { + return skyframeEnv; + } + + @Override + public Artifact getStableWorkspaceStatusArtifact() { + return ((WorkspaceStatusValue) skyframeEnv.getValue(WorkspaceStatusValue.SKY_KEY)) + .getStableArtifact(); + } + + @Override + public Artifact getVolatileWorkspaceStatusArtifact() { + return ((WorkspaceStatusValue) skyframeEnv.getValue(WorkspaceStatusValue.SKY_KEY)) + .getVolatileArtifact(); + } + + @Override + public ImmutableList<Artifact> getBuildInfo(RuleContext ruleContext, BuildInfoKey key) { + boolean stamp = AnalysisUtils.isStampingEnabled(ruleContext); + BuildInfoCollection collection = + ((BuildInfoCollectionValue) skyframeEnv.getValue(BuildInfoCollectionValue.key( + new BuildInfoCollectionValue.BuildInfoKeyAndConfig(key, ruleContext.getConfiguration())))) + .getCollection(); + return stamp ? collection.getStampedBuildInfo() : collection.getRedactedBuildInfo(); + } + + @Override + public ArtifactOwner getOwner() { + return owner; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/CommandHelper.java b/src/main/java/com/google/devtools/build/lib/analysis/CommandHelper.java new file mode 100644 index 0000000..2c05f0f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/CommandHelper.java
@@ -0,0 +1,307 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +/** + * Provides shared functionality for parameterized command-line launching + * e.g. {@link com.google.devtools.build.lib.view.genrule.GenRule} + * Also used by {@link com.google.devtools.build.lib.rules.extra.ExtraActionFactory}. + * + * Two largely independent separate sets of functionality are provided: + * 1- string interpolation for {@code $(location[s] ...)} and {@code $(MakeVariable)} + * 2- a utility to build potentially large command lines (presumably made of multiple commands), + * that if presumed too large for the kernel's taste can be dumped into a shell script + * that will contain the same commands, + * at which point the shell script is added to the list of inputs. + */ +@SkylarkModule(name = "command_helper", doc = "A helper class to create shell commands.") +public final class CommandHelper { + + /** + * Maximum total command-line length, in bytes, not counting "/bin/bash -c ". + * If the command is very long, then we write the command to a script file, + * to avoid overflowing any limits on command-line length. + * For short commands, we just use /bin/bash -c command. + */ + @VisibleForTesting + public static int maxCommandLength = 64000; + + /** + * A map of remote path prefixes and corresponding runfiles manifests for tools + * used by this rule. + */ + private final ImmutableMap<PathFragment, Artifact> remoteRunfileManifestMap; + + /** + * Use labelMap for heuristically expanding labels (does not include "outs") + * This is similar to heuristic location expansion in LocationExpander + * and should be kept in sync. + */ + private final ImmutableMap<Label, ImmutableCollection<Artifact>> labelMap; + + /** + * The ruleContext this helper works on + */ + private final RuleContext ruleContext; + + /** + * Output executable files from the 'tools' attribute. + */ + private final ImmutableList<Artifact> resolvedTools; + + /** + * Creates an {@link CommandHelper}. + * + * @param tools - Resolves set of tools into set of executable binaries. Populates manifests, + * remoteRunfiles and label map where required. + * @param labelMap - Adds files to set of known files of label. Used for resolving $(location) + * variables. + */ + public CommandHelper(RuleContext ruleContext, + Iterable<FilesToRunProvider> tools, + ImmutableMap<Label, Iterable<Artifact>> labelMap) { + this.ruleContext = ruleContext; + + ImmutableList.Builder<Artifact> resolvedToolsBuilder = ImmutableList.builder(); + ImmutableMap.Builder<PathFragment, Artifact> remoteRunfileManifestBuilder = + ImmutableMap.builder(); + Map<Label, Collection<Artifact>> tempLabelMap = new HashMap<>(); + + for (Map.Entry<Label, Iterable<Artifact>> entry : labelMap.entrySet()) { + Iterables.addAll(mapGet(tempLabelMap, entry.getKey()), entry.getValue()); + } + + for (FilesToRunProvider tool : tools) { // (Note: host configuration) + Label label = tool.getLabel(); + Collection<Artifact> files = tool.getFilesToRun(); + resolvedToolsBuilder.addAll(files); + Artifact executableArtifact = tool.getExecutable(); + // If the label has an executable artifact add that to the multimaps. + if (executableArtifact != null) { + mapGet(tempLabelMap, label).add(executableArtifact); + // Also send the runfiles when running remotely. + Artifact runfilesManifest = tool.getRunfilesManifest(); + if (runfilesManifest != null) { + remoteRunfileManifestBuilder.put( + BaseSpawn.runfilesForFragment(executableArtifact.getExecPath()), runfilesManifest); + } + } else { + // Map all depArtifacts to the respective label using the multimaps. + Iterables.addAll(mapGet(tempLabelMap, label), files); + } + } + + this.resolvedTools = resolvedToolsBuilder.build(); + this.remoteRunfileManifestMap = remoteRunfileManifestBuilder.build(); + ImmutableMap.Builder<Label, ImmutableCollection<Artifact>> labelMapBuilder = + ImmutableMap.builder(); + for (Entry<Label, Collection<Artifact>> entry : tempLabelMap.entrySet()) { + labelMapBuilder.put(entry.getKey(), ImmutableList.copyOf(entry.getValue())); + } + this.labelMap = labelMapBuilder.build(); + } + + @SkylarkCallable(name = "resolved_tools", doc = "", structField = true) + public List<Artifact> getResolvedTools() { + return resolvedTools; + } + + @SkylarkCallable(name = "runfiles_manifests", doc = "", structField = true) + public ImmutableMap<PathFragment, Artifact> getRemoteRunfileManifestMap() { + return remoteRunfileManifestMap; + } + + // Returns the value in the specified corresponding to 'key', creating and + // inserting an empty container if absent. We use Map not Multimap because + // we need to distinguish the cases of "empty value" and "absent key". + private static Collection<Artifact> mapGet(Map<Label, Collection<Artifact>> map, Label key) { + Collection<Artifact> values = map.get(key); + if (values == null) { + // We use sets not lists, because it's conceivable that the same artifact + // could appear twice, e.g. in "srcs" and "deps". + values = Sets.newHashSet(); + map.put(key, values); + } + return values; + } + + /** + * Resolves the 'cmd' attribute, and expands known locations for $(location) + * variables. + */ + @SkylarkCallable(doc = "") + public String resolveCommandAndExpandLabels(Boolean supportLegacyExpansion, + Boolean allowDataInLabel) { + String command = ruleContext.attributes().get("cmd", Type.STRING); + command = new LocationExpander(ruleContext, allowDataInLabel).expand("cmd", command); + + if (supportLegacyExpansion) { + command = expandLabels(command, labelMap); + } + return command; + } + + /** + * Expands labels occurring in the string "expr" in the rule 'cmd'. + * Each label must be valid, be a declared prerequisite, and expand to a + * unique path. + * + * <p>If the expansion fails, an attribute error is reported and the original + * expression is returned. + */ + private <T extends Iterable<Artifact>> String expandLabels(String expr, Map<Label, T> labelMap) { + try { + return LabelExpander.expand(expr, labelMap, ruleContext.getLabel()); + } catch (LabelExpander.NotUniqueExpansionException nuee) { + ruleContext.attributeError("cmd", nuee.getMessage()); + return expr; + } + } + + private static Pair<List<String>, Artifact> buildCommandLineMaybeWithScriptFile( + RuleContext ruleContext, String command, String scriptPostFix) { + List<String> argv; + Artifact scriptFileArtifact = null; + if (command.length() <= maxCommandLength) { + argv = buildCommandLineSimpleArgv(ruleContext, command); + } else { + // Use script file. + scriptFileArtifact = buildCommandLineArtifact(ruleContext, command, scriptPostFix); + argv = buildCommandLineArgvWithArtifact(ruleContext, scriptFileArtifact); + } + return Pair.of(argv, scriptFileArtifact); + + } + + /** + * Builds the set of command-line arguments. Creates a bash script if the + * command line is longer than the allowed maximum {@link + * #maxCommandLength}. Fixes up the input artifact list with the + * created bash script when required. + * TODO(bazel-team): do away with the side effect on inputs (ugh). + */ + public static List<String> buildCommandLine(RuleContext ruleContext, + String command, NestedSetBuilder<Artifact> inputs, String scriptPostFix) { + Pair<List<String>, Artifact> argvAndScriptFile = + buildCommandLineMaybeWithScriptFile(ruleContext, command, scriptPostFix); + if (argvAndScriptFile.second != null) { + inputs.add(argvAndScriptFile.second); + } + return argvAndScriptFile.first; + } + + /** + * Builds the set of command-line arguments. Creates a bash script if the + * command line is longer than the allowed maximum {@link + * #maxCommandLength}. Fixes up the input artifact list with the + * created bash script when required. + * TODO(bazel-team): do away with the side effect on inputs (ugh). + */ + public static List<String> buildCommandLine(RuleContext ruleContext, + String command, List<Artifact> inputs, String scriptPostFix) { + Pair<List<String>, Artifact> argvAndScriptFile = + buildCommandLineMaybeWithScriptFile(ruleContext, command, scriptPostFix); + if (argvAndScriptFile.second != null) { + inputs.add(argvAndScriptFile.second); + } + return argvAndScriptFile.first; + } + + /** + * Builds the set of command-line arguments. Creates a bash script if the + * command line is longer than the allowed maximum {@link + * #maxCommandLength}. Fixes up the input artifact list with the + * created bash script when required. + * TODO(bazel-team): do away with the side effect on inputs (ugh). + */ + public static List<String> buildCommandLine(RuleContext ruleContext, + String command, ImmutableSet.Builder<Artifact> inputs, String scriptPostFix) { + Pair<List<String>, Artifact> argvAndScriptFile = + buildCommandLineMaybeWithScriptFile(ruleContext, command, scriptPostFix); + if (argvAndScriptFile.second != null) { + inputs.add(argvAndScriptFile.second); + } + return argvAndScriptFile.first; + } + + private static ImmutableList<String> buildCommandLineArgvWithArtifact(RuleContext ruleContext, + Artifact scriptFileArtifact) { + return ImmutableList.of( + ruleContext.getConfiguration().getShExecutable().getPathString(), + scriptFileArtifact.getExecPathString()); + } + + private static Artifact buildCommandLineArtifact(RuleContext ruleContext, String command, + String scriptPostFix) { + String scriptFileName = ruleContext.getTarget().getName() + scriptPostFix; + String scriptFileContents = "#!/bin/bash\n" + command; + Artifact scriptFileArtifact = FileWriteAction.createFile( + ruleContext, scriptFileName, scriptFileContents, /*executable=*/true); + return scriptFileArtifact; + } + + private static ImmutableList<String> buildCommandLineSimpleArgv(RuleContext ruleContext, + String command) { + return ImmutableList.of( + ruleContext.getConfiguration().getShExecutable().getPathString(), "-c", command); + } + + /** + * Builds the set of command-line arguments. Creates a bash script if the + * command line is longer than the allowed maximum {@link + * #maxCommandLength}. Fixes up the input artifact list with the + * created bash script when required. + */ + public List<String> buildCommandLine( + String command, NestedSetBuilder<Artifact> inputs, String scriptPostFix) { + return buildCommandLine(ruleContext, command, inputs, scriptPostFix); + } + + /** + * Builds the set of command-line arguments. Creates a bash script if the + * command line is longer than the allowed maximum {@link + * #maxCommandLength}. Fixes up the input artifact list with the + * created bash script when required. + */ + @SkylarkCallable(doc = "") + public List<String> buildCommandLine( + String command, List<Artifact> inputs, String scriptPostFix) { + return buildCommandLine(ruleContext, command, inputs, scriptPostFix); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/CompilationHelper.java b/src/main/java/com/google/devtools/build/lib/analysis/CompilationHelper.java new file mode 100644 index 0000000..23dd8d4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/CompilationHelper.java
@@ -0,0 +1,92 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.MiddlemanFactory; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; + +import java.util.List; + +/** + * A helper class for compilation helpers. + */ +public final class CompilationHelper { + /** + * Returns a middleman for all files to build for the given configured target. + * If multiple calls are made, then it returns the same artifact for + * configurations with the same internal directory. + * + * <p>The resulting middleman only aggregates the inputs and must be expanded + * before an action using it can be sent to a distributed execution-system. + */ + public static NestedSet<Artifact> getAggregatingMiddleman( + RuleContext ruleContext, String purpose, NestedSet<Artifact> filesToBuild) { + return NestedSetBuilder.wrap(Order.STABLE_ORDER, getMiddlemanInternal( + ruleContext.getAnalysisEnvironment(), ruleContext, ruleContext.getActionOwner(), purpose, + filesToBuild)); + } + + /** + * Internal implementation for getAggregatingMiddleman / getAggregatingMiddlemanWithSolibSymlinks. + */ + private static List<Artifact> getMiddlemanInternal(AnalysisEnvironment env, + RuleContext ruleContext, ActionOwner actionOwner, String purpose, + NestedSet<Artifact> filesToBuild) { + if (filesToBuild == null) { + return ImmutableList.of(); + } + MiddlemanFactory factory = env.getMiddlemanFactory(); + return ImmutableList.of(factory.createMiddlemanAllowMultiple( + env, actionOwner, purpose, filesToBuild, + ruleContext.getConfiguration().getMiddlemanDirectory())); + } + + // TODO(bazel-team): remove this duplicated code after the ConfiguredTarget migration + /** + * Returns a middleman for all files to build for the given configured target. + * If multiple calls are made, then it returns the same artifact for + * configurations with the same internal directory. + * + * <p>The resulting middleman only aggregates the inputs and must be expanded + * before an action using it can be sent to a distributed execution-system. + */ + public static NestedSet<Artifact> getAggregatingMiddleman( + RuleContext ruleContext, String purpose, TransitiveInfoCollection dep) { + return NestedSetBuilder.wrap(Order.STABLE_ORDER, getMiddlemanInternal( + ruleContext.getAnalysisEnvironment(), ruleContext, ruleContext.getActionOwner(), purpose, + dep)); + } + + /** + * Internal implementation for getAggregatingMiddleman / getAggregatingMiddlemanWithSolibSymlinks. + */ + private static List<Artifact> getMiddlemanInternal(AnalysisEnvironment env, + RuleContext ruleContext, ActionOwner actionOwner, String purpose, + TransitiveInfoCollection dep) { + if (dep == null) { + return ImmutableList.of(); + } + MiddlemanFactory factory = env.getMiddlemanFactory(); + Iterable<Artifact> artifacts = dep.getProvider(FileProvider.class).getFilesToBuild(); + return ImmutableList.of(factory.createMiddlemanAllowMultiple( + env, actionOwner, purpose, artifacts, + ruleContext.getConfiguration().getMiddlemanDirectory())); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/CompilationPrerequisitesProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/CompilationPrerequisitesProvider.java new file mode 100644 index 0000000..8c3a6ce --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/CompilationPrerequisitesProvider.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A provider for compilation prerequisites of a given target. + */ +@Immutable +public final class CompilationPrerequisitesProvider implements TransitiveInfoProvider { + + private final NestedSet<Artifact> compilationPrerequisites; + + public CompilationPrerequisitesProvider(NestedSet<Artifact> compilationPrerequisites) { + this.compilationPrerequisites = compilationPrerequisites; + } + + /** + * Returns compilation prerequisites (e.g., generated sources and headers) for a rule. + */ + public NestedSet<Artifact> getCompilationPrerequisites() { + return compilationPrerequisites; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationCollectionFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationCollectionFactory.java new file mode 100644 index 0000000..8ffac43 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationCollectionFactory.java
@@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.analysis.config.PackageProviderForConfigurations; +import com.google.devtools.build.lib.events.EventHandler; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * A factory for configuration collection creation. + */ +public interface ConfigurationCollectionFactory { + /** + * Creates the top-level configuration for a build. + * + * <p>Also it may create a set of BuildConfigurations and define a transition table over them. + * All configurations during a build should be accessible from this top-level configuration + * via configuration transitions. + * @param loadedPackageProvider the package provider + * @param buildOptions top-level build options representing the command-line + * @param clientEnv the system environment + * @param errorEventListener the event listener for errors + * @param performSanityCheck flag to signal about performing sanity check. Can be false only for + * tests in skyframe. Legacy mode uses loadedPackageProvider == null condition for this. + * @return the top-level configuration + * @throws InvalidConfigurationException + */ + @Nullable + public BuildConfiguration createConfigurations( + ConfigurationFactory configurationFactory, + PackageProviderForConfigurations loadedPackageProvider, + BuildOptions buildOptions, + Map<String, String> clientEnv, + EventHandler errorEventListener, + boolean performSanityCheck) throws InvalidConfigurationException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationMakeVariableContext.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationMakeVariableContext.java new file mode 100644 index 0000000..8c98f5f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationMakeVariableContext.java
@@ -0,0 +1,56 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.analysis.MakeVariableExpander.ExpansionException; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Package; + +import java.util.Map; + +/** + * Implements make variable expansion for make variables that depend on the + * configuration and the target (not on behavior of the + * {@link ConfiguredTarget} implementation) + */ +public class ConfigurationMakeVariableContext implements MakeVariableExpander.Context { + private final Package pkg; + private final Map<String, String> commandLineEnv; + private final Map<String, String> globalEnv; + private final String platform; + + public ConfigurationMakeVariableContext(Package pkg, BuildConfiguration configuration) { + this.pkg = pkg; + commandLineEnv = configuration.getCommandLineDefines(); + globalEnv = configuration.getGlobalMakeEnvironment(); + platform = configuration.getPlatformName(); + } + + @Override + public String lookupMakeVariable(String var) throws ExpansionException { + String value = commandLineEnv.get(var); + if (value == null) { + value = pkg.lookupMakeVariable(var, platform); + } + if (value == null) { + value = globalEnv.get(var); + } + if (value == null) { + throw new MakeVariableExpander.ExpansionException("$(" + var + ") not defined"); + } + + return value; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationsCreatedEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationsCreatedEvent.java new file mode 100644 index 0000000..8b0621a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationsCreatedEvent.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; + +/** + * This event is fired when the build configurations are created. + */ +public class ConfigurationsCreatedEvent { + + private final BuildConfigurationCollection configurations; + + /** + * Construct the ConfigurationsCreatedEvent. + * + * @param configurations the build configuration collection + */ + public ConfigurationsCreatedEvent(BuildConfigurationCollection configurations) { + this.configurations = configurations; + } + + public BuildConfigurationCollection getBuildConfigurationCollection() { + return configurations; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAspectFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAspectFactory.java new file mode 100644 index 0000000..955241a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAspectFactory.java
@@ -0,0 +1,28 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.packages.AspectFactory; + +/** + * Instantiation of {@link AspectFactory} with the actual types. + * + * <p>This is needed because {@link AspectFactory} is needed in the {@code packages} package to + * do loading phase things properly and to be able to specify them on attributes, but the actual + * classes are in the {@code view} package, which is not available there. + */ +public interface ConfiguredAspectFactory + extends AspectFactory<ConfiguredTarget, RuleContext, Aspect> { + +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAttributeMapper.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAttributeMapper.java new file mode 100644 index 0000000..84029b2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAttributeMapper.java
@@ -0,0 +1,158 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider; +import com.google.devtools.build.lib.packages.AbstractAttributeMapper; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Map; +import java.util.Set; + +/** + * {@link AttributeMap} implementation that binds a rule's attribute as follows: + * + * <ol> + * <li>If the attribute is selectable (i.e. its BUILD declaration is of the form + * "attr = { config1: "value1", "config2: "value2", ... }", returns the subset of values + * chosen by the current configuration in accordance with Bazel's documented policy on + * configurable attribute selection. + * <li>If the attribute is not selectable (i.e. its value is static), returns that value with + * no additional processing. + * </ol> + * + * <p>Example usage: + * <pre> + * Label fooLabel = ConfiguredAttributeMapper.of(ruleConfiguredTarget).get("foo", Type.LABEL); + * </pre> + */ +public class ConfiguredAttributeMapper extends AbstractAttributeMapper { + + private final Map<Label, ConfigMatchingProvider> configConditions; + private Rule rule; + + private ConfiguredAttributeMapper(Rule rule, Set<ConfigMatchingProvider> configConditions) { + super(Preconditions.checkNotNull(rule).getPackage(), rule.getRuleClassObject(), rule.getLabel(), + rule.getAttributeContainer()); + ImmutableMap.Builder<Label, ConfigMatchingProvider> builder = ImmutableMap.builder(); + for (ConfigMatchingProvider configCondition : configConditions) { + builder.put(configCondition.label(), configCondition); + } + this.configConditions = builder.build(); + this.rule = rule; + } + + /** + * "Do-it-all" constructor that just needs a {@link RuleConfiguredTarget}. + */ + public static ConfiguredAttributeMapper of(RuleConfiguredTarget ct) { + return new ConfiguredAttributeMapper(ct.getTarget(), ct.getConfigConditions()); + } + + /** + * "Manual" constructor that requires the caller to pass the set of configurability conditions + * that trigger this rule's configurable attributes. + * + * <p>If you don't know how to do this, you really want to use one of the "do-it-all" + * constructors. + */ + static ConfiguredAttributeMapper of(Rule rule, Set<ConfigMatchingProvider> configConditions) { + return new ConfiguredAttributeMapper(rule, configConditions); + } + + /** + * Checks that all attributes can be mapped to their configured values. This is + * useful for checking that the configuration space in a configured attribute doesn't + * contain unresolvable contradictions. + * + * @throws EvalException if any attribute's value can't be resolved under this mapper + */ + public void validateAttributes() throws EvalException { + for (String attrName : getAttributeNames()) { + getAndValidate(attrName, getAttributeType(attrName)); + } + } + + /** + * Variation of {@link #get} that throws an informative exception if the attribute + * can't be resolved due to intrinsic contradictions in the configuration. + */ + private <T> T getAndValidate(String attributeName, Type<T> type) throws EvalException { + Type.Selector<T> selector = getSelector(attributeName, type); + if (selector == null) { + // This is a normal attribute. + return super.get(attributeName, type); + } + + // We expect exactly one of this attribute's conditions to match (including the default + // condition, if specified). Throw an exception if our expectations aren't met. + Label matchingCondition = null; + T matchingValue = null; + + // Find the matching condition and record its value (checking for duplicates). + for (Map.Entry<Label, T> entry : selector.getEntries().entrySet()) { + Label curCondition = entry.getKey(); + if (Type.Selector.isReservedLabel(curCondition)) { + continue; + } else if (Preconditions.checkNotNull(configConditions.get(curCondition)).matches()) { + if (matchingCondition != null) { + throw new EvalException(rule.getAttributeLocation(attributeName), + "Both " + matchingCondition.toString() + " and " + curCondition.toString() + + " match configurable attribute \"" + attributeName + "\" in " + getLabel() + + ". At most one match is allowed"); + } + matchingCondition = curCondition; + matchingValue = entry.getValue(); + } + } + + // If nothing matched, choose the default condition. + if (matchingCondition == null) { + if (!selector.hasDefault()) { + throw new EvalException(rule.getAttributeLocation(attributeName), + "Configurable attribute \"" + attributeName + "\" doesn't match this " + + "configuration (would a default condition help?)"); + } + matchingValue = selector.getDefault(); + } + + return matchingValue; + } + + @Override + public <T> T get(String attributeName, Type<T> type) { + try { + return getAndValidate(attributeName, type); + } catch (EvalException e) { + // Callers that reach this branch should explicitly validate the attribute through an + // appropriate call and handle the exception directly. This method assumes + // pre-validated attributes. + throw new IllegalStateException( + "lookup failed on attribute " + attributeName + ": " + e.getMessage()); + } + } + + @Override + protected <T> Iterable<T> visitAttribute(String attributeName, Type<T> type) { + T value = get(attributeName, type); + return value == null ? ImmutableList.<T>of() : ImmutableList.of(value); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredRuleClassProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredRuleClassProvider.java new file mode 100644 index 0000000..e84f9aa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredRuleClassProvider.java
@@ -0,0 +1,376 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import static com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType.ABSTRACT; +import static com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType.TEST; + +import com.google.common.base.Preconditions; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; +import com.google.devtools.build.lib.analysis.config.DefaultsPackage; +import com.google.devtools.build.lib.analysis.config.FragmentOptions; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.graph.Digraph; +import com.google.devtools.build.lib.graph.Node; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClassProvider; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.SkylarkModules; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.syntax.SkylarkType; +import com.google.devtools.build.lib.syntax.ValidationEnvironment; +import com.google.devtools.common.options.OptionsClassProvider; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Knows about every rule Blaze supports and the associated configuration options. + * + * <p>This class is initialized on server startup and the set of rules, build info factories + * and configuration options is guarantees not to change over the life time of the Blaze server. + */ +public class ConfiguredRuleClassProvider implements RuleClassProvider { + /** + * Custom dependency validation logic. + */ + public static interface PrerequisiteValidator { + /** + * Checks whether the rule in {@code contextBuilder} is allowed to depend on + * {@code prerequisite} through the attribute {@code attribute}. + * + * <p>Can be used for enforcing any organization-specific policies about the layout of the + * workspace. + */ + void validate( + RuleContext.Builder contextBuilder, ConfiguredTarget prerequisite, Attribute attribute); + } + + /** + * Builder for {@link ConfiguredRuleClassProvider}. + */ + public static class Builder implements RuleDefinitionEnvironment { + private final List<ConfigurationFragmentFactory> configurationFragments = new ArrayList<>(); + private final List<BuildInfoFactory> buildInfoFactories = new ArrayList<>(); + private final List<Class<? extends FragmentOptions>> configurationOptions = new ArrayList<>(); + + private final Map<String, RuleClass> ruleClassMap = new HashMap<>(); + private final Map<String, Class<? extends RuleDefinition>> ruleDefinitionMap = + new HashMap<>(); + private final Map<Class<? extends RuleDefinition>, RuleClass> ruleMap = new HashMap<>(); + private final Digraph<Class<? extends RuleDefinition>> dependencyGraph = + new Digraph<>(); + private ConfigurationCollectionFactory configurationCollectionFactory; + private PrerequisiteValidator prerequisiteValidator; + private ImmutableMap<String, SkylarkType> skylarkAccessibleJavaClasses = ImmutableMap.of(); + + public Builder setPrerequisiteValidator(PrerequisiteValidator prerequisiteValidator) { + this.prerequisiteValidator = prerequisiteValidator; + return this; + } + + public Builder addBuildInfoFactory(BuildInfoFactory factory) { + buildInfoFactories.add(factory); + return this; + } + + public Builder addRuleDefinition(Class<? extends RuleDefinition> ruleDefinition) { + dependencyGraph.createNode(ruleDefinition); + BlazeRule annotation = ruleDefinition.getAnnotation(BlazeRule.class); + for (Class<? extends RuleDefinition> ancestor : annotation.ancestors()) { + dependencyGraph.addEdge(ancestor, ruleDefinition); + } + + return this; + } + + public Builder addConfigurationOptions(Class<? extends FragmentOptions> configurationOptions) { + this.configurationOptions.add(configurationOptions); + return this; + } + + public Builder addConfigurationFragment(ConfigurationFragmentFactory factory) { + configurationFragments.add(factory); + return this; + } + + public Builder setConfigurationCollectionFactory(ConfigurationCollectionFactory factory) { + this.configurationCollectionFactory = factory; + return this; + } + + public Builder setSkylarkAccessibleJavaClasses(ImmutableMap<String, SkylarkType> objects) { + this.skylarkAccessibleJavaClasses = objects; + return this; + } + + private RuleConfiguredTargetFactory createFactory( + Class<? extends RuleConfiguredTargetFactory> factoryClass) { + try { + Constructor<? extends RuleConfiguredTargetFactory> ctor = factoryClass.getConstructor(); + return ctor.newInstance(); + } catch (NoSuchMethodException | IllegalAccessException | InstantiationException + | InvocationTargetException e) { + throw new IllegalStateException(e); + } + } + + private RuleClass commitRuleDefinition(Class<? extends RuleDefinition> definitionClass) { + BlazeRule annotation = definitionClass.getAnnotation(BlazeRule.class); + Preconditions.checkArgument(ruleClassMap.get(annotation.name()) == null, annotation.name()); + + Preconditions.checkArgument( + annotation.type() == ABSTRACT ^ + annotation.factoryClass() != RuleConfiguredTargetFactory.class); + Preconditions.checkArgument( + (annotation.type() != TEST) || + Arrays.asList(annotation.ancestors()).contains( + BaseRuleClasses.TestBaseRule.class)); + + RuleDefinition instance; + try { + instance = definitionClass.newInstance(); + } catch (IllegalAccessException | InstantiationException e) { + throw new IllegalStateException(e); + } + RuleClass[] ancestorClasses = new RuleClass[annotation.ancestors().length]; + for (int i = 0; i < annotation.ancestors().length; i++) { + ancestorClasses[i] = ruleMap.get(annotation.ancestors()[i]); + if (ancestorClasses[i] == null) { + // Ancestors should have been initialized by now + throw new IllegalStateException("Ancestor " + annotation.ancestors()[i] + " of " + + annotation.name() + " is not initialized"); + } + } + + RuleConfiguredTargetFactory factory = null; + if (annotation.type() != ABSTRACT) { + factory = createFactory(annotation.factoryClass()); + } + + RuleClass.Builder builder = new RuleClass.Builder( + annotation.name(), annotation.type(), false, ancestorClasses); + builder.factory(factory); + RuleClass ruleClass = instance.build(builder, this); + ruleMap.put(definitionClass, ruleClass); + ruleClassMap.put(ruleClass.getName(), ruleClass); + ruleDefinitionMap.put(ruleClass.getName(), definitionClass); + + return ruleClass; + } + + public ConfiguredRuleClassProvider build() { + for (Node<Class<? extends RuleDefinition>> ruleDefinition : + dependencyGraph.getTopologicalOrder()) { + commitRuleDefinition(ruleDefinition.getLabel()); + } + + return new ConfiguredRuleClassProvider( + ImmutableMap.copyOf(ruleClassMap), + ImmutableMap.copyOf(ruleDefinitionMap), + ImmutableList.copyOf(buildInfoFactories), + ImmutableList.copyOf(configurationOptions), + ImmutableList.copyOf(configurationFragments), + configurationCollectionFactory, + prerequisiteValidator, + skylarkAccessibleJavaClasses); + } + + @Override + public Label getLabel(String labelValue) { + return LABELS.getUnchecked(labelValue); + } + } + + /** + * Used to make the label instances unique, so that we don't create a new + * instance for every rule. + */ + private static LoadingCache<String, Label> LABELS = CacheBuilder.newBuilder().build( + new CacheLoader<String, Label>() { + @Override + public Label load(String from) { + try { + return Label.parseAbsolute(from); + } catch (Label.SyntaxException e) { + throw new IllegalArgumentException(from); + } + } + }); + + /** + * Maps rule class name to the metaclass instance for that rule. + */ + private final ImmutableMap<String, RuleClass> ruleClassMap; + + /** + * Maps rule class name to the rule definition metaclasses. + */ + private final ImmutableMap<String, Class<? extends RuleDefinition>> ruleDefinitionMap; + + /** + * The configuration options that affect the behavior of the rules. + */ + private final ImmutableList<Class<? extends FragmentOptions>> configurationOptions; + + /** + * The set of configuration fragment factories. + */ + private final ImmutableList<ConfigurationFragmentFactory> configurationFragments; + + /** + * The factory that creates the configuration collection. + */ + private final ConfigurationCollectionFactory configurationCollectionFactory; + + private final ImmutableList<BuildInfoFactory> buildInfoFactories; + + private final PrerequisiteValidator prerequisiteValidator; + + private final ImmutableMap<String, SkylarkType> skylarkAccessibleJavaClasses; + + private final ValidationEnvironment skylarkValidationEnvironment; + + public ConfiguredRuleClassProvider( + ImmutableMap<String, RuleClass> ruleClassMap, + ImmutableMap<String, Class<? extends RuleDefinition>> ruleDefinitionMap, + ImmutableList<BuildInfoFactory> buildInfoFactories, + ImmutableList<Class<? extends FragmentOptions>> configurationOptions, + ImmutableList<ConfigurationFragmentFactory> configurationFragments, + ConfigurationCollectionFactory configurationCollectionFactory, + PrerequisiteValidator prerequisiteValidator, + ImmutableMap<String, SkylarkType> skylarkAccessibleJavaClasses) { + + this.ruleClassMap = ruleClassMap; + this.ruleDefinitionMap = ruleDefinitionMap; + this.buildInfoFactories = buildInfoFactories; + this.configurationOptions = configurationOptions; + this.configurationFragments = configurationFragments; + this.configurationCollectionFactory = configurationCollectionFactory; + this.prerequisiteValidator = prerequisiteValidator; + this.skylarkAccessibleJavaClasses = skylarkAccessibleJavaClasses; + this.skylarkValidationEnvironment = SkylarkModules.getValidationEnvironment( + ImmutableMap.<String, SkylarkType>builder() + .putAll(skylarkAccessibleJavaClasses) + .put("native", SkylarkType.of(NativeModule.class)) + .build()); + } + + public PrerequisiteValidator getPrerequisiteValidator() { + return prerequisiteValidator; + } + + @Override + public Map<String, RuleClass> getRuleClassMap() { + return ruleClassMap; + } + + /** + * Returns a list of build info factories that are needed for the supported languages. + */ + public ImmutableList<BuildInfoFactory> getBuildInfoFactories() { + return buildInfoFactories; + } + + /** + * Returns the set of configuration fragments provided by this module. + */ + public ImmutableList<ConfigurationFragmentFactory> getConfigurationFragments() { + return configurationFragments; + } + + /** + * Returns the set of configuration options that are supported in this module. + */ + public ImmutableList<Class<? extends FragmentOptions>> getConfigurationOptions() { + return configurationOptions; + } + + /** + * Returns the definition of the rule class definition with the specified name. + */ + public Class<? extends RuleDefinition> getRuleClassDefinition(String ruleClassName) { + return ruleDefinitionMap.get(ruleClassName); + } + + /** + * Returns the configuration collection creator. + */ + public ConfigurationCollectionFactory getConfigurationCollectionFactory() { + return configurationCollectionFactory; + } + + /** + * Returns the defaults package for the default settings. + */ + public String getDefaultsPackageContent() { + return DefaultsPackage.getDefaultsPackageContent(configurationOptions); + } + + /** + * Returns the defaults package for the given options taken from an optionsProvider. + */ + public String getDefaultsPackageContent(OptionsClassProvider optionsProvider) { + return DefaultsPackage.getDefaultsPackageContent( + BuildOptions.of(configurationOptions, optionsProvider)); + } + + /** + * Creates a BuildOptions class for the given options taken from an optionsProvider. + */ + public BuildOptions createBuildOptions(OptionsClassProvider optionsProvider) { + return BuildOptions.of(configurationOptions, optionsProvider); + } + + @SkylarkModule(name = "native", namespace = true, onlyLoadingPhase = true, + doc = "Module for native rules.") + private static final class NativeModule {} + + public static final NativeModule nativeModule = new NativeModule(); + + @Override + public SkylarkEnvironment createSkylarkRuleClassEnvironment( + EventHandler eventHandler, String astFileContentHashCode) { + SkylarkEnvironment env = SkylarkModules.getNewEnvironment(eventHandler, astFileContentHashCode); + for (Map.Entry<String, SkylarkType> entry : skylarkAccessibleJavaClasses.entrySet()) { + env.update(entry.getKey(), entry.getValue().getType()); + } + return env; + } + + @Override + public ValidationEnvironment getSkylarkValidationEnvironment() { + return skylarkValidationEnvironment; + } + + @Override + public Object getNativeModule() { + return nativeModule; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredTarget.java new file mode 100644 index 0000000..7aad367 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredTarget.java
@@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Target; + +import javax.annotation.Nullable; + +/** + * A {@link ConfiguredTarget} is conceptually a {@link TransitiveInfoCollection} coupled + * with the {@link Target} and {@link BuildConfiguration} objects it was created from. + * + * <p>This interface is supposed to only be used in {@link BuildView} and above. In particular, + * rule implementations should not be able to access the {@link ConfiguredTarget} objects + * associated with their direct dependencies, only the corresponding + * {@link TransitiveInfoCollection}s. Also, {@link ConfiguredTarget} objects should not be + * accessible from the action graph. + */ +public interface ConfiguredTarget extends TransitiveInfoCollection { + /** + * Returns the Target with which this {@link ConfiguredTarget} is associated. + */ + Target getTarget(); + + /** + * <p>Returns the {@link BuildConfiguration} for which this {@link ConfiguredTarget} is + * defined. Configuration is defined for all configured targets with exception + * of the {@link InputFileConfiguredTarget} for which it is always + * <b>null</b>.</p> + */ + @Override + @Nullable + BuildConfiguration getConfiguration(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredTargetFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredTargetFactory.java new file mode 100644 index 0000000..d265533 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredTargetFactory.java
@@ -0,0 +1,302 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ListMultimap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.actions.FailAction; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.ConstantRuleVisibility; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.PackageGroup; +import com.google.devtools.build.lib.packages.PackageGroupsRuleVisibility; +import com.google.devtools.build.lib.packages.PackageSpecification; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleVisibility; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.rules.SkylarkRuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * This class creates {@link ConfiguredTarget} instances using a given {@link + * ConfiguredRuleClassProvider}. + */ +@ThreadSafe +public final class ConfiguredTargetFactory { + // This class is not meant to be outside of the analysis phase machinery and is only public + // in order to be accessible from the .view.skyframe package. + + private final ConfiguredRuleClassProvider ruleClassProvider; + + public ConfiguredTargetFactory(ConfiguredRuleClassProvider ruleClassProvider) { + this.ruleClassProvider = ruleClassProvider; + } + + /** + * Returns the visibility of the given target. Errors during package group resolution are reported + * to the {@code AnalysisEnvironment}. + */ + private NestedSet<PackageSpecification> convertVisibility( + ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap, EventHandler reporter, + Target target, BuildConfiguration packageGroupConfiguration) { + RuleVisibility ruleVisibility = target.getVisibility(); + if (ruleVisibility instanceof ConstantRuleVisibility) { + return ((ConstantRuleVisibility) ruleVisibility).isPubliclyVisible() + ? NestedSetBuilder.<PackageSpecification>create( + Order.STABLE_ORDER, PackageSpecification.EVERYTHING) + : NestedSetBuilder.<PackageSpecification>emptySet(Order.STABLE_ORDER); + } else if (ruleVisibility instanceof PackageGroupsRuleVisibility) { + PackageGroupsRuleVisibility packageGroupsVisibility = + (PackageGroupsRuleVisibility) ruleVisibility; + + NestedSetBuilder<PackageSpecification> packageSpecifications = + NestedSetBuilder.stableOrder(); + for (Label groupLabel : packageGroupsVisibility.getPackageGroups()) { + // PackageGroupsConfiguredTargets are always in the package-group configuration. + ConfiguredTarget group = + findPrerequisite(prerequisiteMap, groupLabel, packageGroupConfiguration); + PackageSpecificationProvider provider = null; + // group == null can only happen if the package group list comes + // from a default_visibility attribute, because in every other case, + // this missing link is caught during transitive closure visitation or + // if the RuleConfiguredTargetGraph threw out a visibility edge + // because if would have caused a cycle. The filtering should be done + // in a single place, ConfiguredTargetGraph, but for now, this is the + // minimally invasive way of providing a sane error message in case a + // cycle is created by a visibility attribute. + if (group != null) { + provider = group.getProvider(PackageSpecificationProvider.class); + } + if (provider != null) { + packageSpecifications.addTransitive(provider.getPackageSpecifications()); + } else { + reporter.handle(Event.error(target.getLocation(), + String.format("Label '%s' does not refer to a package group", groupLabel))); + } + } + + packageSpecifications.addAll(packageGroupsVisibility.getDirectPackages()); + return packageSpecifications.build(); + } else { + throw new IllegalStateException("unknown visibility"); + } + } + + private ConfiguredTarget findPrerequisite( + ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap, Label label, + BuildConfiguration config) { + for (ConfiguredTarget prerequisite : prerequisiteMap.get(null)) { + if (prerequisite.getLabel().equals(label) && (prerequisite.getConfiguration() == config)) { + return prerequisite; + } + } + return null; + } + + private Artifact getOutputArtifact(OutputFile outputFile, BuildConfiguration configuration, + boolean isFileset, ArtifactFactory artifactFactory) { + Rule rule = outputFile.getAssociatedRule(); + Root root = rule.hasBinaryOutput() + ? configuration.getBinDirectory() + : configuration.getGenfilesDirectory(); + ArtifactOwner owner = + new ConfiguredTargetKey(rule.getLabel(), configuration.getArtifactOwnerConfiguration()); + PathFragment rootRelativePath = Util.getWorkspaceRelativePath(outputFile); + Artifact result = isFileset + ? artifactFactory.getFilesetArtifact(rootRelativePath, root, owner) + : artifactFactory.getDerivedArtifact(rootRelativePath, root, owner); + // The associated rule should have created the artifact. + Preconditions.checkNotNull(result, "no artifact for %s", rootRelativePath); + return result; + } + + /** + * Invokes the appropriate constructor to create a {@link ConfiguredTarget} instance. + * <p>For use in {@code ConfiguredTargetFunction}. + * + * <p>Returns null if Skyframe deps are missing or upon certain errors. + */ + @Nullable + public final ConfiguredTarget createConfiguredTarget(AnalysisEnvironment analysisEnvironment, + ArtifactFactory artifactFactory, Target target, BuildConfiguration config, + ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap, + Set<ConfigMatchingProvider> configConditions) + throws InterruptedException { + if (target instanceof Rule) { + return createRule( + analysisEnvironment, (Rule) target, config, prerequisiteMap, configConditions); + } + + // Visibility, like all package groups, doesn't have a configuration + NestedSet<PackageSpecification> visibility = convertVisibility( + prerequisiteMap, analysisEnvironment.getEventHandler(), target, null); + TargetContext targetContext = new TargetContext(analysisEnvironment, target, config, + prerequisiteMap.get(null), visibility); + if (target instanceof OutputFile) { + OutputFile outputFile = (OutputFile) target; + boolean isFileset = outputFile.getGeneratingRule().getRuleClass().equals("Fileset"); + Artifact artifact = getOutputArtifact(outputFile, config, isFileset, artifactFactory); + TransitiveInfoCollection rule = targetContext.findDirectPrerequisite( + outputFile.getGeneratingRule().getLabel(), config); + if (isFileset) { + return new FilesetOutputConfiguredTarget(targetContext, outputFile, rule, artifact); + } else { + return new OutputFileConfiguredTarget(targetContext, outputFile, rule, artifact); + } + } else if (target instanceof InputFile) { + InputFile inputFile = (InputFile) target; + Artifact artifact = artifactFactory.getSourceArtifact( + inputFile.getExecPath(), + Root.asSourceRoot(inputFile.getPackage().getSourceRoot()), + new ConfiguredTargetKey(target.getLabel(), config)); + + return new InputFileConfiguredTarget(targetContext, inputFile, artifact); + } else if (target instanceof PackageGroup) { + PackageGroup packageGroup = (PackageGroup) target; + return new PackageGroupConfiguredTarget(targetContext, packageGroup); + } else { + throw new AssertionError("Unexpected target class: " + target.getClass().getName()); + } + } + + /** + * Factory method: constructs a RuleConfiguredTarget of the appropriate class, based on the rule + * class. May return null if an error occurred. + */ + @Nullable + private ConfiguredTarget createRule( + AnalysisEnvironment env, Rule rule, BuildConfiguration configuration, + ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap, + Set<ConfigMatchingProvider> configConditions) throws InterruptedException { + // Visibility computation and checking is done for every rule. + RuleContext ruleContext = new RuleContext.Builder(env, rule, configuration, + ruleClassProvider.getPrerequisiteValidator()) + .setVisibility(convertVisibility(prerequisiteMap, env.getEventHandler(), rule, null)) + .setPrerequisites(prerequisiteMap) + .setConfigConditions(configConditions) + .build(); + if (ruleContext.hasErrors()) { + return null; + } + if (!rule.getRuleClassObject().getRequiredConfigurationFragments().isEmpty()) { + if (!configuration.hasAllFragments( + rule.getRuleClassObject().getRequiredConfigurationFragments())) { + if (rule.getRuleClassObject().failIfMissingConfigurationFragment()) { + ruleContext.ruleError(missingFragmentError(ruleContext)); + return null; + } + return createFailConfiguredTarget(ruleContext); + } + } + if (rule.getRuleClassObject().isSkylarkExecutable()) { + // TODO(bazel-team): maybe merge with RuleConfiguredTargetBuilder? + return SkylarkRuleConfiguredTargetBuilder.buildRule( + ruleContext, rule.getRuleClassObject().getConfiguredTargetFunction()); + } else { + RuleClass.ConfiguredTargetFactory<ConfiguredTarget, RuleContext> factory = + rule.getRuleClassObject().<ConfiguredTarget, RuleContext>getConfiguredTargetFactory(); + Preconditions.checkNotNull(factory, rule.getRuleClassObject()); + return factory.create(ruleContext); + } + } + + private String missingFragmentError(RuleContext ruleContext) { + RuleClass ruleClass = ruleContext.getRule().getRuleClassObject(); + Set<Class<?>> missingFragments = new LinkedHashSet<>(); + for (Class<?> fragment : ruleClass.getRequiredConfigurationFragments()) { + if (!ruleContext.getConfiguration().hasFragment(fragment.asSubclass(Fragment.class))) { + missingFragments.add(fragment); + } + } + Preconditions.checkState(!missingFragments.isEmpty()); + StringBuilder result = new StringBuilder(); + result.append("all rules of type " + ruleClass.getName() + " require the presence of "); + List<String> names = new ArrayList<>(); + for (Class<?> fragment : missingFragments) { + // TODO(bazel-team): Using getSimpleName here is sub-optimal, but we don't have anything + // better right now. + names.add(fragment.getSimpleName()); + } + result.append("all of ["); + result.append(Joiner.on(",").join(names)); + result.append("], but these were all disabled"); + return result.toString(); + } + + /** + * Constructs an {@link Aspect}. Returns null if an error occurs; in that case, + * {@code aspectFactory} should call one of the error reporting methods of {@link RuleContext}. + */ + public Aspect createAspect( + AnalysisEnvironment env, RuleConfiguredTarget associatedTarget, + ConfiguredAspectFactory aspectFactory, + ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap, + Set<ConfigMatchingProvider> configConditions) { + RuleContext.Builder builder = new RuleContext.Builder(env, + associatedTarget.getTarget(), + associatedTarget.getConfiguration(), + ruleClassProvider.getPrerequisiteValidator()); + RuleContext ruleContext = builder + .setVisibility(convertVisibility( + prerequisiteMap, env.getEventHandler(), associatedTarget.getTarget(), null)) + .setPrerequisites(prerequisiteMap) + .setConfigConditions(configConditions) + .build(); + if (ruleContext.hasErrors()) { + return null; + } + + return aspectFactory.create(associatedTarget, ruleContext); + } + + /** + * A pseudo-implementation for configured targets that creates fail actions for all declared + * outputs, both implicit and explicit. + */ + private static ConfiguredTarget createFailConfiguredTarget(RuleContext ruleContext) { + RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(ruleContext); + if (!ruleContext.getOutputArtifacts().isEmpty()) { + ruleContext.registerAction(new FailAction(ruleContext.getActionOwner(), + ruleContext.getOutputArtifacts(), "Can't build this")); + } + builder.add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY)); + return builder.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/DependencyResolver.java b/src/main/java/com/google/devtools/build/lib/analysis/DependencyResolver.java new file mode 100644 index 0000000..64cddb1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/DependencyResolver.java
@@ -0,0 +1,573 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider; +import com.google.devtools.build.lib.collect.ImmutableSortedKeyListMultimap; +import com.google.devtools.build.lib.packages.AspectDefinition; +import com.google.devtools.build.lib.packages.AspectFactory; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition; +import com.google.devtools.build.lib.packages.Attribute.LateBoundDefault; +import com.google.devtools.build.lib.packages.Attribute.SplitTransition; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.EnvironmentGroup; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.PackageGroup; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import javax.annotation.Nullable; + +/** + * Resolver for dependencies between configured targets. + * + * <p>Includes logic to derive the right configurations depending on transition type. + */ +public abstract class DependencyResolver { + /** + * A dependency of a configured target through a label. + * + * <p>Includes the target and the configuration of the dependency configured target and any + * aspects that may be required. + * + * <p>Note that the presence of an aspect here does not necessarily mean that it will be created. + * They will be filtered based on the {@link TransitiveInfoProvider} instances their associated + * configured targets have (we cannot do that here because the configured targets are not + * available yet). No error or warning is reported in this case, because it is expected that rules + * sometimes over-approximate the providers they supply in their definitions. + */ + public static final class Dependency { + + /** + * Returns the {@link ConfiguredTargetKey} for a direct dependency. + * + * <p>Essentially the same information as {@link Dependency} minus the aspects. + */ + public static final Function<Dependency, ConfiguredTargetKey> + TO_CONFIGURED_TARGET_KEY = new Function<Dependency, ConfiguredTargetKey>() { + @Override + public ConfiguredTargetKey apply(Dependency input) { + return new ConfiguredTargetKey(input.getLabel(), input.getConfiguration()); + } + }; + + private final Label label; + private final BuildConfiguration configuration; + private final ImmutableSet<Class<? extends ConfiguredAspectFactory>> aspects; + + public Dependency(Label label, BuildConfiguration configuration, + ImmutableSet<Class<? extends ConfiguredAspectFactory>> aspects) { + this.label = label; + this.configuration = configuration; + this.aspects = aspects; + } + + public Dependency(Label label, BuildConfiguration configuration) { + this(label, configuration, ImmutableSet.<Class<? extends ConfiguredAspectFactory>>of()); + } + + public Label getLabel() { + return label; + } + + public BuildConfiguration getConfiguration() { + return configuration; + } + + public ImmutableSet<Class<? extends ConfiguredAspectFactory>> getAspects() { + return aspects; + } + } + + protected DependencyResolver() { + } + + /** + * Returns ids for dependent nodes of a given node, sorted by attribute. Note that some + * dependencies do not have a corresponding attribute here, and we use the null attribute to + * represent those edges. Visibility attributes are only visited if {@code visitVisibility} is + * {@code true}. + * + * <p>If {@code aspect} is null, returns the dependent nodes of the configured target node + * representing the given target and configuration, otherwise that of the aspect node accompanying + * the aforementioned configured target node for the specified aspect. + * + * <p>The values are not simply labels because this also implements the first step of applying + * configuration transitions, namely, split transitions. This needs to be done before the labels + * are resolved because late bound attributes depend on the configuration. A good example for this + * is @{code :cc_toolchain}. + * + * <p>The long-term goal is that most configuration transitions be applied here. However, in order + * to do that, we first have to eliminate transitions that depend on the rule class of the + * dependency. + */ + public final ListMultimap<Attribute, Dependency> dependentNodeMap( + TargetAndConfiguration node, AspectDefinition aspect, + Set<ConfigMatchingProvider> configConditions) + throws EvalException { + Target target = node.getTarget(); + BuildConfiguration config = node.getConfiguration(); + ListMultimap<Attribute, Dependency> outgoingEdges = ArrayListMultimap.create(); + if (target instanceof OutputFile) { + Preconditions.checkNotNull(config); + visitTargetVisibility(node, outgoingEdges.get(null)); + Rule rule = ((OutputFile) target).getGeneratingRule(); + outgoingEdges.get(null).add(new Dependency(rule.getLabel(), config)); + } else if (target instanceof InputFile) { + visitTargetVisibility(node, outgoingEdges.get(null)); + } else if (target instanceof EnvironmentGroup) { + visitTargetVisibility(node, outgoingEdges.get(null)); + } else if (target instanceof Rule) { + Preconditions.checkNotNull(config); + visitTargetVisibility(node, outgoingEdges.get(null)); + Rule rule = (Rule) target; + ListMultimap<Attribute, LabelAndConfiguration> labelMap = + resolveAttributes(rule, aspect, config, configConditions); + visitRule(rule, aspect, labelMap, outgoingEdges); + } else if (target instanceof PackageGroup) { + visitPackageGroup(node, (PackageGroup) target, outgoingEdges.get(null)); + } else { + throw new IllegalStateException(target.getLabel().toString()); + } + return outgoingEdges; + } + + private ListMultimap<Attribute, LabelAndConfiguration> resolveAttributes( + Rule rule, AspectDefinition aspect, BuildConfiguration configuration, + Set<ConfigMatchingProvider> configConditions) + throws EvalException { + ConfiguredAttributeMapper attributeMap = ConfiguredAttributeMapper.of(rule, configConditions); + attributeMap.validateAttributes(); + List<Attribute> attributes; + if (aspect == null) { + attributes = rule.getRuleClassObject().getAttributes(); + } else { + attributes = new ArrayList<>(); + attributes.addAll(rule.getRuleClassObject().getAttributes()); + if (aspect != null) { + attributes.addAll(aspect.getAttributes().values()); + } + } + + ImmutableSortedKeyListMultimap.Builder<Attribute, LabelAndConfiguration> result = + ImmutableSortedKeyListMultimap.builder(); + + resolveExplicitAttributes(rule, configuration, attributeMap, result); + resolveImplicitAttributes(rule, configuration, attributeMap, attributes, result); + resolveLateBoundAttributes(rule, configuration, attributeMap, attributes, result); + + // Add the rule's visibility labels (which may come from the rule or from package defaults). + addExplicitDeps(result, rule, "visibility", rule.getVisibility().getDependencyLabels(), + configuration); + + // Add package default constraints when the rule doesn't explicitly declare them. + // + // Note that this can have subtle implications for constraint semantics. For example: say that + // package defaults declare compatibility with ':foo' and rule R declares compatibility with + // ':bar'. Does that mean that R is compatible with [':foo', ':bar'] or just [':bar']? In other + // words, did R's author intend to add additional compatibility to the package defaults or to + // override them? More severely, what if package defaults "restrict" support to just [':baz']? + // Should R's declaration signify [':baz'] + ['bar'], [ORIGINAL_DEFAULTS] + ['bar'], or + // something else? + // + // Rather than try to answer these questions with possibly confusing logic, we take the + // simple approach of assigning the rule's "restriction" attribute to the rule-declared value if + // it exists, else the package defaults value (and likewise for "compatibility"). This may not + // always provide what users want, but it makes it easy for them to understand how rule + // declarations and package defaults intermix (and how to refactor them to get what they want). + // + // An alternative model would be to apply the "rule declaration" / "rule class defaults" + // relationship, i.e. the rule class' "compatibility" and "restriction" declarations are merged + // to generate a set of default environments, then the rule's declarations are independently + // processed on top of that. This protects against obscure coupling behavior between + // declarations from wildly different places (e.g. it offers clear answers to the examples posed + // above). But within the scope of a single package it seems better to keep the model simple and + // make the user responsible for resolving ambiguities. + if (!rule.isAttributeValueExplicitlySpecified(RuleClass.COMPATIBLE_ENVIRONMENT_ATTR)) { + addExplicitDeps(result, rule, RuleClass.COMPATIBLE_ENVIRONMENT_ATTR, + rule.getPackage().getDefaultCompatibleWith(), configuration); + } + if (!rule.isAttributeValueExplicitlySpecified(RuleClass.RESTRICTED_ENVIRONMENT_ATTR)) { + addExplicitDeps(result, rule, RuleClass.RESTRICTED_ENVIRONMENT_ATTR, + rule.getPackage().getDefaultRestrictedTo(), configuration); + } + + return result.build(); + } + + /** + * Adds new dependencies to the given rule under the given attribute name + * + * @param result the builder for the attribute --> dependency labels map + * @param rule the rule being evaluated + * @param attrName the name of the attribute to add dependency labels to + * @param labels the dependencies to add + * @param configuration the configuration to apply to those dependencies + */ + private void addExplicitDeps( + ImmutableSortedKeyListMultimap.Builder<Attribute, LabelAndConfiguration> result, Rule rule, + String attrName, Iterable<Label> labels, BuildConfiguration configuration) { + if (!rule.isAttrDefined(attrName, Type.LABEL_LIST) + && !rule.isAttrDefined(attrName, Type.NODEP_LABEL_LIST)) { + return; + } + Attribute attribute = rule.getRuleClassObject().getAttributeByName(attrName); + for (Label label : labels) { + // The configuration must be the configuration after the first transition step (applying + // split configurations). The proper configuration (null) for package groups will be set + // later. + result.put(attribute, LabelAndConfiguration.of(label, configuration)); + } + } + + private void resolveExplicitAttributes(Rule rule, final BuildConfiguration configuration, + AttributeMap attributes, + final ImmutableSortedKeyListMultimap.Builder<Attribute, LabelAndConfiguration> builder) { + attributes.visitLabels( + new AttributeMap.AcceptsLabelAttribute() { + @Override + public void acceptLabelAttribute(Label label, Attribute attribute) { + String attributeName = attribute.getName(); + if (attributeName.equals("abi_deps")) { + // abi_deps is handled specially: we visit only the branch that + // needs to be taken based on the configuration. + return; + } + + if (attribute.getType() == Type.NODEP_LABEL) { + return; + } + + if (attribute.isImplicit() || attribute.isLateBound()) { + return; + } + + builder.put(attribute, LabelAndConfiguration.of(label, configuration)); + } + }); + + // TODO(bazel-team): Remove this in favor of the new configurable attributes. + if (attributes.getAttributeDefinition("abi_deps") != null) { + Attribute depsAttribute = attributes.getAttributeDefinition("deps"); + MakeVariableExpander.Context context = new ConfigurationMakeVariableContext( + rule.getPackage(), configuration); + String abi = null; + try { + abi = MakeVariableExpander.expand(attributes.get("abi", Type.STRING), context); + } catch (MakeVariableExpander.ExpansionException e) { + // Ignore this. It will be handled during the analysis phase. + } + + if (abi != null) { + for (Map.Entry<String, List<Label>> entry + : attributes.get("abi_deps", Type.LABEL_LIST_DICT).entrySet()) { + try { + if (Pattern.matches(entry.getKey(), abi)) { + for (Label label : entry.getValue()) { + builder.put(depsAttribute, LabelAndConfiguration.of(label, configuration)); + } + } + } catch (PatternSyntaxException e) { + // Ignore this. It will be handled during the analysis phase. + } + } + } + } + } + + private void resolveImplicitAttributes(Rule rule, BuildConfiguration configuration, + AttributeMap attributeMap, Iterable<Attribute> attributes, + ImmutableSortedKeyListMultimap.Builder<Attribute, LabelAndConfiguration> builder) { + // Since the attributes that come from aspects do not appear in attributeMap, we have to get + // their values from somewhere else. This incidentally means that aspects attributes are not + // configurable. It would be nice if that wasn't the case, but we'd have to revamp how + // attribute mapping works, which is a large chunk of work. + ImmutableSet<String> mappedAttributes = ImmutableSet.copyOf(attributeMap.getAttributeNames()); + for (Attribute attribute : attributes) { + if (!attribute.isImplicit() || !attribute.getCondition().apply(attributeMap)) { + continue; + } + + if (attribute.getType() == Type.LABEL) { + Label label = mappedAttributes.contains(attribute.getName()) + ? attributeMap.get(attribute.getName(), Type.LABEL) + : Type.LABEL.cast(attribute.getDefaultValue(rule)); + + if (label != null) { + builder.put(attribute, LabelAndConfiguration.of(label, configuration)); + } + } else if (attribute.getType() == Type.LABEL_LIST) { + List<Label> labelList = mappedAttributes.contains(attribute.getName()) + ? attributeMap.get(attribute.getName(), Type.LABEL_LIST) + : Type.LABEL_LIST.cast(attribute.getDefaultValue(rule)); + + for (Label label : labelList) { + builder.put(attribute, LabelAndConfiguration.of(label, configuration)); + } + } + } + } + + private void resolveLateBoundAttributes(Rule rule, BuildConfiguration configuration, + AttributeMap attributeMap, Iterable<Attribute> attributes, + ImmutableSortedKeyListMultimap.Builder<Attribute, LabelAndConfiguration> builder) + throws EvalException { + for (Attribute attribute : attributes) { + if (!attribute.isLateBound() || !attribute.getCondition().apply(attributeMap)) { + continue; + } + + List<BuildConfiguration> actualConfigurations = ImmutableList.of(configuration); + if (attribute.getConfigurationTransition() instanceof SplitTransition<?>) { + Preconditions.checkState(attribute.getConfigurator() == null); + // TODO(bazel-team): This ends up applying the split transition twice, both here and in the + // visitRule method below - this is not currently a problem, because the configuration graph + // never contains nested split transitions, so the second application is idempotent. + actualConfigurations = configuration.getSplitConfigurations( + (SplitTransition<?>) attribute.getConfigurationTransition()); + } + + for (BuildConfiguration actualConfig : actualConfigurations) { + @SuppressWarnings("unchecked") + LateBoundDefault<BuildConfiguration> lateBoundDefault = + (LateBoundDefault<BuildConfiguration>) attribute.getLateBoundDefault(); + if (lateBoundDefault.useHostConfiguration()) { + actualConfig = + actualConfig.getConfiguration(ConfigurationTransition.HOST); + } + // TODO(bazel-team): This might be too expensive - can we cache this somehow? + if (!lateBoundDefault.getRequiredConfigurationFragments().isEmpty()) { + if (!actualConfig.hasAllFragments(lateBoundDefault.getRequiredConfigurationFragments())) { + continue; + } + } + + // TODO(bazel-team): We should check if the implementation tries to access an undeclared + // fragment. + Object actualValue = lateBoundDefault.getDefault(rule, actualConfig); + if (attribute.getType() == Type.LABEL) { + Label label; + label = Type.LABEL.cast(actualValue); + if (label != null) { + builder.put(attribute, LabelAndConfiguration.of(label, actualConfig)); + } + } else if (attribute.getType() == Type.LABEL_LIST) { + for (Label label : Type.LABEL_LIST.cast(actualValue)) { + builder.put(attribute, LabelAndConfiguration.of(label, actualConfig)); + } + } else { + throw new IllegalStateException(String.format( + "Late bound attribute '%s' is not a label or a label list", attribute.getName())); + } + } + } + } + + /** + * A variant of {@link #dependentNodeMap} that only returns the values of the resulting map, and + * also converts any internally thrown {@link EvalException} instances into {@link + * IllegalStateException}. + */ + public final Collection<Dependency> dependentNodes( + TargetAndConfiguration node, Set<ConfigMatchingProvider> configConditions) { + try { + return dependentNodeMap(node, null, configConditions).values(); + } catch (EvalException e) { + throw new IllegalStateException(e); + } + } + + /** + * Converts the given multimap of attributes to labels into a multi map of attributes to + * {@link Dependency} objects using the proper configuration transition for each attribute. + * + * @throws IllegalArgumentException if the {@code node} does not refer to a {@link Rule} instance + */ + public final Collection<Dependency> resolveRuleLabels( + TargetAndConfiguration node, AspectDefinition aspect, ListMultimap<Attribute, + LabelAndConfiguration> labelMap) { + Preconditions.checkArgument(node.getTarget() instanceof Rule); + Rule rule = (Rule) node.getTarget(); + ListMultimap<Attribute, Dependency> outgoingEdges = ArrayListMultimap.create(); + visitRule(rule, aspect, labelMap, outgoingEdges); + return outgoingEdges.values(); + } + + private void visitPackageGroup(TargetAndConfiguration node, PackageGroup packageGroup, + Collection<Dependency> outgoingEdges) { + for (Label label : packageGroup.getIncludes()) { + try { + Target target = getTarget(label); + if (target == null) { + return; + } + if (!(target instanceof PackageGroup)) { + // Note that this error could also be caught in PackageGroupConfiguredTarget, but since + // these have the null configuration, visiting the corresponding target would trigger an + // analysis of a rule with a null configuration, which doesn't work. + invalidPackageGroupReferenceHook(node, label); + continue; + } + + outgoingEdges.add(new Dependency(label, node.getConfiguration())); + } catch (NoSuchThingException e) { + // Don't visit targets that don't exist (--keep_going) + } + } + } + + private ImmutableSet<Class<? extends ConfiguredAspectFactory>> requiredAspects( + AspectDefinition aspect, Attribute attribute, Target target) { + if (!(target instanceof Rule)) { + return ImmutableSet.of(); + } + + RuleClass ruleClass = ((Rule) target).getRuleClassObject(); + + // The order of this set will be deterministic. This is necessary because this order eventually + // influences the order in which aspects are merged into the main configured target, which in + // turn influences which aspect takes precedence if two emit the same provider (maybe this + // should be an error) + Set<Class<? extends AspectFactory<?, ?, ?>>> aspectCandidates = new LinkedHashSet<>(); + aspectCandidates.addAll(attribute.getAspects()); + if (aspect != null) { + aspectCandidates.addAll(aspect.getAttributeAspects().get(attribute.getName())); + } + + ImmutableSet.Builder<Class<? extends ConfiguredAspectFactory>> result = ImmutableSet.builder(); + for (Class<? extends AspectFactory<?, ?, ?>> candidateClass : aspectCandidates) { + ConfiguredAspectFactory candidate = + (ConfiguredAspectFactory) AspectFactory.Util.create(candidateClass); + if (Sets.difference( + candidate.getDefinition().getRequiredProviders(), + ruleClass.getAdvertisedProviders()).isEmpty()) { + result.add(candidateClass.asSubclass(ConfiguredAspectFactory.class)); + } + } + + return result.build(); + } + + private void visitRule(Rule rule, AspectDefinition aspect, + ListMultimap<Attribute, LabelAndConfiguration> labelMap, + ListMultimap<Attribute, Dependency> outgoingEdges) { + Preconditions.checkNotNull(labelMap); + for (Map.Entry<Attribute, Collection<LabelAndConfiguration>> entry : + labelMap.asMap().entrySet()) { + Attribute attribute = entry.getKey(); + for (LabelAndConfiguration dep : entry.getValue()) { + Label label = dep.getLabel(); + BuildConfiguration config = dep.getConfiguration(); + + Target toTarget; + try { + toTarget = getTarget(label); + } catch (NoSuchThingException e) { + throw new IllegalStateException("not found: " + label + " from " + rule + " in " + + attribute.getName()); + } + if (toTarget == null) { + continue; + } + Iterable<BuildConfiguration> toConfigurations = config.evaluateTransition( + rule, attribute, toTarget); + for (BuildConfiguration toConfiguration : toConfigurations) { + outgoingEdges.get(entry.getKey()).add(new Dependency( + label, toConfiguration, requiredAspects(aspect, attribute, toTarget))); + } + } + } + } + + private void visitTargetVisibility(TargetAndConfiguration node, + Collection<Dependency> outgoingEdges) { + for (Label label : node.getTarget().getVisibility().getDependencyLabels()) { + try { + Target visibilityTarget = getTarget(label); + if (visibilityTarget == null) { + return; + } + if (!(visibilityTarget instanceof PackageGroup)) { + // Note that this error could also be caught in + // AbstractConfiguredTarget.convertVisibility(), but we have an + // opportunity here to avoid dependency cycles that result from + // the visibility attribute of a rule referring to a rule that + // depends on it (instead of its package) + invalidVisibilityReferenceHook(node, label); + continue; + } + + // Visibility always has null configuration + outgoingEdges.add(new Dependency(label, null)); + } catch (NoSuchThingException e) { + // Don't visit targets that don't exist (--keep_going) + } + } + } + + /** + * Hook for the error case when an invalid visibility reference is found. + * + * @param node the node with the visibility attribute + * @param label the invalid visibility reference + */ + protected abstract void invalidVisibilityReferenceHook(TargetAndConfiguration node, Label label); + + /** + * Hook for the error case when an invalid package group reference is found. + * + * @param node the package group node with the includes attribute + * @param label the invalid reference + */ + protected abstract void invalidPackageGroupReferenceHook(TargetAndConfiguration node, + Label label); + + /** + * Returns the target by the given label. + * + * <p>Throws {@link NoSuchThingException} if the target is known not to exist. + * + * <p>Returns null if the target is not ready to be returned at this moment. If getTarget returns + * null once or more during a {@link #dependentNodeMap} call, the results of that call will be + * incomplete. For use within Skyframe, where several iterations may be needed to discover + * all dependencies. + */ + @Nullable + protected abstract Target getTarget(Label label) throws NoSuchThingException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ErrorConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/ErrorConfiguredTarget.java new file mode 100644 index 0000000..aa07a7d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/ErrorConfiguredTarget.java
@@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.UnmodifiableIterator; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Target; + +/** + * A configured target that is used instead of a real configured target if there + * are cyclic dependencies or if any of the prerequisites has errors. This + * avoids accessing state that shouldn't be accessed. + */ +final class ErrorConfiguredTarget extends AbstractConfiguredTarget { + ErrorConfiguredTarget(Target target, BuildConfiguration configuration) { + super(target, configuration); + } + + @Override + public Object get(String providerKey) { + throw new UnsupportedOperationException(); + } + + @Override + public UnmodifiableIterator<TransitiveInfoProvider> iterator() { + throw new IllegalStateException(); + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ExtraActionArtifactsProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/ExtraActionArtifactsProvider.java new file mode 100644 index 0000000..05b5f37 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/ExtraActionArtifactsProvider.java
@@ -0,0 +1,103 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; + +/** + * A {@link TransitiveInfoProvider} that creates extra actions. + */ +@Immutable +public final class ExtraActionArtifactsProvider implements TransitiveInfoProvider { + public static final ExtraActionArtifactsProvider EMPTY = + new ExtraActionArtifactsProvider( + ImmutableList.<Artifact>of(), + NestedSetBuilder.<ExtraArtifactSet>emptySet(Order.STABLE_ORDER)); + + /** + * The set of extra artifacts provided by a single configured target. + */ + @Immutable + public static final class ExtraArtifactSet { + private final Label label; + private final ImmutableList<Artifact> artifacts; + + private ExtraArtifactSet(Label label, Iterable<Artifact> artifacts) { + this.label = label; + this.artifacts = ImmutableList.copyOf(artifacts); + } + + public Label getLabel() { + return label; + } + + public ImmutableList<Artifact> getArtifacts() { + return artifacts; + } + + public static ExtraArtifactSet of(Label label, Iterable<Artifact> artifacts) { + return new ExtraArtifactSet(label, artifacts); + } + + @Override + public int hashCode() { + return label.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + + if (!(other instanceof ExtraArtifactSet)) { + return false; + } + + return label.equals(((ExtraArtifactSet) other).getLabel()); + } + } + + /** The outputs of the extra actions associated with this target. */ + private ImmutableList<Artifact> extraActionArtifacts = ImmutableList.of(); + private NestedSet<ExtraArtifactSet> transitiveExtraActionArtifacts = + NestedSetBuilder.emptySet(Order.STABLE_ORDER); + + public ExtraActionArtifactsProvider(ImmutableList<Artifact> extraActionArtifacts, + NestedSet<ExtraArtifactSet> transitiveExtraActionArtifacts) { + this.extraActionArtifacts = extraActionArtifacts; + this.transitiveExtraActionArtifacts = transitiveExtraActionArtifacts; + } + + /** + * The outputs of the extra actions associated with this target. + */ + public ImmutableList<Artifact> getExtraActionArtifacts() { + return extraActionArtifacts; + } + + /** + * The outputs of the extra actions in the whole transitive closure. + */ + public NestedSet<ExtraArtifactSet> getTransitiveExtraActionArtifacts() { + return transitiveExtraActionArtifacts; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ExtraActionsVisitor.java b/src/main/java/com/google/devtools/build/lib/analysis/ExtraActionsVisitor.java new file mode 100644 index 0000000..79ffe80 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/ExtraActionsVisitor.java
@@ -0,0 +1,84 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionGraphVisitor; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.rules.extra.ExtraActionSpec; + +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A bipartite graph visitor which accumulates extra actions for a target. + */ +final class ExtraActionsVisitor extends ActionGraphVisitor { + private final RuleContext ruleContext; + private final Multimap<String, ExtraActionSpec> mnemonicToExtraActionMap; + private final List<Artifact> extraArtifacts; + public final Set<Action> actions = Sets.newHashSet(); + + /** Creates a new visitor for the extra actions associated with the given target. */ + public ExtraActionsVisitor(RuleContext ruleContext, + Multimap<String, ExtraActionSpec> mnemonicToExtraActionMap) { + super(getActionGraph(ruleContext)); + this.ruleContext = ruleContext; + this.mnemonicToExtraActionMap = mnemonicToExtraActionMap; + extraArtifacts = Lists.newArrayList(); + } + + public void addExtraAction(Action original) { + Collection<ExtraActionSpec> extraActions = mnemonicToExtraActionMap.get( + original.getMnemonic()); + if (extraActions != null) { + for (ExtraActionSpec extraAction : extraActions) { + extraArtifacts.addAll(extraAction.addExtraAction(ruleContext, original)); + } + } + } + + @Override + protected void visitAction(Action action) { + actions.add(action); + addExtraAction(action); + } + + /** Retrieves the collected artifacts since this method was last called and clears the list. */ + public ImmutableList<Artifact> getAndResetExtraArtifacts() { + ImmutableList<Artifact> collected = ImmutableList.copyOf(extraArtifacts); + extraArtifacts.clear(); + return collected; + } + + /** Gets an action graph wrapper for the given target through its analysis environment. */ + private static ActionGraph getActionGraph(final RuleContext ruleContext) { + return new ActionGraph() { + @Override + @Nullable + public Action getGeneratingAction(Artifact artifact) { + return ruleContext.getAnalysisEnvironment().getLocalGeneratingAction(artifact); + } + }; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/FileConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/FileConfiguredTarget.java new file mode 100644 index 0000000..815eea7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/FileConfiguredTarget.java
@@ -0,0 +1,93 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import com.google.common.collect.UnmodifiableIterator; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.FileTarget; +import com.google.devtools.build.lib.rules.fileset.FilesetProvider; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.util.FileType; + +/** + * A ConfiguredTarget for a source FileTarget. (Generated files use a + * subclass, OutputFileConfiguredTarget.) + */ +public abstract class FileConfiguredTarget extends AbstractConfiguredTarget + implements FileType.HasFilename, LicensesProvider { + + private final Artifact artifact; + private final ImmutableMap<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> + providers; + + FileConfiguredTarget(TargetContext targetContext, Artifact artifact) { + super(targetContext); + NestedSet<Artifact> filesToBuild = NestedSetBuilder.create(Order.STABLE_ORDER, artifact); + this.artifact = artifact; + Builder<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> builder = ImmutableMap + .<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider>builder() + .put(VisibilityProvider.class, this) + .put(LicensesProvider.class, this) + .put(FileProvider.class, new FileProvider(targetContext.getLabel(), filesToBuild)) + .put(FilesToRunProvider.class, new FilesToRunProvider(targetContext.getLabel(), + ImmutableList.copyOf(filesToBuild), null, artifact)); + if (this instanceof FilesetProvider) { + builder.put(FilesetProvider.class, this); + } + if (this instanceof InstrumentedFilesProvider) { + builder.put(InstrumentedFilesProvider.class, this); + } + this.providers = builder.build(); + } + + @Override + public FileTarget getTarget() { + return (FileTarget) super.getTarget(); + } + + public Artifact getArtifact() { + return artifact; + } + + /** + * Returns the file type of this file target. + */ + @Override + public String getFilename() { + return getTarget().getFilename(); + } + + @Override + public <P extends TransitiveInfoProvider> P getProvider(Class<P> provider) { + AnalysisUtils.checkProvider(provider); + return provider.cast(providers.get(provider)); + } + + @Override + public Object get(String providerKey) { + return null; + } + + @Override + public UnmodifiableIterator<TransitiveInfoProvider> iterator() { + return providers.values().iterator(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/FileProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/FileProvider.java new file mode 100644 index 0000000..893f211 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/FileProvider.java
@@ -0,0 +1,76 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; + +import javax.annotation.Nullable; + +/** + * A representation of the concept "this transitive info provider builds these files". + * + * <p>Every transitive info collection contains at least this provider. + */ +@Immutable +@SkylarkModule(name = "file_provider", doc = "An interface for rules that provide files.") +public final class FileProvider implements TransitiveInfoProvider { + + @Nullable private final Label label; + private final NestedSet<Artifact> filesToBuild; + + public FileProvider(@Nullable Label label, NestedSet<Artifact> filesToBuild) { + this.label = label; + this.filesToBuild = filesToBuild; + } + + /** + * Returns the label that is associated with this piece of information. + * + * <p>This is usually the label of the target that provides the information. + */ + @SkylarkCallable(name = "label", doc = "", structField = true) + public Label getLabel() { + if (label == null) { + throw new UnsupportedOperationException(); + } + return label; + } + + /** + * Returns the set of artifacts that are the "output" of this rule. + * + * <p>The term "output" is somewhat hazily defined; it is vaguely the set of files that are + * passed on to dependent rules that list the rule in their {@code srcs} attribute and the + * set of files that are built when a rule is mentioned on the command line. It does + * <b>not</b> include the runfiles; that is the bailiwick of {@code FilesToRunProvider}. + * + * <p>Note that the above definition is somewhat imprecise; in particular, when a rule is + * mentioned on the command line, some other files are also built + * {@code TopLevelArtifactHelper} and dependent rules are free to filter this set of artifacts + * e.g. based on their extension. + * + * <p>Also, some rules may generate artifacts that are not listed here by way of defining other + * implicit targets, for example, deploy jars. + */ + @SkylarkCallable(name = "files_to_build", doc = "", structField = true) + public NestedSet<Artifact> getFilesToBuild() { + return filesToBuild; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/FilesToCompileProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/FilesToCompileProvider.java new file mode 100644 index 0000000..025392c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/FilesToCompileProvider.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A {@link TransitiveInfoProvider} that provides files to be built when the {@code --compile_only} + * command line option is in effect. This is to avoid expensive build steps when the user only + * wants a quick syntax check. + */ +@Immutable +public final class FilesToCompileProvider implements TransitiveInfoProvider { + + private final ImmutableList<Artifact> filesToCompile; + + public FilesToCompileProvider(ImmutableList<Artifact> filesToCompile) { + this.filesToCompile = filesToCompile; + } + + /** + * Returns the list of artifacts to be built when the {@code --compile_only} command line option + * is in effect. + */ + public ImmutableList<Artifact> getFilesToCompile() { + return filesToCompile; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/FilesToRunProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/FilesToRunProvider.java new file mode 100644 index 0000000..0e024b1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/FilesToRunProvider.java
@@ -0,0 +1,80 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; + +import javax.annotation.Nullable; + +/** + * Returns information about executables produced by a target and the files needed to run it. + */ +@Immutable +public final class FilesToRunProvider implements TransitiveInfoProvider { + + private final Label label; + private final ImmutableList<Artifact> filesToRun; + @Nullable private final RunfilesSupport runfilesSupport; + @Nullable private final Artifact executable; + + public FilesToRunProvider(Label label, ImmutableList<Artifact> filesToRun, + @Nullable RunfilesSupport runfilesSupport, @Nullable Artifact executable) { + this.label = label; + this.filesToRun = filesToRun; + this.runfilesSupport = runfilesSupport; + this.executable = executable; + } + + /** + * Returns the label that is associated with this piece of information. + * + * <p>This is usually the label of the target that provides the information. + */ + public Label getLabel() { + return label; + } + + /** + * Returns artifacts needed to run the executable for this target. + */ + public ImmutableList<Artifact> getFilesToRun() { + return filesToRun; + } + + /** + * Returns the {@RunfilesSupport} object associated with the target or null if it does not exist. + */ + @Nullable public RunfilesSupport getRunfilesSupport() { + return runfilesSupport; + } + + /** + * Returns the Executable or null if it does not exist. + */ + @Nullable public Artifact getExecutable() { + return executable; + } + + /** + * Returns the RunfilesManifest or null if it does not exist. It is a shortcut to + * getRunfilesSupport().getRunfilesManifest(). + */ + @Nullable public Artifact getRunfilesManifest() { + return runfilesSupport != null ? runfilesSupport.getRunfilesManifest() : null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/FilesetOutputConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/FilesetOutputConfiguredTarget.java new file mode 100644 index 0000000..860024d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/FilesetOutputConfiguredTarget.java
@@ -0,0 +1,55 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.rules.fileset.FilesetProvider; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * A configured target for output files generated by {@code Fileset} rules. They are almost the + * same thing as output files except that they implement {@link FilesetProvider} so that + * {@code Fileset} can figure out the link tree behind them. + * + * <p>In an ideal world, this would not be needed: Filesets would depend on other Filesets and not + * their output directories. However, sometimes a Fileset depends on the output directory of + * another Fileset. Thus, we need this hack. + */ +public final class FilesetOutputConfiguredTarget extends OutputFileConfiguredTarget + implements FilesetProvider { + private final Artifact filesetInputManifest; + private final PathFragment filesetLinkDir; + + FilesetOutputConfiguredTarget(TargetContext targetContext, OutputFile outputFile, + TransitiveInfoCollection generatingRule, Artifact outputArtifact) { + super(targetContext, outputFile, generatingRule, outputArtifact); + FilesetProvider provider = generatingRule.getProvider(FilesetProvider.class); + Preconditions.checkArgument(provider != null); + filesetInputManifest = provider.getFilesetInputManifest(); + filesetLinkDir = provider.getFilesetLinkDir(); + } + + @Override + public Artifact getFilesetInputManifest() { + return filesetInputManifest; + } + + @Override + public PathFragment getFilesetLinkDir() { + return filesetLinkDir; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/InputFileConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/InputFileConfiguredTarget.java new file mode 100644 index 0000000..9e56033 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/InputFileConfiguredTarget.java
@@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.License; + +/** + * A ConfiguredTarget for an InputFile. + * + * All InputFiles for the same target are equivalent, so configuration does not + * play any role here and is always set to <b>null</b>. + */ +public final class InputFileConfiguredTarget extends FileConfiguredTarget { + private final Artifact artifact; + private final NestedSet<TargetLicense> licenses; + + InputFileConfiguredTarget(TargetContext targetContext, InputFile inputFile, Artifact artifact) { + super(targetContext, artifact); + Preconditions.checkArgument(targetContext.getTarget() == inputFile, getLabel()); + Preconditions.checkArgument(getConfiguration() == null, getLabel()); + this.artifact = artifact; + + if (inputFile.getLicense() != License.NO_LICENSE) { + licenses = NestedSetBuilder.create(Order.LINK_ORDER, + new TargetLicense(getLabel(), inputFile.getLicense())); + } else { + licenses = NestedSetBuilder.emptySet(Order.LINK_ORDER); + } + } + + @Override + public InputFile getTarget() { + return (InputFile) super.getTarget(); + } + + @Override + public Artifact getArtifact() { + return artifact; + } + + @Override + public String toString() { + return "InputFileConfiguredTarget(" + getTarget().getLabel() + ")"; + } + + @Override + public final NestedSet<TargetLicense> getTransitiveLicenses() { + return licenses; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/LabelAndConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/LabelAndConfiguration.java new file mode 100644 index 0000000..66efba3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/LabelAndConfiguration.java
@@ -0,0 +1,76 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Objects; + +import javax.annotation.Nullable; + +/** +* A (label,configuration) pair. +*/ +public final class LabelAndConfiguration { + private final Label label; + @Nullable + private final BuildConfiguration configuration; + + private LabelAndConfiguration(Label label, @Nullable BuildConfiguration configuration) { + this.label = Preconditions.checkNotNull(label); + this.configuration = configuration; + } + + public LabelAndConfiguration(ConfiguredTarget rule) { + this(rule.getTarget().getLabel(), rule.getConfiguration()); + } + + public Label getLabel() { + return label; + } + + @Nullable + public BuildConfiguration getConfiguration() { + return configuration; + } + + @Override + public int hashCode() { + int configVal = configuration == null ? 79 : configuration.hashCode(); + return 31 * label.hashCode() + configVal; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof LabelAndConfiguration)) { + return false; + } + LabelAndConfiguration other = (LabelAndConfiguration) obj; + return Objects.equals(label, other.label) && Objects.equals(configuration, other.configuration); + } + + public static LabelAndConfiguration of( + Label label, @Nullable BuildConfiguration configuration) { + return new LabelAndConfiguration(label, configuration); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/LabelExpander.java b/src/main/java/com/google/devtools/build/lib/analysis/LabelExpander.java new file mode 100644 index 0000000..89ce2e7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/LabelExpander.java
@@ -0,0 +1,181 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.CharMatcher; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Helper class encapsulating string scanning state used during "heuristic" + * expansion of labels embedded within rules. + */ +public final class LabelExpander { + /** + * An exception that is thrown when a label is expanded to zero or multiple + * files during expansion. + */ + public static class NotUniqueExpansionException extends Exception { + public NotUniqueExpansionException(int sizeOfResultSet, String labelText) { + super("heuristic label expansion found '" + labelText + "', which expands to " + + sizeOfResultSet + " files" + + (sizeOfResultSet > 1 + ? ", please use $(locations " + labelText + ") instead" + : "")); + } + } + + // This is a utility class, no need to instantiate. + private LabelExpander() {} + + /** + * CharMatcher to determine if a given character is valid for labels. + * + * <p>The Build Concept Reference additionally allows '=' and ',' to appear in labels, + * but for the purposes of the heuristic, this function does not, as it would cause + * "--foo=:rule1,:rule2" to scan as a single possible label, instead of three + * ("--foo", ":rule1", ":rule2"). + */ + private static final CharMatcher LABEL_CHAR_MATCHER = + CharMatcher.inRange('a', 'z') + .or(CharMatcher.inRange('A', 'Z')) + .or(CharMatcher.inRange('0', '9')) + .or(CharMatcher.anyOf(":/_.-+" + PathFragment.SEPARATOR_CHAR)); + + /** + * Expands all references to labels embedded within a string using the + * provided expansion mapping from labels to artifacts. + * + * <p>Since this pass is heuristic, references to non-existent labels (such + * as arbitrary words) or invalid labels are simply ignored and are unchanged + * in the output. However, if the heuristic discovers a label, which + * identifies an existing target producing zero or multiple files, an error + * is reported. + * + * @param expression the expression to expand. + * @param labelMap the mapping from labels to artifacts, whose relative path + * is to be used as the expansion. + * @param labelResolver the {@code Label} that can resolve label strings + * to {@code Label} objects. The resolved label is either relative to + * {@code labelResolver} or is a global label (i.e. starts with "//"). + * @return the expansion of the string. + * @throws NotUniqueExpansionException if a label that is present in the + * mapping expands to zero or multiple files. + */ + public static <T extends Iterable<Artifact>> String expand(@Nullable String expression, + Map<Label, T> labelMap, Label labelResolver) throws NotUniqueExpansionException { + if (Strings.isNullOrEmpty(expression)) { + return ""; + } + Preconditions.checkNotNull(labelMap); + Preconditions.checkNotNull(labelResolver); + + int offset = 0; + StringBuilder result = new StringBuilder(); + while (offset < expression.length()) { + String labelText = scanLabel(expression, offset); + if (labelText != null) { + offset += labelText.length(); + result.append(tryResolvingLabelTextToArtifactPath(labelText, labelMap, labelResolver)); + } else { + result.append(expression.charAt(offset)); + offset++; + } + } + return result.toString(); + } + + /** + * Tries resolving a label text to a full label for the associated {@code + * Artifact}, using the provided mapping. + * + * <p>The method succeeds if the label text can be resolved to a {@code + * Label} object, which is present in the {@code labelMap} and maps to + * exactly one {@code Artifact}. + * + * @param labelText the text to resolve. + * @param labelMap the mapping from labels to artifacts, whose relative path + * is to be used as the expansion. + * @param labelResolver the {@code Label} that can resolve label strings + * to {@code Label} objects. The resolved label is either relative to + * {@code labelResolver} or is a global label (i.e. starts with "//"). + * @return an absolute label to an {@code Artifact} if the resolving was + * successful or the original label text. + * @throws NotUniqueExpansionException if a label that is present in the + * mapping expands to zero or multiple files. + */ + private static <T extends Iterable<Artifact>> String tryResolvingLabelTextToArtifactPath( + String labelText, Map<Label, T> labelMap, Label labelResolver) + throws NotUniqueExpansionException { + Label resolvedLabel = resolveLabelText(labelText, labelResolver); + if (resolvedLabel != null) { + Iterable<Artifact> artifacts = labelMap.get(resolvedLabel); + if (artifacts != null) { // resolvedLabel identifies an existing target + List<String> locations = new ArrayList<>(); + Artifact.addExecPaths(artifacts, locations); + int resultSetSize = locations.size(); + if (resultSetSize == 1) { + return Iterables.getOnlyElement(locations); // success! + } else { + throw new NotUniqueExpansionException(resultSetSize, labelText); + } + } + } + return labelText; + } + + /** + * Resolves a string to a label text. Uses {@code labelResolver} to do so. + * The result is either relative to {@code labelResolver} or is an absolute + * label. In case of an invalid label text, the return value is null. + */ + private static Label resolveLabelText(String labelText, Label labelResolver) { + try { + return labelResolver.getRelative(labelText); + } catch (Label.SyntaxException e) { + // It's a heuristic, so quietly ignore "errors". Because Label.getRelative never + // returns null, we can use null to indicate an error. + return null; + } + } + + /** + * Scans the argument string from a given start position until the name of a + * potential label has been consumed, then returns the label text. If + * the expression contains no possible label starting at the start position, + * the return value is null. + */ + private static String scanLabel(String expression, int start) { + int offset = start; + while (offset < expression.length() && LABEL_CHAR_MATCHER.matches(expression.charAt(offset))) { + ++offset; + } + if (offset > start) { + return expression.substring(start, offset); + } else { + return null; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/LanguageDependentFragment.java b/src/main/java/com/google/devtools/build/lib/analysis/LanguageDependentFragment.java new file mode 100644 index 0000000..d42d625 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/LanguageDependentFragment.java
@@ -0,0 +1,109 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Set; + +/** + * Transitive info provider for rules that behave differently when used from + * different languages. + * + * <p>Most rules generate code for a particular language or are totally language independent. + * Some rules, however, behave differently when depended upon from different languages. + * They might generate different libraries when used from different languages (and with + * different API versions). This interface allows code sharing between implementations. + * + * <p>This provider is not really a roll-up of transitive information. + */ +@Immutable +public final class LanguageDependentFragment implements TransitiveInfoProvider { + /** + * A language that can be supported by a multi-language configured target. + * + * <p>Note that no {@code hashCode}/{@code equals} methods are provided, because these + * objects are expected to be compared for object identity, which is the default. + */ + public static final class LibraryLanguage { + private final String displayName; + + public LibraryLanguage(String displayName) { + this.displayName = displayName; + } + + @Override + public String toString() { + return displayName; + } + } + + private final Label label; + private final ImmutableSet<LibraryLanguage> languages; + + public LanguageDependentFragment(Label label, Set<LibraryLanguage> languages) { + this.label = label; + this.languages = ImmutableSet.copyOf(languages); + } + + /** + * Returns the label that is associated with this piece of information. + * + * <p>This is usually the label of the target that provides the information. + */ + public Label getLabel() { + return label; + } + + /** + * Returns a set of the languages the ConfiguredTarget generates output for. + * For use only by rules that directly depend on this library via a "deps" attribute. + */ + public ImmutableSet<LibraryLanguage> getSupportedLanguages() { + return languages; + } + + /** + * Routines for verifying that dependency provide the right output. + */ + public static final class Checker { + /** + * Checks that given dep supports the given language. + */ + public static boolean depSupportsLanguage( + RuleContext context, LanguageDependentFragment dep, LibraryLanguage language) { + if (dep.getSupportedLanguages().contains(language)) { + return true; + } else { + context.attributeError( + "deps", String.format("'%s' does not produce output for %s", dep.getLabel(), language)); + return false; + } + } + + /** + * Checks that all LanguageDependentFragment support the given language. + */ + public static void depsSupportsLanguage(RuleContext context, LibraryLanguage language) { + for (LanguageDependentFragment dep : + context.getPrerequisites("deps", Mode.TARGET, LanguageDependentFragment.class)) { + depSupportsLanguage(context, dep, language); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/LicensesProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/LicensesProvider.java new file mode 100644 index 0000000..548a1f2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/LicensesProvider.java
@@ -0,0 +1,88 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.packages.License; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Objects; + +/** + * A {@link ConfiguredTarget} that has licensed targets in its transitive closure. + */ +public interface LicensesProvider extends TransitiveInfoProvider { + + /** + * The set of label - license associations in the transitive closure. + * + * <p>Always returns an empty set if {@link BuildConfiguration#checkLicenses()} is false. + */ + NestedSet<TargetLicense> getTransitiveLicenses(); + + /** + * License association for a particular target. + */ + public static final class TargetLicense { + + private final Label label; + private final License license; + + public TargetLicense(Label label, License license) { + Preconditions.checkNotNull(label); + Preconditions.checkNotNull(license); + this.label = label; + this.license = license; + } + + /** + * Returns the label of the associated target. + */ + public Label getLabel() { + return label; + } + + /** + * Returns the license for the target. + */ + public License getLicense() { + return license; + } + + @Override + public int hashCode() { + return Objects.hash(label, license); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof TargetLicense)) { + return false; + } + TargetLicense other = (TargetLicense) obj; + return label.equals(other.label) && license.equals(other.license); + } + + @Override + public String toString() { + return label + " => " + license; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/LicensesProviderImpl.java b/src/main/java/com/google/devtools/build/lib/analysis/LicensesProviderImpl.java new file mode 100644 index 0000000..ffdf9fd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/LicensesProviderImpl.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A {@link ConfiguredTarget} that has licensed targets in its transitive closure. + */ +@Immutable +public final class LicensesProviderImpl implements LicensesProvider { + public static final LicensesProvider EMPTY = + new LicensesProviderImpl(NestedSetBuilder.<TargetLicense>emptySet(Order.LINK_ORDER)); + + private final NestedSet<TargetLicense> transitiveLicenses; + + public LicensesProviderImpl(NestedSet<TargetLicense> transitiveLicenses) { + this.transitiveLicenses = transitiveLicenses; + } + + @Override + public NestedSet<TargetLicense> getTransitiveLicenses() { + return transitiveLicenses; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/LocationExpander.java b/src/main/java/com/google/devtools/build/lib/analysis/LocationExpander.java new file mode 100644 index 0000000..8feb28e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/LocationExpander.java
@@ -0,0 +1,260 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Joiner; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Expands $(location) tags inside target attributes. + * You can specify something like this in the BUILD file: + * + * somerule(name='some name', + * someopt = [ '$(location //mypackage:myhelper)' ], + * ...) + * + * and location will be substituted with //mypackage:myhelper executable output. + * Note that //mypackage:myhelper should have just one output. + */ +public class LocationExpander { + private static final int MAX_PATHS_SHOWN = 5; + private static final String LOCATION = "$(location"; + private final RuleContext ruleContext; + private Map<Label, Collection<Artifact>> locationMap; + private boolean allowDataAttributeEntriesInLabel = false; + + /** + * Creates location expander helper bound to specific target and with default + * location map. + * + * @param ruleContext BUILD rule + */ + public LocationExpander(RuleContext ruleContext) { + this(ruleContext, false); + } + + public LocationExpander(RuleContext ruleContext, + boolean allowDataAttributeEntriesInLabel) { + this.ruleContext = ruleContext; + this.allowDataAttributeEntriesInLabel = allowDataAttributeEntriesInLabel; + } + + public Map<Label, Collection<Artifact>> getLocationMap() { + if (locationMap == null) { + locationMap = buildLocationMap(ruleContext, allowDataAttributeEntriesInLabel); + } + return locationMap; + } + + /** + * Expands attribute's location and locations tags based on the target and + * location map. + * + * @param attrName name of the attribute + * @param attrValue initial value of the attribute + * @return attribute value with expanded location tags or original value in + * case of errors + */ + public String expand(String attrName, String attrValue) { + int restart = 0; + + int attrLength = attrValue.length(); + StringBuilder result = new StringBuilder(attrValue.length()); + + while (true) { + // (1) find '$(location ' or '$(locations ' + String message = "$(location)"; + boolean multiple = false; + int start = attrValue.indexOf(LOCATION, restart); + int scannedLength = LOCATION.length(); + if (start == -1 || start + scannedLength == attrLength) { + result.append(attrValue.substring(restart)); + break; + } + + if (attrValue.charAt(start + scannedLength) == 's') { + scannedLength++; + if (start + scannedLength == attrLength) { + result.append(attrValue.substring(restart)); + break; + } + message = "$(locations)"; + multiple = true; + } + + if (attrValue.charAt(start + scannedLength) != ' ') { + result.append(attrValue.substring(restart, start + scannedLength)); + restart = start + scannedLength; + continue; + } + scannedLength++; + + int end = attrValue.indexOf(')', start + scannedLength); + if (end == -1) { + ruleContext.attributeError(attrName, "unterminated " + message + " expression"); + return attrValue; + } + + // (2) parse label + String labelText = attrValue.substring(start + scannedLength, end); + Label label; + try { + label = ruleContext.getLabel().getRelative(labelText); + } catch (Label.SyntaxException e) { + ruleContext.attributeError(attrName, + "invalid label in " + message + " expression: " + e.getMessage()); + return attrValue; + } + + // (3) replace with singleton artifact, iff unique. + Collection<Artifact> artifacts = getLocationMap().get(label); + if (artifacts == null) { + ruleContext.attributeError(attrName, + "label '" + label + "' in " + message + " expression is not a " + + "declared prerequisite of this rule"); + return attrValue; + } + List<String> paths = getPaths(artifacts); + if (paths.isEmpty()) { + ruleContext.attributeError(attrName, + "label '" + label + "' in " + message + " expression expands to no " + + "files"); + return attrValue; + } + + result.append(attrValue.substring(restart, start)); + if (multiple) { + Collections.sort(paths); + Joiner.on(' ').appendTo(result, paths); + } else { + if (paths.size() > 1) { + ruleContext.attributeError(attrName, + String.format( + "label '%s' in %s expression expands to more than one file, " + + "please use $(locations %s) instead. Files (at most %d shown) are: %s", + label, message, label, + MAX_PATHS_SHOWN, Iterables.limit(paths, MAX_PATHS_SHOWN))); + return attrValue; + } + result.append(Iterables.getOnlyElement(paths)); + } + restart = end + 1; + } + return result.toString(); + } + + /** + * Extracts all possible target locations from target specification. + * + * @param ruleContext BUILD target object + * @return map of all possible target locations + */ + private static Map<Label, Collection<Artifact>> buildLocationMap(RuleContext ruleContext, + boolean allowDataAttributeEntriesInLabel) { + Map<Label, Collection<Artifact>> locationMap = new HashMap<>(); + + // Add all destination locations. + for (OutputFile out : ruleContext.getRule().getOutputFiles()) { + mapGet(locationMap, out.getLabel()).add(ruleContext.createOutputArtifact(out)); + } + + if (ruleContext.getRule().isAttrDefined("srcs", Type.LABEL_LIST)) { + for (FileProvider src : ruleContext + .getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) { + Iterables.addAll(mapGet(locationMap, src.getLabel()), src.getFilesToBuild()); + } + } + + // Add all locations associated with dependencies and tools + List<FilesToRunProvider> depsDataAndTools = new ArrayList<>(); + if (ruleContext.getRule().isAttrDefined("deps", Type.LABEL_LIST)) { + Iterables.addAll(depsDataAndTools, + ruleContext.getPrerequisites("deps", Mode.DONT_CHECK, FilesToRunProvider.class)); + } + if (allowDataAttributeEntriesInLabel + && ruleContext.getRule().isAttrDefined("data", Type.LABEL_LIST)) { + Iterables.addAll(depsDataAndTools, + ruleContext.getPrerequisites("data", Mode.DATA, FilesToRunProvider.class)); + } + if (ruleContext.getRule().isAttrDefined("tools", Type.LABEL_LIST)) { + Iterables.addAll(depsDataAndTools, + ruleContext.getPrerequisites("tools", Mode.HOST, FilesToRunProvider.class)); + } + + for (FilesToRunProvider dep : depsDataAndTools) { + Label label = dep.getLabel(); + Artifact executableArtifact = dep.getExecutable(); + + // If the label has an executable artifact add that to the multimaps. + if (executableArtifact != null) { + mapGet(locationMap, label).add(executableArtifact); + } else { + mapGet(locationMap, label).addAll(dep.getFilesToRun()); + } + } + return locationMap; + } + + /** + * Extracts list of all executables associated with given collection of label + * artifacts. + * + * @param artifacts to get the paths of + * @return all associated executable paths + */ + private static List<String> getPaths(Collection<Artifact> artifacts) { + List<String> paths = Lists.newArrayListWithCapacity(artifacts.size()); + for (Artifact artifact : artifacts) { + PathFragment execPath = artifact.getExecPath(); + if (execPath != null) { // omit middlemen etc + paths.add(execPath.getPathString()); + } + } + return paths; + } + + /** + * Returns the value in the specified map corresponding to 'key', creating and + * inserting an empty container if absent. We use Map not Multimap because + * we need to distinguish the cases of "empty value" and "absent key". + * + * @return the value in the specified map corresponding to 'key' + */ + private static <K, V> Collection<V> mapGet(Map<K, Collection<V>> map, K key) { + Collection<V> values = map.get(key); + if (values == null) { + // We use sets not lists, because it's conceivable that the same label + // could appear twice, in "srcs" and "deps". + values = Sets.newHashSet(); + map.put(key, values); + } + return values; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/MakeEnvironmentEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/MakeEnvironmentEvent.java new file mode 100644 index 0000000..f4b9ca8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/MakeEnvironmentEvent.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +/** + * This event is fired once the global make environment is available. + */ +public final class MakeEnvironmentEvent { + + private final Map<String, String> makeEnvMap; + + /** + * Construct the event. + */ + public MakeEnvironmentEvent(Map<String, String> makeEnv) { + makeEnvMap = ImmutableMap.copyOf(makeEnv); + } + + /** + * Returns make environment variable names and values as a map. + */ + public Map<String, String> getMakeEnvMap() { + return makeEnvMap; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/MakeVariableExpander.java b/src/main/java/com/google/devtools/build/lib/analysis/MakeVariableExpander.java new file mode 100644 index 0000000..55366da --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/MakeVariableExpander.java
@@ -0,0 +1,201 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +/** + * MakeVariableExpander defines a utility method, <code>expand</code>, for + * expanding references to "Make" variables embedded within a string. The + * caller provides a Context instance which defines the expansion of each + * variable. + * + * <p>Note that neither <code>$(location x)</code> nor Make-isms are treated + * specially in any way by this class. + */ +public class MakeVariableExpander { + + private final char[] buffer; + private final int length; + private int offset; + + private MakeVariableExpander(String expression) { + buffer = expression.toCharArray(); + length = buffer.length; + offset = 0; + } + + /** + * Interface to be implemented by callers of MakeVariableExpander which + * defines the expansion of each "Make" variable. + */ + public interface Context { + + /** + * Returns the expansion of the specified "Make" variable. + * + * @param var the variable to expand. + * @return the expansion of the variable. + * @throws ExpansionException if the variable "var" was not defined or + * there was any other error while expanding "var". + */ + String lookupMakeVariable(String var) throws ExpansionException; + } + + /** + * Exception thrown by MakeVariableExpander.Context.expandVariable when an + * unknown variable is passed. + */ + public static class ExpansionException extends Exception { + public ExpansionException(String message) { + super(message); + } + } + + /** + * Expands all references to "Make" variables embedded within string "expr", + * using the provided Context instance to expand individual variables. + * + * @param expression the string to expand. + * @param context the context which defines the expansion of each individual + * variable. + * @return the expansion of "expr". + * @throws ExpansionException if "expr" contained undefined or ill-formed + * variables references. + */ + public static String expand(String expression, Context context) throws ExpansionException { + if (expression.indexOf('$') < 0) { + return expression; + } + return expand(expression, context, 0); + } + + /** + * If the string contains a single variable, return the expansion of that variable. + * Otherwise, return null. + */ + public static String expandSingleVariable(String expression, Context context) + throws ExpansionException { + String var = new MakeVariableExpander(expression).getSingleVariable(); + return (var != null) ? context.lookupMakeVariable(var) : null; + } + + // Helper method for counting recursion depth. + private static String expand(String expression, Context context, int depth) + throws ExpansionException { + if (depth > 10) { // plenty! + throw new ExpansionException("potentially unbounded recursion during " + + "expansion of '" + expression + "'"); + } + return new MakeVariableExpander(expression).expand(context, depth); + } + + private String expand(Context context, int depth) throws ExpansionException { + StringBuilder result = new StringBuilder(); + while (offset < length) { + char c = buffer[offset]; + if (c == '$') { // variable + offset++; + if (offset >= length) { + throw new ExpansionException("unterminated $"); + } + if (buffer[offset] == '$') { + result.append('$'); + } else { + String var = scanVariable(); + String value = context.lookupMakeVariable(var); + // To prevent infinite recursion for the ignored shell variables + if (!value.equals(var)) { + // recursively expand using Make's ":=" semantics: + value = expand(value, context, depth + 1); + } + result.append(value); + } + } else { + result.append(c); + } + offset++; + } + return result.toString(); + } + + /** + * Starting at the current position, scans forward until the name of a Make + * variable has been consumed. Returns the variable name and advances the + * position. If the variable is a potential shell variable returns the shell + * variable expression itself, so that we can let the shell handle the + * expansion. + * + * @return the name of the variable found at the current point. + * @throws ExpansionException if the variable reference was ill-formed. + */ + private String scanVariable() throws ExpansionException { + char c = buffer[offset]; + switch (c) { + case '(': { // $(SRCS) + offset++; + int start = offset; + while (offset < length && buffer[offset] != ')') { + offset++; + } + if (offset >= length) { + throw new ExpansionException("unterminated variable reference"); + } + return new String(buffer, start, offset - start); + } + case '{': { // ${SRCS} + offset++; + int start = offset; + while (offset < length && buffer[offset] != '}') { + offset++; + } + if (offset >= length) { + throw new ExpansionException("unterminated variable reference"); + } + String expr = new String(buffer, start, offset - start); + throw new ExpansionException("'${" + expr + "}' syntax is not supported; use '$(" + expr + + ")' instead for \"Make\" variables, or escape the '$' as " + + "'$$' if you intended this for the shell"); + } + case '@': + case '<': + case '^': + return String.valueOf(c); + default: { + int start = offset; + while (offset + 1 < length && Character.isJavaIdentifierPart(buffer[offset + 1])) { + offset++; + } + String expr = new String(buffer, start, offset + 1 - start); + throw new ExpansionException("'$" + expr + "' syntax is not supported; use '$(" + expr + + ")' instead for \"Make\" variables, or escape the '$' as " + + "'$$' if you intended this for the shell"); + } + } + } + + /** + * @return the variable name if the variable spans from offset to the end of + * the buffer, otherwise return null. + * @throws ExpansionException if the variable reference was ill-formed. + */ + public String getSingleVariable() throws ExpansionException { + if (buffer[offset] == '$') { + offset++; + String result = scanVariable(); + if (offset + 1 == length) { + return result; + } + } + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/MiddlemanProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/MiddlemanProvider.java new file mode 100644 index 0000000..d8425f2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/MiddlemanProvider.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A provider class that supplies an aggregating middleman to the targets that depend on it. + */ +@Immutable +public final class MiddlemanProvider implements TransitiveInfoProvider { + + private final NestedSet<Artifact> middlemanArtifact; + + public MiddlemanProvider(NestedSet<Artifact> middlemanArtifact) { + this.middlemanArtifact = middlemanArtifact; + } + + /** + * Returns the middleman for the files produced by the transitive info collection. + */ + public NestedSet<Artifact> getMiddlemanArtifact() { + return middlemanArtifact; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/NoSuchConfiguredTargetException.java b/src/main/java/com/google/devtools/build/lib/analysis/NoSuchConfiguredTargetException.java new file mode 100644 index 0000000..2e9bf8c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/NoSuchConfiguredTargetException.java
@@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.syntax.Label; + +/** + * Exception indicating that the required configured target is not in the + * analysis cache. + */ +public class NoSuchConfiguredTargetException extends NoSuchThingException { + public NoSuchConfiguredTargetException(Label label, BuildConfiguration configuration) { + super("not in cache: " + label + " " + configuration); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/OutputFileConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/OutputFileConfiguredTarget.java new file mode 100644 index 0000000..51122e2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/OutputFileConfiguredTarget.java
@@ -0,0 +1,80 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProviderImpl; + +/** + * A ConfiguredTarget for an OutputFile. + */ +public class OutputFileConfiguredTarget extends FileConfiguredTarget + implements InstrumentedFilesProvider { + + private final TransitiveInfoCollection generatingRule; + + OutputFileConfiguredTarget( + TargetContext targetContext, OutputFile outputFile, + TransitiveInfoCollection generatingRule, Artifact outputArtifact) { + super(targetContext, outputArtifact); + Preconditions.checkArgument(targetContext.getTarget() == outputFile); + this.generatingRule = generatingRule; + } + + @Override + public OutputFile getTarget() { + return (OutputFile) super.getTarget(); + } + + public TransitiveInfoCollection getGeneratingRule() { + return generatingRule; + } + + @Override + public NestedSet<TargetLicense> getTransitiveLicenses() { + return getProvider(LicensesProvider.class, LicensesProviderImpl.EMPTY) + .getTransitiveLicenses(); + } + + @Override + public NestedSet<Artifact> getInstrumentedFiles() { + return getProvider(InstrumentedFilesProvider.class, InstrumentedFilesProviderImpl.EMPTY) + .getInstrumentedFiles(); + } + + @Override + public NestedSet<Artifact> getInstrumentationMetadataFiles() { + return getProvider(InstrumentedFilesProvider.class, InstrumentedFilesProviderImpl.EMPTY) + .getInstrumentationMetadataFiles(); + } + + /** + * Returns the corresponding provider from the generating rule, if it is non-null, or {@code + * defaultValue} otherwise. + */ + private <T extends TransitiveInfoProvider> T getProvider(Class<T> clazz, T defaultValue) { + if (generatingRule != null) { + T result = generatingRule.getProvider(clazz); + if (result != null) { + return result; + } + } + return defaultValue; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PackageGroupConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/PackageGroupConfiguredTarget.java new file mode 100644 index 0000000..75e2981 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/PackageGroupConfiguredTarget.java
@@ -0,0 +1,77 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.common.collect.UnmodifiableIterator; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.PackageGroup; +import com.google.devtools.build.lib.packages.PackageSpecification; +import com.google.devtools.build.lib.syntax.Label; + +/** + * Dummy ConfiguredTarget for package groups. Contains no functionality, since + * package groups are not really first-class Targets. + */ +public final class PackageGroupConfiguredTarget extends AbstractConfiguredTarget + implements PackageSpecificationProvider { + private final NestedSet<PackageSpecification> packageSpecifications; + + PackageGroupConfiguredTarget(TargetContext targetContext, PackageGroup packageGroup) { + super(targetContext); + Preconditions.checkArgument(targetContext.getConfiguration() == null); + + NestedSetBuilder<PackageSpecification> builder = + NestedSetBuilder.stableOrder(); + for (Label label : packageGroup.getIncludes()) { + TransitiveInfoCollection include = targetContext.findDirectPrerequisite( + label, targetContext.getConfiguration()); + PackageSpecificationProvider provider = include == null ? null : + include.getProvider(PackageSpecificationProvider.class); + if (provider == null) { + targetContext.getAnalysisEnvironment().getEventHandler().handle(Event.error(getTarget().getLocation(), + String.format("label '%s' does not refer to a package group", label))); + continue; + } + + builder.addTransitive(provider.getPackageSpecifications()); + } + + builder.addAll(packageGroup.getPackageSpecifications()); + packageSpecifications = builder.build(); + } + + @Override + public PackageGroup getTarget() { + return (PackageGroup) super.getTarget(); + } + + @Override + public NestedSet<PackageSpecification> getPackageSpecifications() { + return packageSpecifications; + } + + @Override + public Object get(String providerKey) { + throw new UnsupportedOperationException(); + } + + @Override + public UnmodifiableIterator<TransitiveInfoProvider> iterator() { + throw new IllegalStateException(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java new file mode 100644 index 0000000..3f852c7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java
@@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.packages.PackageSpecification; + +/** + * A {@link TransitiveInfoProvider} that describes a set of transitive package specifications + * used in package groups. + */ +public interface PackageSpecificationProvider extends TransitiveInfoProvider { + NestedSet<PackageSpecification> getPackageSpecifications(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PrerequisiteArtifacts.java b/src/main/java/com/google/devtools/build/lib/analysis/PrerequisiteArtifacts.java new file mode 100644 index 0000000..8932d5c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/PrerequisiteArtifacts.java
@@ -0,0 +1,106 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; + +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Contains a sequence of prerequisite artifacts and supplies methods for filtering and reporting + * errors on those artifacts. + */ +public final class PrerequisiteArtifacts { + private final RuleContext ruleContext; + private final String attributeName; + private final ImmutableList<Artifact> artifacts; + + private PrerequisiteArtifacts( + RuleContext ruleContext, String attributeName, ImmutableList<Artifact> artifacts) { + this.ruleContext = Preconditions.checkNotNull(ruleContext); + this.attributeName = Preconditions.checkNotNull(attributeName); + this.artifacts = Preconditions.checkNotNull(artifacts); + } + + static PrerequisiteArtifacts get(RuleContext ruleContext, String attributeName, Mode mode) { + Set<Artifact> result = new LinkedHashSet<>(); + for (FileProvider target : + ruleContext.getPrerequisites(attributeName, mode, FileProvider.class)) { + Iterables.addAll(result, target.getFilesToBuild()); + } + return new PrerequisiteArtifacts(ruleContext, attributeName, ImmutableList.copyOf(result)); + } + + /** + * Returns the artifacts this instance contains in an {@link ImmutableList}. + */ + public ImmutableList<Artifact> list() { + return artifacts; + } + + private PrerequisiteArtifacts filter(Predicate<String> fileType, boolean errorsForNonMatching) { + ImmutableList.Builder<Artifact> filtered = new ImmutableList.Builder<Artifact>(); + + for (Artifact artifact : artifacts) { + if (fileType.apply(artifact.getFilename())) { + filtered.add(artifact); + } else if (errorsForNonMatching) { + ruleContext.attributeError( + attributeName, + String.format("%s does not match expected type: %s", artifact, fileType)); + } + } + + return new PrerequisiteArtifacts(ruleContext, attributeName, filtered.build()); + } + + /** + * Returns an equivalent instance but only containing artifacts of the given type, reporting + * errors for non-matching artifacts. + */ + public PrerequisiteArtifacts errorsForNonMatching(FileType fileType) { + return filter(fileType, /*errorsForNonMatching=*/true); + } + + /** + * Returns an equivalent instance but only containing artifacts of the given types, reporting + * errors for non-matching artifacts. + */ + public PrerequisiteArtifacts errorsForNonMatching(FileTypeSet fileTypeSet) { + return filter(fileTypeSet, /*errorsForNonMatching=*/true); + } + + /** + * Returns an equivalent instance but only containing artifacts of the given type. + */ + public PrerequisiteArtifacts filter(FileType fileType) { + return filter(fileType, /*errorsForNonMatching=*/false); + } + + /** + * Returns an equivalent instance but only containing artifacts of the given types. + */ + public PrerequisiteArtifacts filter(FileTypeSet fileTypeSet) { + return filter(fileTypeSet, /*errorsForNonMatching=*/false); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PrintActionVisitor.java b/src/main/java/com/google/devtools/build/lib/analysis/PrintActionVisitor.java new file mode 100644 index 0000000..c852734 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/PrintActionVisitor.java
@@ -0,0 +1,66 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionGraphVisitor; +import com.google.devtools.build.lib.actions.ActionOwner; + +import java.util.List; + +/** + * A bipartite graph visitor which accumulates actions with matching mnemonics for a target. + */ +public final class PrintActionVisitor extends ActionGraphVisitor { + private final ConfiguredTarget target; + private final List<Action> actions; + private final Predicate<Action> actionMnemonicMatcher; + private final String targetConfigurationKey; + + /** + * Creates a new visitor for the actions associated with the given target that have a matching + * mnemonic. + */ + public PrintActionVisitor(ActionGraph actionGraph, ConfiguredTarget target, + Predicate<Action> actionMnemonicMatcher) { + super(actionGraph); + this.target = target; + this.actionMnemonicMatcher = actionMnemonicMatcher; + actions = Lists.newArrayList(); + targetConfigurationKey = target.getConfiguration().shortCacheKey(); + } + + @Override + protected boolean shouldVisit(Action action) { + ActionOwner owner = action.getOwner(); + return owner != null && target.getLabel().equals(owner.getLabel()) + && targetConfigurationKey.equals(owner.getConfigurationShortCacheKey()); + } + + @Override + protected void visitAction(Action action) { + if (actionMnemonicMatcher.apply(action)) { + actions.add(action); + } + } + + /** Retrieves the collected actions since this method was last called and clears the list. */ + public ImmutableList<Action> getActions() { + return ImmutableList.copyOf(actions); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PseudoAction.java b/src/main/java/com/google/devtools/build/lib/analysis/PseudoAction.java new file mode 100644 index 0000000..00d43a3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/PseudoAction.java
@@ -0,0 +1,95 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.protobuf.GeneratedMessage.GeneratedExtension; +import com.google.protobuf.MessageLite; + +import java.util.Collection; +import java.util.UUID; + +/** + * An action that is inserted into the build graph only to provide info + * about rules to extra_actions. + */ +public class PseudoAction<InfoType extends MessageLite> extends AbstractAction { + + private final UUID uuid; + private final String mnemonic; + private final GeneratedExtension<ExtraActionInfo, InfoType> infoExtension; + private final InfoType info; + + public PseudoAction(UUID uuid, ActionOwner owner, + Collection<Artifact> inputs, Collection<Artifact> outputs, + String mnemonic, + GeneratedExtension<ExtraActionInfo, InfoType> infoExtension, InfoType info) { + super(owner, inputs, outputs); + this.uuid = uuid; + this.mnemonic = mnemonic; + this.infoExtension = infoExtension; + this.info = info; + } + + @Override + public String describeStrategy(Executor executor) { + return null; + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException { + throw new ActionExecutionException( + mnemonic + "ExtraAction should not be executed.", this, false); + } + + @Override + public String getMnemonic() { + return mnemonic; + } + + @Override + protected String computeKey() { + return new Fingerprint() + .addUUID(uuid) + .addBytes(info.toByteArray()) + .hexDigestAndReset(); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + + @Override + public ExtraActionInfo.Builder getExtraActionInfo() { + return super.getExtraActionInfo().setExtension(infoExtension, info); + } + + public static Artifact getDummyOutput(RuleContext ruleContext) { + return ruleContext.getAnalysisEnvironment().getDerivedArtifact( + ruleContext.getLabel().toPathFragment().replaceName( + ruleContext.getLabel().getName() + ".extra_action_dummy"), + ruleContext.getConfiguration().getGenfilesDirectory()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RedirectChaser.java b/src/main/java/com/google/devtools/build/lib/analysis/RedirectChaser.java new file mode 100644 index 0000000..108a577 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/RedirectChaser.java
@@ -0,0 +1,114 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.packages.AbstractAttributeMapper; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Tool for chasing filegroup redirects. This is mainly intended to be used during + * BuildConfiguration creation. + */ +public final class RedirectChaser { + + /** + * Custom attribute mapper that throws an exception if an attribute's value depends on the + * build configuration. + */ + private static class StaticValuedAttributeMapper extends AbstractAttributeMapper { + public StaticValuedAttributeMapper(Rule rule) { + super(rule.getPackage(), rule.getRuleClassObject(), rule.getLabel(), + rule.getAttributeContainer()); + } + + /** + * Returns the value of the given attribute. + * + * @throws InvalidConfigurationException if the value is configuration-dependent + */ + public <T> T getAndValidate(String attributeName, Type<T> type) + throws InvalidConfigurationException { + if (getSelector(attributeName, type) != null) { + throw new InvalidConfigurationException + ("The value of '" + attributeName + "' cannot be configuration-dependent"); + } + return super.get(attributeName, type); + } + + @Override + protected <T> Iterable<T> visitAttribute(String attributeName, Type<T> type) { + throw new IllegalStateException("Attribute visitation not supported redirect resolution"); + } + } + + /** + * Follows the 'srcs' attribute of the given label recursively. Keeps repeating as long as the + * labels are filegroups with a single srcs entry. + * + * @param env for loading the packages + * @param label the label to start at + * @param name user-meaningful description of the content being resolved + * @return the label which cannot be further resolved + * @throws InvalidConfigurationException if something goes wrong + */ + @Nullable + public static Label followRedirects(ConfigurationEnvironment env, Label label, String name) + throws InvalidConfigurationException { + Set<Label> visitedLabels = new HashSet<>(); + visitedLabels.add(label); + try { + while (true) { + Target possibleRedirect = env.getTarget(label); + if (possibleRedirect == null) { + return null; + } + if ((possibleRedirect instanceof Rule) && + "filegroup".equals(((Rule) possibleRedirect).getRuleClass())) { + List<Label> labels = new StaticValuedAttributeMapper((Rule) possibleRedirect) + .getAndValidate("srcs", Type.LABEL_LIST); + if (labels.size() != 1) { + // We can't distinguish redirects from the final filegroup, so we assume this must be + // the final one. + return label; + } + label = labels.get(0); + if (!visitedLabels.add(label)) { + throw new InvalidConfigurationException("The " + name + " points to a filegroup which " + + "recursively includes itself. The label " + label + " is part of the loop"); + } + } else { + return label; + } + } + } catch (NoSuchPackageException e) { + throw new InvalidConfigurationException(e.getMessage(), e); + } catch (NoSuchTargetException e) { + return label; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTarget.java new file mode 100644 index 0000000..602e949 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTarget.java
@@ -0,0 +1,226 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.UnmodifiableIterator; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider; +import com.google.devtools.build.lib.analysis.config.RunUnder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.Rule; + +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +/** + * A generic implementation of RuleConfiguredTarget. Do not use directly. Use {@link + * RuleConfiguredTargetBuilder} instead. + */ +public final class RuleConfiguredTarget extends AbstractConfiguredTarget { + /** + * The configuration transition for an attribute through which a prerequisite + * is requested. + */ + public enum Mode { + TARGET, + HOST, + DATA, + SPLIT, + DONT_CHECK + } + + private final ImmutableMap<Class<? extends TransitiveInfoProvider>, Object> providers; + private final ImmutableList<Artifact> mandatoryStampFiles; + private final Set<ConfigMatchingProvider> configConditions; + private final ImmutableList<Aspect> aspects; + + RuleConfiguredTarget(RuleContext ruleContext, + ImmutableList<Artifact> mandatoryStampFiles, + ImmutableMap<String, Object> skylarkProviders, + Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers) { + super(ruleContext); + // We don't use ImmutableMap.Builder here to allow augmenting the initial list of 'default' + // providers by passing them in. + Map<Class<? extends TransitiveInfoProvider>, Object> providerBuilder = new LinkedHashMap<>(); + providerBuilder.putAll(providers); + Preconditions.checkState(providerBuilder.containsKey(RunfilesProvider.class)); + Preconditions.checkState(providerBuilder.containsKey(FileProvider.class)); + Preconditions.checkState(providerBuilder.containsKey(FilesToRunProvider.class)); + + providerBuilder.put(SkylarkProviders.class, new SkylarkProviders(skylarkProviders)); + + this.providers = ImmutableMap.copyOf(providerBuilder); + this.mandatoryStampFiles = mandatoryStampFiles; + this.configConditions = ruleContext.getConfigConditions(); + this.aspects = ImmutableList.of(); + + // If this rule is the run_under target, then check that we have an executable; note that + // run_under is only set in the target configuration, and the target must also be analyzed for + // the target configuration. + RunUnder runUnder = getConfiguration().getRunUnder(); + if (runUnder != null && getLabel().equals(runUnder.getLabel())) { + if (getProvider(FilesToRunProvider.class).getExecutable() == null) { + ruleContext.ruleError("run_under target " + runUnder.getLabel() + " is not executable"); + } + } + + // Make sure that all declared output files are also created as artifacts. The + // CachingAnalysisEnvironment makes sure that they all have generating actions. + if (!ruleContext.hasErrors()) { + for (OutputFile out : ruleContext.getRule().getOutputFiles()) { + ruleContext.createOutputArtifact(out); + } + } + } + + /** + * Merge a configured target with its associated aspects. + * + * <p>If aspects are present, the configured target must be created from a rule (instead of e.g. + * an input or an output file). + */ + public static ConfiguredTarget mergeAspects( + ConfiguredTarget base, Iterable<Aspect> aspects) { + if (Iterables.isEmpty(aspects)) { + // If there are no aspects, don't bother with creating a proxy object + return base; + } else { + // Aspects can only be attached to rules for now. This invariant is upheld by + // DependencyResolver#requiredAspects() + return new RuleConfiguredTarget((RuleConfiguredTarget) base, aspects); + } + } + + /** + * Creates an instance based on a configured target and a set of aspects. + */ + private RuleConfiguredTarget(RuleConfiguredTarget base, Iterable<Aspect> aspects) { + super(base.getTarget(), base.getConfiguration()); + + Set<Class<? extends TransitiveInfoProvider>> providers = new HashSet<>(); + + providers.addAll(base.providers.keySet()); + for (Aspect aspect : aspects) { + for (TransitiveInfoProvider aspectProvider : aspect) { + if (!providers.add(aspectProvider.getClass())) { + throw new IllegalStateException( + "Provider " + aspectProvider.getClass() + " provided twice"); + } + } + } + this.providers = base.providers; + this.mandatoryStampFiles = base.mandatoryStampFiles; + this.configConditions = base.configConditions; + this.aspects = ImmutableList.copyOf(aspects); + } + + /** + * The configuration conditions that trigger this rule's configurable attributes. + */ + Set<ConfigMatchingProvider> getConfigConditions() { + return configConditions; + } + + @Override + public <P extends TransitiveInfoProvider> P getProvider(Class<P> providerClass) { + AnalysisUtils.checkProvider(providerClass); + // TODO(bazel-team): Should aspects be allowed to override providers on the configured target + // class? + Object provider = providers.get(providerClass); + if (provider == null) { + for (Aspect aspect : aspects) { + provider = aspect.getProviders().get(providerClass); + if (provider != null) { + break; + } + } + } + + return providerClass.cast(provider); + } + + /** + * Returns a value provided by this target. Only meant to use from Skylark. + */ + @Override + public Object get(String providerKey) { + return getProvider(SkylarkProviders.class).skylarkProviders.get(providerKey); + } + + public ImmutableList<Artifact> getMandatoryStampFiles() { + return mandatoryStampFiles; + } + + @Override + public final Rule getTarget() { + return (Rule) super.getTarget(); + } + + /** + * A helper class for transitive infos provided by Skylark rule implementations. + */ + @Immutable + public static final class SkylarkProviders implements TransitiveInfoProvider { + private final ImmutableMap<String, Object> skylarkProviders; + + private SkylarkProviders(ImmutableMap<String, Object> skylarkProviders) { + Preconditions.checkNotNull(skylarkProviders); + this.skylarkProviders = skylarkProviders; + } + + /** + * Returns the keys for the Skylark providers. + */ + public ImmutableCollection<String> getKeys() { + return skylarkProviders.keySet(); + } + } + + @Override + public UnmodifiableIterator<TransitiveInfoProvider> iterator() { + Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> allProviders = + new LinkedHashMap<>(); + for (int i = aspects.size() - 1; i >= 0; i++) { + for (TransitiveInfoProvider tip : aspects.get(i)) { + allProviders.put(tip.getClass(), tip); + } + } + + for (Map.Entry<Class<? extends TransitiveInfoProvider>, Object> entry : providers.entrySet()) { + allProviders.put(entry.getKey(), entry.getKey().cast(entry.getValue())); + } + + return ImmutableList.copyOf(allProviders.values()).iterator(); + } + + @Override + public String errorMessage(String name) { + return String.format("target (rule class of '%s') doesn't have provider '%s'.", + getTarget().getRuleClass(), name); + } + + @Override + public ImmutableCollection<String> getKeys() { + return ImmutableList.<String>builder().addAll(super.getKeys()) + .addAll(getProvider(SkylarkProviders.class).skylarkProviders.keySet()).build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetBuilder.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetBuilder.java new file mode 100644 index 0000000..b82713f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetBuilder.java
@@ -0,0 +1,423 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ExtraActionArtifactsProvider.ExtraArtifactSet; +import com.google.devtools.build.lib.analysis.LicensesProvider.TargetLicense; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.constraints.ConstraintSemantics; +import com.google.devtools.build.lib.analysis.constraints.EnvironmentCollection; +import com.google.devtools.build.lib.analysis.constraints.SupportedEnvironments; +import com.google.devtools.build.lib.analysis.constraints.SupportedEnvironmentsProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.License; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.extra.ExtraActionMapProvider; +import com.google.devtools.build.lib.rules.extra.ExtraActionSpec; +import com.google.devtools.build.lib.rules.test.ExecutionInfoProvider; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.rules.test.TestActionBuilder; +import com.google.devtools.build.lib.rules.test.TestProvider; +import com.google.devtools.build.lib.rules.test.TestProvider.TestParams; +import com.google.devtools.build.lib.syntax.ClassObject; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.EvalUtils; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +/** + * Builder class for analyzed rule instances (i.e., instances of {@link ConfiguredTarget}). + */ +public final class RuleConfiguredTargetBuilder { + private final RuleContext ruleContext; + private final Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers = + new LinkedHashMap<>(); + private final ImmutableMap.Builder<String, Object> skylarkProviders = ImmutableMap.builder(); + + /** These are supported by all configured targets and need to be specially handled. */ + private NestedSet<Artifact> filesToBuild = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + private RunfilesSupport runfilesSupport; + private Artifact executable; + private ImmutableList<Artifact> mandatoryStampFiles; + private ImmutableSet<Action> actionsWithoutExtraAction = ImmutableSet.of(); + + public RuleConfiguredTargetBuilder(RuleContext ruleContext) { + this.ruleContext = ruleContext; + add(LicensesProvider.class, initializeLicensesProvider()); + add(VisibilityProvider.class, new VisibilityProviderImpl(ruleContext.getVisibility())); + } + + /** + * Constructs the RuleConfiguredTarget instance based on the values set for this Builder. + */ + public ConfiguredTarget build() { + if (ruleContext.getConfiguration().enforceConstraints()) { + checkConstraints(); + } + if (ruleContext.hasErrors()) { + return null; + } + + FilesToRunProvider filesToRunProvider = new FilesToRunProvider(ruleContext.getLabel(), + RuleContext.getFilesToRun(runfilesSupport, filesToBuild), runfilesSupport, executable); + add(FileProvider.class, new FileProvider(ruleContext.getLabel(), filesToBuild)); + add(FilesToRunProvider.class, filesToRunProvider); + + // Create test action and artifacts if target was successfully initialized + // and is a test. + if (TargetUtils.isTestRule(ruleContext.getTarget())) { + Preconditions.checkState(runfilesSupport != null); + add(TestProvider.class, initializeTestProvider(filesToRunProvider)); + } + add(ExtraActionArtifactsProvider.class, initializeExtraActions()); + return new RuleConfiguredTarget( + ruleContext, mandatoryStampFiles, skylarkProviders.build(), providers); + } + + /** + * Invokes Blaze's constraint enforcement system: checks that this rule's dependencies + * support its environments and reports appropriate errors if violations are found. Also + * publishes this rule's supported environments for the rules that depend on it. + */ + private void checkConstraints() { + if (providers.get(SupportedEnvironmentsProvider.class) == null) { + // Note the "environment" rule sets its own SupportedEnvironmentProvider instance, so this + // logic is for "normal" rules that just want to apply default semantics. + EnvironmentCollection supportedEnvironments = + ConstraintSemantics.getSupportedEnvironments(ruleContext); + if (supportedEnvironments != null) { + add(SupportedEnvironmentsProvider.class, new SupportedEnvironments(supportedEnvironments)); + ConstraintSemantics.checkConstraints(ruleContext, supportedEnvironments); + } + } + } + + private TestProvider initializeTestProvider(FilesToRunProvider filesToRunProvider) { + int explicitShardCount = ruleContext.attributes().get("shard_count", Type.INTEGER); + if (explicitShardCount < 0 + && ruleContext.getRule().isAttributeValueExplicitlySpecified("shard_count")) { + ruleContext.attributeError("shard_count", "Must not be negative."); + } + if (explicitShardCount > 50) { + ruleContext.attributeError("shard_count", + "Having more than 50 shards is indicative of poor test organization. " + + "Please reduce the number of shards."); + } + final TestParams testParams = new TestActionBuilder(ruleContext) + .setFilesToRunProvider(filesToRunProvider) + .setInstrumentedFiles(findProvider(InstrumentedFilesProvider.class)) + .setExecutionRequirements(findProvider(ExecutionInfoProvider.class)) + .setShardCount(explicitShardCount) + .build(); + final ImmutableList<String> testTags = + ImmutableList.copyOf(ruleContext.getRule().getRuleTags()); + return new TestProvider(testParams, testTags); + } + + private LicensesProvider initializeLicensesProvider() { + if (!ruleContext.getConfiguration().checkLicenses()) { + return LicensesProviderImpl.EMPTY; + } + + NestedSetBuilder<TargetLicense> builder = NestedSetBuilder.linkOrder(); + BuildConfiguration configuration = ruleContext.getConfiguration(); + Rule rule = ruleContext.getRule(); + License toolOutputLicense = rule.getToolOutputLicense(ruleContext.attributes()); + if (configuration.isHostConfiguration() && toolOutputLicense != null) { + if (toolOutputLicense != License.NO_LICENSE) { + builder.add(new TargetLicense(rule.getLabel(), toolOutputLicense)); + } + } else { + if (rule.getLicense() != License.NO_LICENSE) { + builder.add(new TargetLicense(rule.getLabel(), rule.getLicense())); + } + + for (TransitiveInfoCollection dep : ruleContext.getConfiguredTargetMap().values()) { + LicensesProvider provider = dep.getProvider(LicensesProvider.class); + if (provider != null) { + builder.addTransitive(provider.getTransitiveLicenses()); + } + } + } + + return new LicensesProviderImpl(builder.build()); + } + + /** + * Scans {@code action_listeners} associated with this build to see if any + * {@code extra_actions} should be added to this configured target. If any + * action_listeners are present, a partial visit of the artifact/action graph + * is performed (for as long as actions found are owned by this {@link + * ConfiguredTarget}). Any actions that match the {@code action_listener} + * get an {@code extra_action} associated. The output artifacts of the + * extra_action are reported to the {@link AnalysisEnvironment} for + * bookkeeping. + */ + private ExtraActionArtifactsProvider initializeExtraActions() { + BuildConfiguration configuration = ruleContext.getConfiguration(); + if (configuration.isHostConfiguration()) { + return ExtraActionArtifactsProvider.EMPTY; + } + + ImmutableList<Artifact> extraActionArtifacts = ImmutableList.of(); + NestedSetBuilder<ExtraArtifactSet> builder = NestedSetBuilder.stableOrder(); + + List<Label> actionListenerLabels = configuration.getActionListeners(); + if (!actionListenerLabels.isEmpty() + && ruleContext.getRule().getAttributeDefinition(":action_listener") != null) { + ExtraActionsVisitor visitor = new ExtraActionsVisitor(ruleContext, + computeMnemonicsToExtraActionMap()); + + // The action list is modified within the body of the loop by the addExtraAction() call, + // thus the copy + for (Action action : ImmutableList.copyOf( + ruleContext.getAnalysisEnvironment().getRegisteredActions())) { + if (!actionsWithoutExtraAction.contains(action)) { + visitor.addExtraAction(action); + } + } + + extraActionArtifacts = visitor.getAndResetExtraArtifacts(); + if (!extraActionArtifacts.isEmpty()) { + builder.add(ExtraArtifactSet.of(ruleContext.getLabel(), extraActionArtifacts)); + } + } + + // Add extra action artifacts from dependencies + for (TransitiveInfoCollection dep : ruleContext.getConfiguredTargetMap().values()) { + ExtraActionArtifactsProvider provider = + dep.getProvider(ExtraActionArtifactsProvider.class); + if (provider != null) { + builder.addTransitive(provider.getTransitiveExtraActionArtifacts()); + } + } + + if (mandatoryStampFiles != null && !mandatoryStampFiles.isEmpty()) { + builder.add(ExtraArtifactSet.of(ruleContext.getLabel(), mandatoryStampFiles)); + } + + if (extraActionArtifacts.isEmpty() && builder.isEmpty()) { + return ExtraActionArtifactsProvider.EMPTY; + } + return new ExtraActionArtifactsProvider(extraActionArtifacts, builder.build()); + } + + /** + * Populates the configuration specific mnemonicToExtraActionMap + * based on all action_listers selected by the user (via the blaze option + * --experimental_action_listener=<target>). + */ + private Multimap<String, ExtraActionSpec> computeMnemonicsToExtraActionMap() { + // We copy the multimap here every time. This could be expensive. + Multimap<String, ExtraActionSpec> mnemonicToExtraActionMap = HashMultimap.create(); + for (TransitiveInfoCollection actionListener : + ruleContext.getPrerequisites(":action_listener", Mode.HOST)) { + ExtraActionMapProvider provider = actionListener.getProvider(ExtraActionMapProvider.class); + if (provider == null) { + ruleContext.ruleError(String.format( + "Unable to match experimental_action_listeners to this rule. " + + "Specified target %s is not an action_listener rule", + actionListener.getLabel().toString())); + } else { + mnemonicToExtraActionMap.putAll(provider.getExtraActionMap()); + } + } + return mnemonicToExtraActionMap; + } + + private <T extends TransitiveInfoProvider> T findProvider(Class<T> clazz) { + return clazz.cast(providers.get(clazz)); + } + + /** + * Add a specific provider with a given value. + */ + public <T extends TransitiveInfoProvider> RuleConfiguredTargetBuilder add(Class<T> key, T value) { + return addProvider(key, value); + } + + /** + * Add a specific provider with a given value. + */ + public RuleConfiguredTargetBuilder addProvider( + Class<? extends TransitiveInfoProvider> key, TransitiveInfoProvider value) { + Preconditions.checkNotNull(key); + Preconditions.checkNotNull(value); + AnalysisUtils.checkProvider(key); + providers.put(key, value); + return this; + } + + /** + * Add multiple providers with given values. + */ + public RuleConfiguredTargetBuilder addProviders( + Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers) { + for (Entry<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> provider : + providers.entrySet()) { + addProvider(provider.getKey(), provider.getValue()); + } + return this; + } + + /** + * Add a Skylark transitive info. The provider value must be safe (i.e. a String, a Boolean, + * an Integer, an Artifact, a Label, None, a Java TransitiveInfoProvider or something composed + * from these in Skylark using lists, sets, structs or dicts). Otherwise an EvalException is + * thrown. + */ + public RuleConfiguredTargetBuilder addSkylarkTransitiveInfo( + String name, Object value, Location loc) throws EvalException { + try { + checkSkylarkObjectSafe(value); + } catch (IllegalArgumentException e) { + throw new EvalException(loc, String.format("Value of provider '%s' is of an illegal type: %s", + name, e.getMessage())); + } + skylarkProviders.put(name, value); + return this; + } + + /** + * Add a Skylark transitive info. The provider value must be safe. + */ + public RuleConfiguredTargetBuilder addSkylarkTransitiveInfo( + String name, Object value) { + checkSkylarkObjectSafe(value); + skylarkProviders.put(name, value); + return this; + } + + /** + * Check if the value provided by a Skylark provider is safe (i.e. can be a + * TransitiveInfoProvider value). + */ + private void checkSkylarkObjectSafe(Object value) { + if (!isSimpleSkylarkObjectSafe(value.getClass()) + // Java transitive Info Providers are accessible from Skylark. + || value instanceof TransitiveInfoProvider) { + checkCompositeSkylarkObjectSafe(value); + } + } + + private void checkCompositeSkylarkObjectSafe(Object object) { + if (object instanceof SkylarkList) { + SkylarkList list = (SkylarkList) object; + if (list == SkylarkList.EMPTY_LIST || isSimpleSkylarkObjectSafe(list.getGenericType())) { + // Try not to iterate over the list if avoidable. + return; + } + // The list can be a tuple or a list of composite items. + for (Object listItem : list) { + checkSkylarkObjectSafe(listItem); + } + return; + } else if (object instanceof SkylarkNestedSet) { + // SkylarkNestedSets cannot have composite items. + Class<?> genericType = ((SkylarkNestedSet) object).getGenericType(); + if (!genericType.equals(Object.class) && !isSimpleSkylarkObjectSafe(genericType)) { + throw new IllegalArgumentException(EvalUtils.getDatatypeName(genericType)); + } + return; + } else if (object instanceof Map<?, ?>) { + for (Map.Entry<?, ?> entry : ((Map<?, ?>) object).entrySet()) { + checkSkylarkObjectSafe(entry.getKey()); + checkSkylarkObjectSafe(entry.getValue()); + } + return; + } else if (object instanceof ClassObject) { + ClassObject struct = (ClassObject) object; + for (String key : struct.getKeys()) { + checkSkylarkObjectSafe(struct.getValue(key)); + } + return; + } + throw new IllegalArgumentException(EvalUtils.getDatatypeName(object)); + } + + private boolean isSimpleSkylarkObjectSafe(Class<?> type) { + return type.equals(String.class) + || type.equals(Integer.class) + || type.equals(Boolean.class) + || Artifact.class.isAssignableFrom(type) + || type.equals(Label.class) + || type.equals(Environment.NoneType.class); + } + + /** + * Set the runfiles support for executable targets. + */ + public RuleConfiguredTargetBuilder setRunfilesSupport( + RunfilesSupport runfilesSupport, Artifact executable) { + this.runfilesSupport = runfilesSupport; + this.executable = executable; + return this; + } + + /** + * Set the files to build. + */ + public RuleConfiguredTargetBuilder setFilesToBuild(NestedSet<Artifact> filesToBuild) { + this.filesToBuild = filesToBuild; + return this; + } + + /** + * Set the baseline coverage Artifacts. + */ + public RuleConfiguredTargetBuilder setBaselineCoverageArtifacts( + Collection<Artifact> artifacts) { + return add(BaselineCoverageArtifactsProvider.class, + new BaselineCoverageArtifactsProvider(ImmutableList.copyOf(artifacts))); + } + + /** + * Set the mandatory stamp files. + */ + public RuleConfiguredTargetBuilder setMandatoryStampFiles(ImmutableList<Artifact> files) { + this.mandatoryStampFiles = files; + return this; + } + + /** + * Set the extra action pseudo actions. + */ + public RuleConfiguredTargetBuilder setActionsWithoutExtraAction( + ImmutableSet<Action> actions) { + this.actionsWithoutExtraAction = actions; + return this; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java new file mode 100644 index 0000000..9ad7c70 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java
@@ -0,0 +1,1391 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Multimaps; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.ActionRegistry; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider.PrerequisiteValidator; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider; +import com.google.devtools.build.lib.collect.ImmutableSortedKeyListMultimap; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition; +import com.google.devtools.build.lib.packages.Attribute.SplitTransition; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.FileTarget; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.PackageSpecification; +import com.google.devtools.build.lib.packages.RawAttributeMapper; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleErrorConsumer; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.fileset.FilesetProvider; +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.FilesetEntry; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A helper class for rule implementations building and initialization. Objects of this + * class are intended to be passed to the builder for the configured target, which then creates the + * configured target. + */ +public final class RuleContext extends TargetContext + implements ActionConstructionContext, ActionRegistry, RuleErrorConsumer { + + /** + * The configured version of FilesetEntry. + */ + @Immutable + public static final class ConfiguredFilesetEntry { + private final FilesetEntry entry; + private final TransitiveInfoCollection src; + private final ImmutableList<TransitiveInfoCollection> files; + + ConfiguredFilesetEntry(FilesetEntry entry, TransitiveInfoCollection src) { + this.entry = entry; + this.src = src; + this.files = null; + } + + ConfiguredFilesetEntry(FilesetEntry entry, ImmutableList<TransitiveInfoCollection> files) { + this.entry = entry; + this.src = null; + this.files = files; + } + + public FilesetEntry getEntry() { + return entry; + } + + public TransitiveInfoCollection getSrc() { + return src; + } + + /** + * Targets from FilesetEntry.files, or null if the user omitted it. + */ + @Nullable + public List<TransitiveInfoCollection> getFiles() { + return files; + } + } + + static final String HOST_CONFIGURATION_PROGRESS_TAG = "for host"; + + private final Rule rule; + private final ListMultimap<String, ConfiguredTarget> targetMap; + private final ListMultimap<String, ConfiguredFilesetEntry> filesetEntryMap; + private final Set<ConfigMatchingProvider> configConditions; + private final AttributeMap attributes; + private final ImmutableSet<String> features; + + private ActionOwner actionOwner; + + /* lazily computed cache for Make variables, computed from the above. See get... method */ + private transient ConfigurationMakeVariableContext configurationMakeVariableContext = null; + + private RuleContext(Builder builder, ListMultimap<String, ConfiguredTarget> targetMap, + ListMultimap<String, ConfiguredFilesetEntry> filesetEntryMap, + Set<ConfigMatchingProvider> configConditions, ImmutableSet<String> features) { + super(builder.env, builder.rule, builder.configuration, builder.prerequisiteMap.get(null), + builder.visibility); + this.rule = builder.rule; + this.targetMap = targetMap; + this.filesetEntryMap = filesetEntryMap; + this.configConditions = configConditions; + this.attributes = + ConfiguredAttributeMapper.of(builder.rule, configConditions); + this.features = features; + } + + @Override + public Rule getRule() { + return rule; + } + + /** + * The configuration conditions that trigger this rule's configurable attributes. + */ + Set<ConfigMatchingProvider> getConfigConditions() { + return configConditions; + } + + /** + * Returns the host configuration for this rule; keep in mind that there may be multiple different + * host configurations, even during a single build. + */ + public BuildConfiguration getHostConfiguration() { + BuildConfiguration configuration = getConfiguration(); + // Note: the Builder checks that the configuration is non-null. + return configuration.getConfiguration(ConfigurationTransition.HOST); + } + + /** + * Accessor for the Rule's attribute values. + */ + public AttributeMap attributes() { + return attributes; + } + + /** + * Returns whether this instance is known to have errors at this point during analysis. Do not + * call this method after the initializationHook has returned. + */ + public boolean hasErrors() { + return getAnalysisEnvironment().hasErrors(); + } + + /** + * Returns an immutable map from attribute name to list of configured targets for that attribute. + */ + public ListMultimap<String, ? extends TransitiveInfoCollection> getConfiguredTargetMap() { + return targetMap; + } + + /** + * Returns an immutable map from attribute name to list of fileset entries. + */ + public ListMultimap<String, ConfiguredFilesetEntry> getFilesetEntryMap() { + return filesetEntryMap; + } + + @Override + public ActionOwner getActionOwner() { + if (actionOwner == null) { + actionOwner = new RuleActionOwner(rule, getConfiguration()); + } + return actionOwner; + } + + /** + * Returns a configuration fragment for this this target. + */ + @Nullable + public <T extends Fragment> T getFragment(Class<T> fragment) { + // TODO(bazel-team): The fragments can also be accessed directly through BuildConfiguration. + // Can we lock that down somehow? + Preconditions.checkArgument( + rule.getRuleClassObject().isLegalConfigurationFragment(fragment), + "%s does not have access to %s", rule.getRuleClass(), fragment); + return getConfiguration().getFragment(fragment); + } + + @Override + public ArtifactOwner getOwner() { + return getAnalysisEnvironment().getOwner(); + } + + // TODO(bazel-team): This class could be simpler if Rule and BuildConfiguration classes + // were immutable. Then we would need to store only references those two. + @Immutable + private static final class RuleActionOwner implements ActionOwner { + private final Label label; + private final Location location; + private final String configurationName; + private final String mnemonic; + private final String targetKind; + private final String shortCacheKey; + private final boolean hostConfiguration; + + private RuleActionOwner(Rule rule, BuildConfiguration configuration) { + this.label = rule.getLabel(); + this.location = rule.getLocation(); + this.targetKind = rule.getTargetKind(); + this.configurationName = configuration.getShortName(); + this.mnemonic = configuration.getMnemonic(); + this.shortCacheKey = configuration.shortCacheKey(); + this.hostConfiguration = configuration.isHostConfiguration(); + } + + @Override + public Location getLocation() { + return location; + } + + @Override + public Label getLabel() { + return label; + } + + @Override + public String getConfigurationName() { + return configurationName; + } + + @Override + public String getConfigurationMnemonic() { + return mnemonic; + } + + @Override + public String getConfigurationShortCacheKey() { + return shortCacheKey; + } + + @Override + public String getTargetKind() { + return targetKind; + } + + @Override + public String getAdditionalProgressInfo() { + return hostConfiguration ? HOST_CONFIGURATION_PROGRESS_TAG : null; + } + } + + @Override + public void registerAction(Action... action) { + getAnalysisEnvironment().registerAction(action); + } + + /** + * Convenience function for subclasses to report non-attribute-specific + * errors in the current rule. + */ + @Override + public void ruleError(String message) { + reportError(rule.getLocation(), prefixRuleMessage(message)); + } + + /** + * Convenience function for subclasses to report non-attribute-specific + * warnings in the current rule. + */ + @Override + public void ruleWarning(String message) { + reportWarning(rule.getLocation(), prefixRuleMessage(message)); + } + + /** + * Convenience function for subclasses to report attribute-specific errors in + * the current rule. + * + * <p>If the name of the attribute starts with <code>$</code> + * it is replaced with a string <code>(an implicit dependency)</code>. + */ + @Override + public void attributeError(String attrName, String message) { + reportError(rule.getAttributeLocation(attrName), + prefixAttributeMessage(Attribute.isImplicit(attrName) + ? "(an implicit dependency)" + : attrName, + message)); + } + + /** + * Like attributeError, but does not mark the configured target as errored. + * + * <p>If the name of the attribute starts with <code>$</code> + * it is replaced with a string <code>(an implicit dependency)</code>. + */ + @Override + public void attributeWarning(String attrName, String message) { + reportWarning(rule.getAttributeLocation(attrName), + prefixAttributeMessage(Attribute.isImplicit(attrName) + ? "(an implicit dependency)" + : attrName, + message)); + } + + private String prefixAttributeMessage(String attrName, String message) { + return "in " + attrName + " attribute of " + + rule.getRuleClass() + " rule " + + getLabel() + ": " + message; + } + + private String prefixRuleMessage(String message) { + return "in " + rule.getRuleClass() + " rule " + + getLabel() + ": " + message; + } + + private void reportError(Location location, String message) { + getAnalysisEnvironment().getEventHandler().handle(Event.error(location, message)); + } + + private void reportWarning(Location location, String message) { + getAnalysisEnvironment().getEventHandler().handle(Event.warn(location, message)); + } + + /** + * Returns an artifact beneath the root of either the "bin" or "genfiles" + * tree, whose path is based on the name of this target and the current + * configuration. The choice of which tree to use is based on the rule with + * which this target (which must be an OutputFile or a Rule) is associated. + */ + public Artifact createOutputArtifact() { + return internalCreateOutputArtifact(getTarget()); + } + + /** + * Returns the output artifact of an {@link OutputFile} of this target. + * + * @see #createOutputArtifact() + */ + public Artifact createOutputArtifact(OutputFile out) { + return internalCreateOutputArtifact(out); + } + + /** + * Implementation for {@link #createOutputArtifact()} and + * {@link #createOutputArtifact(OutputFile)}. This is private so that + * {@link #createOutputArtifact(OutputFile)} can have a more specific + * signature. + */ + private Artifact internalCreateOutputArtifact(Target target) { + Root root = getBinOrGenfilesDirectory(); + return getAnalysisEnvironment().getDerivedArtifact(Util.getWorkspaceRelativePath(target), root); + } + + /** + * Returns the root of either the "bin" or "genfiles" + * tree, based on this target and the current configuration. + * The choice of which tree to use is based on the rule with + * which this target (which must be an OutputFile or a Rule) is associated. + */ + public Root getBinOrGenfilesDirectory() { + return rule.hasBinaryOutput() + ? getConfiguration().getBinDirectory() + : getConfiguration().getGenfilesDirectory(); + } + + /** + * Returns the list of transitive info collections that feed into this target through the + * specified attribute. Note that you need to specify the correct mode for the attribute, + * otherwise an assertion will be raised. + */ + public List<? extends TransitiveInfoCollection> getPrerequisites(String attributeName, + Mode mode) { + Attribute attributeDefinition = getRule().getAttributeDefinition(attributeName); + if ((mode == Mode.TARGET) + && (attributeDefinition.getConfigurationTransition() instanceof SplitTransition)) { + // TODO(bazel-team): If you request a split-configured attribute in the target configuration, + // we return only the list of configured targets for the first architecture; this is for + // backwards compatibility with existing code in cases where the call to getPrerequisites is + // deeply nested and we can't easily inject the behavior we want. However, we should fix all + // such call sites. + checkAttribute(attributeName, Mode.SPLIT); + Map<String, ? extends List<? extends TransitiveInfoCollection>> map = + getSplitPrerequisites(attributeName, /*requireSplit=*/false); + return map.isEmpty() + ? ImmutableList.<TransitiveInfoCollection>of() + : map.entrySet().iterator().next().getValue(); + } + + checkAttribute(attributeName, mode); + return targetMap.get(attributeName); + } + + /** + * Returns the a prerequisites keyed by the CPU of their configurations; this method throws an + * exception if the split transition is not active. + */ + public Map<String, ? extends List<? extends TransitiveInfoCollection>> + getSplitPrerequisites(String attributeName) { + return getSplitPrerequisites(attributeName, /*requireSplit*/true); + } + + private Map<String, ? extends List<? extends TransitiveInfoCollection>> + getSplitPrerequisites(String attributeName, boolean requireSplit) { + checkAttribute(attributeName, Mode.SPLIT); + + Attribute attributeDefinition = getRule().getAttributeDefinition(attributeName); + SplitTransition<?> transition = + (SplitTransition<?>) attributeDefinition.getConfigurationTransition(); + List<BuildConfiguration> configurations = + getConfiguration().getTransitions().getSplitConfigurations(transition); + if (configurations.size() == 1) { + // There are two cases here: + // 1. Splitting is enabled, but only one target cpu. + // 2. Splitting is disabled, and no --cpu value was provided on the command line. + // In the first case, the cpu value is non-null, but in the second case it is null. We only + // allow that to proceed if the caller specified that he is going to ignore the cpu value + // anyway. + String cpu = configurations.get(0).getCpu(); + if (cpu == null) { + Preconditions.checkState(!requireSplit); + cpu = "DO_NOT_USE"; + } + return ImmutableMap.of(cpu, targetMap.get(attributeName)); + } + + Set<String> cpus = new HashSet<>(); + for (BuildConfiguration config : configurations) { + // This method should only be called when the split config is enabled on the command line, in + // which case this cpu can't be null. + Preconditions.checkNotNull(config.getCpu()); + cpus.add(config.getCpu()); + } + + // Use an ImmutableListMultimap.Builder here to preserve ordering. + ImmutableListMultimap.Builder<String, TransitiveInfoCollection> result = + ImmutableListMultimap.builder(); + for (TransitiveInfoCollection t : targetMap.get(attributeName)) { + if (t.getConfiguration() != null) { + result.put(t.getConfiguration().getCpu(), t); + } else { + // Source files don't have a configuration, so we add them to all architecture entries. + for (String cpu : cpus) { + result.put(cpu, t); + } + } + } + return Multimaps.asMap(result.build()); + } + + /** + * Returns the specified provider of the prerequisite referenced by the attribute in the + * argument. Note that you need to specify the correct mode for the attribute, otherwise an + * assertion will be raised. If the attribute is empty of it does not support the specified + * provider, returns null. + */ + public <C extends TransitiveInfoProvider> C getPrerequisite( + String attributeName, Mode mode, Class<C> provider) { + TransitiveInfoCollection prerequisite = getPrerequisite(attributeName, mode); + return prerequisite == null ? null : prerequisite.getProvider(provider); + } + + /** + * Returns the transitive info collection that feeds into this target through the specified + * attribute. Note that you need to specify the correct mode for the attribute, otherwise an + * assertion will be raised. Returns null if the attribute is empty. + */ + public TransitiveInfoCollection getPrerequisite(String attributeName, Mode mode) { + checkAttribute(attributeName, mode); + List<? extends TransitiveInfoCollection> elements = targetMap.get(attributeName); + if (elements.size() > 1) { + throw new IllegalStateException(rule.getRuleClass() + " attribute " + attributeName + + " produces more then one prerequisites"); + } + return elements.isEmpty() ? null : elements.get(0); + } + + /** + * Returns all the providers of the specified type that are listed under the specified attribute + * of this target in the BUILD file. + */ + public <C extends TransitiveInfoProvider> Iterable<C> getPrerequisites(String attributeName, + Mode mode, final Class<C> classType) { + AnalysisUtils.checkProvider(classType); + return AnalysisUtils.getProviders(getPrerequisites(attributeName, mode), classType); + } + + /** + * Returns all the providers of the specified type that are listed under the specified attribute + * of this target in the BUILD file, and that contain the specified provider. + */ + public <C extends TransitiveInfoProvider> Iterable<? extends TransitiveInfoCollection> + getPrerequisitesIf(String attributeName, Mode mode, final Class<C> classType) { + AnalysisUtils.checkProvider(classType); + return AnalysisUtils.filterByProvider(getPrerequisites(attributeName, mode), classType); + } + + /** + * Returns the prerequisite referred to by the specified attribute. Also checks whether + * the attribute is marked as executable and that the target referred to can actually be + * executed. + * + * <p>The {@code mode} argument must match the configuration transition specified in the + * definition of the attribute. + * + * @param attributeName the name of the attribute + * @param mode the configuration transition of the attribute + * + * @return the {@link FilesToRunProvider} interface of the prerequisite. + */ + public FilesToRunProvider getExecutablePrerequisite(String attributeName, Mode mode) { + Attribute ruleDefinition = getRule().getAttributeDefinition(attributeName); + + if (ruleDefinition == null) { + throw new IllegalStateException(getRule().getRuleClass() + " attribute " + attributeName + + " is not defined"); + } + if (!ruleDefinition.isExecutable()) { + throw new IllegalStateException(getRule().getRuleClass() + " attribute " + attributeName + + " is not configured to be executable"); + } + + TransitiveInfoCollection prerequisite = getPrerequisite(attributeName, mode); + if (prerequisite == null) { + return null; + } + + FilesToRunProvider result = prerequisite.getProvider(FilesToRunProvider.class); + if (result == null || result.getExecutable() == null) { + attributeError( + attributeName, prerequisite.getLabel() + " does not refer to a valid executable target"); + } + return result; + } + + /** + * Gets an attribute of type STRING_LIST expanding Make variables and + * tokenizes the result. + * + * @param attributeName the name of the attribute to process + * @return a list of strings containing the expanded and tokenized values for the + * attribute + */ + public List<String> getTokenizedStringListAttr(String attributeName) { + if (!getRule().isAttrDefined(attributeName, Type.STRING_LIST)) { + // TODO(bazel-team): This should be an error. + return ImmutableList.of(); + } + List<String> original = attributes().get(attributeName, Type.STRING_LIST); + if (original.isEmpty()) { + return ImmutableList.of(); + } + List<String> tokens = new ArrayList<>(); + for (String token : original) { + tokenizeAndExpandMakeVars(tokens, attributeName, token); + } + return ImmutableList.copyOf(tokens); + } + + /** + * Expands make variables in value and tokenizes the result into tokens. + * + * <p>This methods should be called only during initialization. + */ + public void tokenizeAndExpandMakeVars(List<String> tokens, String attributeName, + String value) { + try { + ShellUtils.tokenize(tokens, expandMakeVariables(attributeName, value)); + } catch (ShellUtils.TokenizationException e) { + attributeError(attributeName, e.getMessage()); + } + } + + /** + * Return a context that maps Make variable names (string) to values (string). + * + * @return a ConfigurationMakeVariableContext. + **/ + public ConfigurationMakeVariableContext getConfigurationMakeVariableContext() { + if (configurationMakeVariableContext == null) { + configurationMakeVariableContext = new ConfigurationMakeVariableContext( + getRule().getPackage(), getConfiguration()); + } + return configurationMakeVariableContext; + } + + /** + * Returns the string "expression" after expanding all embedded references to + * "Make" variables. If any errors are encountered, they are reported, and + * "expression" is returned unchanged. + * + * @param attributeName the name of the attribute from which "expression" comes; + * used for error reporting. + * @param expression the string to expand. + * @return the expansion of "expression". + */ + public String expandMakeVariables(String attributeName, String expression) { + return expandMakeVariables(attributeName, expression, getConfigurationMakeVariableContext()); + } + + /** + * Returns the string "expression" after expanding all embedded references to + * "Make" variables. If any errors are encountered, they are reported, and + * "expression" is returned unchanged. + * + * @param attributeName the name of the attribute from which "expression" comes; + * used for error reporting. + * @param expression the string to expand. + * @param context the ConfigurationMakeVariableContext which can have a customized + * lookupMakeVariable(String) method. + * @return the expansion of "expression". + */ + public String expandMakeVariables(String attributeName, String expression, + ConfigurationMakeVariableContext context) { + try { + return MakeVariableExpander.expand(expression, context); + } catch (MakeVariableExpander.ExpansionException e) { + attributeError(attributeName, e.getMessage()); + return expression; + } + } + + /** + * Gets the value of the STRING_LIST attribute expanding all make variables. + */ + public List<String> expandedMakeVariablesList(String attrName) { + List<String> variables = new ArrayList<>(); + for (String variable : attributes().get(attrName, Type.STRING_LIST)) { + variables.add(expandMakeVariables(attrName, variable)); + } + return variables; + } + + /** + * If the string consists of a single variable, returns the expansion of + * that variable. Otherwise, returns null. Syntax errors are reported. + * + * @param attrName the name of the attribute from which "expression" comes; + * used for error reporting. + * @param expression the string to expand. + * @return the expansion of "expression", or null. + */ + public String expandSingleMakeVariable(String attrName, String expression) { + try { + return MakeVariableExpander.expandSingleVariable(expression, + new ConfigurationMakeVariableContext(getRule().getPackage(), getConfiguration())); + } catch (MakeVariableExpander.ExpansionException e) { + attributeError(attrName, e.getMessage()); + return expression; + } + } + + private void checkAttribute(String attributeName, Mode mode) { + Attribute attributeDefinition = getRule().getAttributeDefinition(attributeName); + if (attributeDefinition == null) { + throw new IllegalStateException(getRule().getLocation() + ": " + getRule().getRuleClass() + + " attribute " + attributeName + " is not defined"); + } + if (!(attributeDefinition.getType() == Type.LABEL + || attributeDefinition.getType() == Type.LABEL_LIST)) { + throw new IllegalStateException(rule.getRuleClass() + " attribute " + attributeName + + " is not a label type attribute"); + } + if (mode == Mode.HOST) { + if (attributeDefinition.getConfigurationTransition() != ConfigurationTransition.HOST) { + throw new IllegalStateException(getRule().getLocation() + ": " + + getRule().getRuleClass() + " attribute " + attributeName + + " is not configured for the host configuration"); + } + } else if (mode == Mode.TARGET) { + if (attributeDefinition.getConfigurationTransition() != ConfigurationTransition.NONE) { + throw new IllegalStateException(getRule().getLocation() + ": " + + getRule().getRuleClass() + " attribute " + attributeName + + " is not configured for the target configuration"); + } + } else if (mode == Mode.DATA) { + if (attributeDefinition.getConfigurationTransition() != ConfigurationTransition.DATA) { + throw new IllegalStateException(getRule().getLocation() + ": " + + getRule().getRuleClass() + " attribute " + attributeName + + " is not configured for the data configuration"); + } + } else if (mode == Mode.SPLIT) { + if (!(attributeDefinition.getConfigurationTransition() instanceof SplitTransition)) { + throw new IllegalStateException(getRule().getLocation() + ": " + + getRule().getRuleClass() + " attribute " + attributeName + + " is not configured for a split transition"); + } + } + } + + /** + * Returns the Mode for which the attribute is configured. + * This is intended for Skylark, where the Mode is implicitly chosen. + */ + public Mode getAttributeMode(String attributeName) { + Attribute attributeDefinition = getRule().getAttributeDefinition(attributeName); + if (attributeDefinition == null) { + throw new IllegalStateException(getRule().getLocation() + ": " + getRule().getRuleClass() + + " attribute " + attributeName + " is not defined"); + } + if (!(attributeDefinition.getType() == Type.LABEL + || attributeDefinition.getType() == Type.LABEL_LIST)) { + throw new IllegalStateException(rule.getRuleClass() + " attribute " + attributeName + + " is not a label type attribute"); + } + if (attributeDefinition.getConfigurationTransition() == ConfigurationTransition.HOST) { + return Mode.HOST; + } else if (attributeDefinition.getConfigurationTransition() == ConfigurationTransition.NONE) { + return Mode.TARGET; + } else if (attributeDefinition.getConfigurationTransition() == ConfigurationTransition.DATA) { + return Mode.DATA; + } else if (attributeDefinition.getConfigurationTransition() instanceof SplitTransition) { + return Mode.SPLIT; + } + throw new IllegalStateException(getRule().getLocation() + ": " + + getRule().getRuleClass() + " attribute " + attributeName + " is not configured"); + } + + /** + * For the specified attribute "attributeName" (which must be of type + * list(label)), resolve all the labels into ConfiguredTargets (for the + * configuration appropriate to the attribute) and return their build + * artifacts as a {@link PrerequisiteArtifacts} instance. + * + * @param attributeName the name of the attribute to traverse + */ + public PrerequisiteArtifacts getPrerequisiteArtifacts(String attributeName, Mode mode) { + return PrerequisiteArtifacts.get(this, attributeName, mode); + } + + /** + * For the specified attribute "attributeName" (which must be of type label), + * resolves the ConfiguredTarget and returns its single build artifact. + * + * <p>If the attribute is optional, has no default and was not specified, then + * null will be returned. Note also that null is returned (and an attribute + * error is raised) if there wasn't exactly one build artifact for the target. + */ + public Artifact getPrerequisiteArtifact(String attributeName, Mode mode) { + TransitiveInfoCollection target = getPrerequisite(attributeName, mode); + return transitiveInfoCollectionToArtifact(attributeName, target); + } + + /** + * Equivalent to getPrerequisiteArtifact(), but also asserts that + * host-configuration is appropriate for the specified attribute. + */ + public Artifact getHostPrerequisiteArtifact(String attributeName) { + TransitiveInfoCollection target = getPrerequisite(attributeName, Mode.HOST); + return transitiveInfoCollectionToArtifact(attributeName, target); + } + + private Artifact transitiveInfoCollectionToArtifact( + String attributeName, TransitiveInfoCollection target) { + if (target != null) { + Iterable<Artifact> artifacts = target.getProvider(FileProvider.class).getFilesToBuild(); + if (Iterables.size(artifacts) == 1) { + return Iterables.getOnlyElement(artifacts); + } else { + attributeError(attributeName, target.getLabel() + " expected a single artifact"); + } + } + return null; + } + + /** + * Returns the sole file in the "srcs" attribute. Reports an error and + * (possibly) returns null if "srcs" does not identify a single file of the + * expected type. + */ + public Artifact getSingleSource(String fileTypeName) { + List<Artifact> srcs = PrerequisiteArtifacts.get(this, "srcs", Mode.TARGET).list(); + switch (srcs.size()) { + case 0 : // error already issued by getSrc() + return null; + case 1 : // ok + return Iterables.getOnlyElement(srcs); + default : + attributeError("srcs", "only a single " + fileTypeName + " is allowed here"); + return srcs.get(0); + } + } + + public Artifact getSingleSource() { + return getSingleSource(getRule().getRuleClass() + " source file"); + } + + /** + * Returns a path fragment qualified by the rule name and unique fragment to + * disambiguate artifacts produced from the source file appearing in + * multiple rules. + * + * <p>For example "pkg/dir/name" -> "pkg/<fragment>/rule/dir/name. + */ + public final PathFragment getUniqueDirectory(String fragment) { + return AnalysisUtils.getUniqueDirectory(getLabel(), new PathFragment(fragment)); + } + + /** + * Check that all targets that were specified as sources are from the same + * package as this rule. Output a warning or an error for every target that is + * imported from a different package. + */ + public void checkSrcsSamePackage(boolean onlyWarn) { + PathFragment packageName = getLabel().getPackageFragment(); + for (Artifact srcItem : PrerequisiteArtifacts.get(this, "srcs", Mode.TARGET).list()) { + if (!srcItem.isSourceArtifact()) { + // In theory, we should not do this check. However, in practice, we + // have a couple of rules that do not obey the "srcs must contain + // files and only files" rule. Thus, we are stuck with this hack here :( + continue; + } + Label associatedLabel = srcItem.getOwner(); + PathFragment itemPackageName = associatedLabel.getPackageFragment(); + if (!itemPackageName.equals(packageName)) { + String message = "please do not import '" + associatedLabel + "' directly. " + + "You should either move the file to this package or depend on " + + "an appropriate rule there"; + if (onlyWarn) { + attributeWarning("srcs", message); + } else { + attributeError("srcs", message); + } + } + } + } + + + /** + * Returns the label to which the {@code NODEP_LABEL} attribute + * {@code attrName} refers, checking that it is a valid label, and that it is + * referring to a local target. Reports a warning otherwise. + */ + public Label getLocalNodepLabelAttribute(String attrName) { + Label label = attributes().get(attrName, Type.NODEP_LABEL); + if (label == null) { + return null; + } + + if (!getTarget().getLabel().getPackageFragment().equals(label.getPackageFragment())) { + attributeWarning(attrName, "does not reference a local rule"); + } + + return label; + } + + /** + * Returns the implicit output artifact for a given template function. If multiple or no artifacts + * can be found as a result of the template, an exception is thrown. + */ + public Artifact getImplicitOutputArtifact(ImplicitOutputsFunction function) { + Iterable<String> result; + try { + result = function.getImplicitOutputs(RawAttributeMapper.of(rule)); + } catch (EvalException e) { + // It's ok as long as we don't use this method from Skylark. + throw new IllegalStateException(e); + } + return getImplicitOutputArtifact(Iterables.getOnlyElement(result)); + } + + /** + * Only use from Skylark. Returns the implicit output artifact for a given output path. + */ + public Artifact getImplicitOutputArtifact(String path) { + Root root = getBinOrGenfilesDirectory(); + PathFragment packageFragment = getLabel().getPackageFragment(); + return getAnalysisEnvironment().getDerivedArtifact(packageFragment.getRelative(path), root); + } + + /** + * Convenience method to return a host configured target for the "compiler" + * attribute. Allows caller to decide whether a warning should be printed if + * the "compiler" attribute is not set to the default value. + * + * @param warnIfNotDefault if true, print a warning if the value for the + * "compiler" attribute is set to something other than the default + * @return a ConfiguredTarget using the host configuration for the "compiler" + * attribute + */ + public final FilesToRunProvider getCompiler(boolean warnIfNotDefault) { + Label label = attributes().get("compiler", Type.LABEL); + if (warnIfNotDefault && !label.equals(getRule().getAttrDefaultValue("compiler"))) { + attributeWarning("compiler", "setting the compiler is strongly discouraged"); + } + return getExecutablePrerequisite("compiler", Mode.HOST); + } + + /** + * Returns the (unmodifiable, ordered) list of artifacts which are the outputs + * of this target. + * + * <p>Each element in this list is associated with a single output, either + * declared implicitly (via setImplicitOutputsFunction()) or explicitly + * (listed in the 'outs' attribute of our rule). + */ + public final ImmutableList<Artifact> getOutputArtifacts() { + ImmutableList.Builder<Artifact> artifacts = ImmutableList.builder(); + for (OutputFile out : getRule().getOutputFiles()) { + artifacts.add(createOutputArtifact(out)); + } + return artifacts.build(); + } + + /** + * Like getFilesToBuild(), except that it also includes the runfiles middleman, if any. + * Middlemen are expanded in the SpawnStrategy or by the Distributor. + */ + public static ImmutableList<Artifact> getFilesToRun( + RunfilesSupport runfilesSupport, NestedSet<Artifact> filesToBuild) { + if (runfilesSupport == null) { + return ImmutableList.copyOf(filesToBuild); + } else { + ImmutableList.Builder<Artifact> allFilesToBuild = ImmutableList.builder(); + allFilesToBuild.addAll(filesToBuild); + allFilesToBuild.add(runfilesSupport.getRunfilesMiddleman()); + return allFilesToBuild.build(); + } + } + + /** + * Like {@link #getOutputArtifacts()} but for a singular output item. + * Reports an error if the "out" attribute is not a singleton. + * + * @return null if the output list is empty, the artifact for the first item + * of the output list otherwise + */ + public Artifact getOutputArtifact() { + List<Artifact> outs = getOutputArtifacts(); + if (outs.size() != 1) { + attributeError("out", "exactly one output file required"); + if (outs.isEmpty()) { + return null; + } + } + return outs.get(0); + } + + /** + * Returns an artifact with a given file extension. All other path components + * are the same as in {@code pathFragment}. + */ + public final Artifact getRelatedArtifact(PathFragment pathFragment, String extension) { + PathFragment file = FileSystemUtils.replaceExtension(pathFragment, extension); + return getAnalysisEnvironment().getDerivedArtifact(file, getConfiguration().getBinDirectory()); + } + + /** + * Returns true if runfiles support should create the runfiles tree, or + * false if it should just create the manifest. + */ + public boolean shouldCreateRunfilesSymlinks() { + // TODO(bazel-team): Ideally we wouldn't need such logic, and we'd + // always use the BuildConfiguration#buildRunfiles() to determine + // whether to build the runfiles. The problem is that certain build + // steps actually consume their runfiles. These include: + // a. par files consumes the runfiles directory + // We should modify autopar to take a list of files instead. + // of the runfiles directory. + // b. host tools could potentially use data files, but currently don't + // (they're run from the execution root, not a runfiles tree). + // Currently hostConfiguration.buildRunfiles() returns true. + if (TargetUtils.isTestRule(getTarget())) { + // Tests are only executed during testing (duh), + // and their runfiles are generated lazily on local + // execution (see LocalTestStrategy). Therefore, it + // is safe not to build their runfiles. + return getConfiguration().buildRunfiles(); + } else { + return true; + } + } + + /** + * @return true if {@code rule} is visible from {@code prerequisite}. + * + * <p>This only computes the logic as implemented by the visibility system. The final decision + * whether a dependency is allowed is made by + * {@link ConfiguredRuleClassProvider.PrerequisiteValidator}. + */ + public static boolean isVisible(Rule rule, TransitiveInfoCollection prerequisite) { + // Check visibility attribute + for (PackageSpecification specification : + prerequisite.getProvider(VisibilityProvider.class).getVisibility()) { + if (specification.containsPackage(rule.getLabel().getPackageFragment())) { + return true; + } + } + + return false; + } + + /** + * @return the set of features applicable for the current rule's package. + */ + public ImmutableSet<String> getFeatures() { + return features; + } + + /** + * Builder class for a RuleContext. + */ + public static final class Builder { + private final AnalysisEnvironment env; + private final Rule rule; + private final BuildConfiguration configuration; + private final PrerequisiteValidator prerequisiteValidator; + private ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap; + private Set<ConfigMatchingProvider> configConditions; + private NestedSet<PackageSpecification> visibility; + + Builder(AnalysisEnvironment env, Rule rule, BuildConfiguration configuration, + PrerequisiteValidator prerequisiteValidator) { + this.env = Preconditions.checkNotNull(env); + this.rule = Preconditions.checkNotNull(rule); + this.configuration = Preconditions.checkNotNull(configuration); + this.prerequisiteValidator = prerequisiteValidator; + } + + RuleContext build() { + Preconditions.checkNotNull(prerequisiteMap); + Preconditions.checkNotNull(configConditions); + Preconditions.checkNotNull(visibility); + ListMultimap<String, ConfiguredTarget> targetMap = createTargetMap(); + ListMultimap<String, ConfiguredFilesetEntry> filesetEntryMap = + createFilesetEntryMap(rule, configConditions); + return new RuleContext(this, targetMap, filesetEntryMap, configConditions, + getEnabledFeatures()); + } + + private ImmutableSet<String> getEnabledFeatures() { + Set<String> enabled = new HashSet<>(); + Set<String> disabled = new HashSet<>(); + for (String feature : Iterables.concat(getConfiguration().getDefaultFeatures(), + getRule().getPackage().getFeatures())) { + if (feature.startsWith("-")) { + disabled.add(feature.substring(1)); + } else if (feature.equals("no_layering_check")) { + // TODO(bazel-team): Remove once we do not have BUILD files left that contain + // 'no_layering_check'. + disabled.add(feature.substring(3)); + } else { + enabled.add(feature); + } + } + return Sets.difference(enabled, disabled).immutableCopy(); + } + + Builder setVisibility(NestedSet<PackageSpecification> visibility) { + this.visibility = visibility; + return this; + } + + /** + * Sets the prerequisites and checks their visibility. It also generates appropriate error or + * warning messages and sets the error flag as appropriate. + */ + Builder setPrerequisites(ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap) { + this.prerequisiteMap = Preconditions.checkNotNull(prerequisiteMap); + return this; + } + + /** + * Sets the configuration conditions needed to determine which paths to follow for this + * rule's configurable attributes. + */ + Builder setConfigConditions(Set<ConfigMatchingProvider> configConditions) { + this.configConditions = Preconditions.checkNotNull(configConditions); + return this; + } + + private boolean validateFilesetEntry(FilesetEntry filesetEntry, ConfiguredTarget src) { + if (src.getProvider(FilesetProvider.class) != null) { + return true; + } + if (filesetEntry.isSourceFileset()) { + return true; + } + + Target srcTarget = src.getTarget(); + if (!(srcTarget instanceof FileTarget)) { + attributeError("entries", String.format( + "Invalid 'srcdir' target '%s'. Must be another Fileset or package", + srcTarget.getLabel())); + return false; + } + + if (srcTarget instanceof OutputFile) { + attributeWarning("entries", String.format("'srcdir' target '%s' is not an input file. " + + "This forces the Fileset to be executed unconditionally", + srcTarget.getLabel())); + } + + return true; + } + + /** + * Determines and returns a map from attribute name to list of configured fileset entries, based + * on a PrerequisiteMap instance. + */ + private ListMultimap<String, ConfiguredFilesetEntry> createFilesetEntryMap( + final Rule rule, Set<ConfigMatchingProvider> configConditions) { + final ImmutableSortedKeyListMultimap.Builder<String, ConfiguredFilesetEntry> mapBuilder = + ImmutableSortedKeyListMultimap.builder(); + for (Attribute attr : rule.getAttributes()) { + if (attr.getType() != Type.FILESET_ENTRY_LIST) { + continue; + } + String attributeName = attr.getName(); + Map<Label, ConfiguredTarget> ctMap = new HashMap<>(); + for (ConfiguredTarget prerequisite : prerequisiteMap.get(attr)) { + ctMap.put(prerequisite.getLabel(), prerequisite); + } + List<FilesetEntry> entries = ConfiguredAttributeMapper.of(rule, configConditions) + .get(attributeName, Type.FILESET_ENTRY_LIST); + for (FilesetEntry entry : entries) { + if (entry.getFiles() == null) { + Label label = entry.getSrcLabel(); + ConfiguredTarget src = ctMap.get(label); + if (!validateFilesetEntry(entry, src)) { + continue; + } + + mapBuilder.put(attributeName, new ConfiguredFilesetEntry(entry, src)); + } else { + ImmutableList.Builder<TransitiveInfoCollection> files = ImmutableList.builder(); + for (Label file : entry.getFiles()) { + files.add(ctMap.get(file)); + } + mapBuilder.put(attributeName, new ConfiguredFilesetEntry(entry, files.build())); + } + } + } + return mapBuilder.build(); + } + + /** + * Determines and returns a map from attribute name to list of configured targets. + */ + private ImmutableSortedKeyListMultimap<String, ConfiguredTarget> createTargetMap() { + ImmutableSortedKeyListMultimap.Builder<String, ConfiguredTarget> mapBuilder = + ImmutableSortedKeyListMultimap.builder(); + + for (Map.Entry<Attribute, Collection<ConfiguredTarget>> entry : + prerequisiteMap.asMap().entrySet()) { + Attribute attribute = entry.getKey(); + if (attribute == null) { + continue; + } + if (attribute.isSilentRuleClassFilter()) { + Predicate<RuleClass> filter = attribute.getAllowedRuleClassesPredicate(); + for (ConfiguredTarget configuredTarget : entry.getValue()) { + Target prerequisiteTarget = configuredTarget.getTarget(); + if ((prerequisiteTarget instanceof Rule) + && filter.apply(((Rule) prerequisiteTarget).getRuleClassObject())) { + validateDirectPrerequisite(attribute, configuredTarget); + mapBuilder.put(attribute.getName(), configuredTarget); + } + } + } else { + for (ConfiguredTarget configuredTarget : entry.getValue()) { + validateDirectPrerequisite(attribute, configuredTarget); + mapBuilder.put(attribute.getName(), configuredTarget); + } + } + } + + // Handle abi_deps+deps error. + Attribute abiDepsAttr = rule.getAttributeDefinition("abi_deps"); + if ((abiDepsAttr != null) && rule.isAttributeValueExplicitlySpecified("abi_deps") + && rule.isAttributeValueExplicitlySpecified("deps")) { + attributeError("deps", "Only one of deps and abi_deps should be provided"); + } + return mapBuilder.build(); + } + + private String prefixRuleMessage(String message) { + return String.format("in %s rule %s: %s", rule.getRuleClass(), rule.getLabel(), message); + } + + private String maskInternalAttributeNames(String name) { + return Attribute.isImplicit(name) ? "(an implicit dependency)" : name; + } + + private String prefixAttributeMessage(String attrName, String message) { + return String.format("in %s attribute of %s rule %s: %s", + maskInternalAttributeNames(attrName), rule.getRuleClass(), rule.getLabel(), message); + } + + public void reportError(Location location, String message) { + env.getEventHandler().handle(Event.error(location, message)); + } + + public void ruleError(String message) { + reportError(rule.getLocation(), prefixRuleMessage(message)); + } + + public void attributeError(String attrName, String message) { + reportError(rule.getAttributeLocation(attrName), prefixAttributeMessage(attrName, message)); + } + + public void reportWarning(Location location, String message) { + env.getEventHandler().handle(Event.warn(location, message)); + } + + public void ruleWarning(String message) { + env.getEventHandler().handle(Event.warn(rule.getLocation(), prefixRuleMessage(message))); + } + + public void attributeWarning(String attrName, String message) { + reportWarning(rule.getAttributeLocation(attrName), prefixAttributeMessage(attrName, message)); + } + + private void reportBadPrerequisite(Attribute attribute, String targetKind, + Label prerequisiteLabel, String reason, boolean isWarning) { + String msgPrefix = targetKind != null ? targetKind + " " : ""; + String msgReason = reason != null ? " (" + reason + ")" : ""; + if (isWarning) { + attributeWarning(attribute.getName(), String.format( + "%s'%s' is unexpected here%s; continuing anyway", + msgPrefix, prerequisiteLabel, msgReason)); + } else { + attributeError(attribute.getName(), String.format( + "%s'%s' is misplaced here%s", msgPrefix, prerequisiteLabel, msgReason)); + } + } + + private void validateDirectPrerequisiteType(ConfiguredTarget prerequisite, + Attribute attribute) { + Target prerequisiteTarget = prerequisite.getTarget(); + Label prerequisiteLabel = prerequisiteTarget.getLabel(); + + if (prerequisiteTarget instanceof Rule) { + Rule prerequisiteRule = (Rule) prerequisiteTarget; + + String reason = attribute.getValidityPredicate().checkValid(rule, prerequisiteRule); + if (reason != null) { + reportBadPrerequisite(attribute, prerequisiteTarget.getTargetKind(), + prerequisiteLabel, reason, false); + } + } + + if (attribute.isStrictLabelCheckingEnabled()) { + if (prerequisiteTarget instanceof Rule) { + RuleClass ruleClass = ((Rule) prerequisiteTarget).getRuleClassObject(); + if (!attribute.getAllowedRuleClassesPredicate().apply(ruleClass)) { + boolean allowedWithWarning = attribute.getAllowedRuleClassesWarningPredicate() + .apply(ruleClass); + reportBadPrerequisite(attribute, prerequisiteTarget.getTargetKind(), prerequisiteLabel, + "expected " + attribute.getAllowedRuleClassesPredicate().toString(), + allowedWithWarning); + } + } else if (prerequisiteTarget instanceof FileTarget) { + if (!attribute.getAllowedFileTypesPredicate() + .apply(((FileTarget) prerequisiteTarget).getFilename())) { + if (prerequisiteTarget instanceof InputFile + && !((InputFile) prerequisiteTarget).getPath().exists()) { + // Misplaced labels, no corresponding target exists + if (attribute.getAllowedFileTypesPredicate().isNone() + && !((InputFile) prerequisiteTarget).getFilename().contains(".")) { + // There are no allowed files in the attribute but it's not a valid rule, + // and the filename doesn't contain a dot --> probably a misspelled rule + attributeError(attribute.getName(), + "rule '" + prerequisiteLabel + "' does not exist"); + } else { + attributeError(attribute.getName(), + "target '" + prerequisiteLabel + "' does not exist"); + } + } else { + // The file exists but has a bad extension + reportBadPrerequisite(attribute, "file", prerequisiteLabel, + "expected " + attribute.getAllowedFileTypesPredicate().toString(), false); + } + } + } + } + } + + public Rule getRule() { + return rule; + } + + public BuildConfiguration getConfiguration() { + return configuration; + } + + /** + * @return true if {@code rule} is visible from {@code prerequisite}. + * + * <p>This only computes the logic as implemented by the visibility system. The final decision + * whether a dependency is allowed is made by + * {@link ConfiguredRuleClassProvider.PrerequisiteValidator}, who is supposed to call this + * method to determine whether a dependency is allowed as per visibility rules. + */ + public boolean isVisible(TransitiveInfoCollection prerequisite) { + return RuleContext.isVisible(rule, prerequisite); + } + + private void validateDirectPrerequisiteFileTypes(ConfiguredTarget prerequisite, + Attribute attribute) { + if (attribute.isSkipAnalysisTimeFileTypeCheck()) { + return; + } + FileTypeSet allowedFileTypes = attribute.getAllowedFileTypesPredicate(); + if (allowedFileTypes == FileTypeSet.ANY_FILE && !attribute.isNonEmpty() + && !attribute.isSingleArtifact()) { + return; + } + + // If we allow any file we still need to check if there are actually files generated + // Note that this check only runs for ANY_FILE predicates if the attribute is NON_EMPTY + // or SINGLE_ARTIFACT + // If we performed this check when allowedFileTypes == NO_FILE this would + // always throw an error in those cases + if (allowedFileTypes != FileTypeSet.NO_FILE) { + Iterable<Artifact> artifacts = prerequisite.getProvider(FileProvider.class) + .getFilesToBuild(); + if (attribute.isSingleArtifact() && Iterables.size(artifacts) != 1) { + attributeError(attribute.getName(), + "'" + prerequisite.getLabel() + "' must produce a single file"); + return; + } + for (Artifact sourceArtifact : artifacts) { + if (allowedFileTypes.apply(sourceArtifact.getFilename())) { + return; + } + } + attributeError(attribute.getName(), "'" + prerequisite.getLabel() + + "' does not produce any " + rule.getRuleClass() + " " + attribute.getName() + + " files (expected " + allowedFileTypes + ")"); + } + } + + private void validateMandatoryProviders(ConfiguredTarget prerequisite, Attribute attribute) { + for (String provider : attribute.getMandatoryProviders()) { + if (prerequisite.get(provider) == null) { + attributeError(attribute.getName(), "'" + prerequisite.getLabel() + + "' does not have mandatory provider '" + provider + "'"); + } + } + } + + private void validateDirectPrerequisite(Attribute attribute, ConfiguredTarget prerequisite) { + validateDirectPrerequisiteType(prerequisite, attribute); + validateDirectPrerequisiteFileTypes(prerequisite, attribute); + validateMandatoryProviders(prerequisite, attribute); + prerequisiteValidator.validate(this, prerequisite, attribute); + } + } + + @Override + public String toString() { + return "RuleContext(" + getLabel() + ", " + getConfiguration() + ")"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleDefinition.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleDefinition.java new file mode 100644 index 0000000..c5e32e3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleDefinition.java
@@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.packages.RuleClass; + +/** + * This class is a common ancestor for every rule object. + * + * <p>Implementors are also required to have the {@link BlazeRule} annotation + * set. + */ +public interface RuleDefinition { + /** + * This method should return a RuleClass object that represents the rule. The usual pattern is + * that various setter methods are called on the builder object passed in as the argument, then + * the object that is built by the builder is returned. + * + * @param builder A {@link com.google.devtools.build.lib.packages.RuleClass.Builder} object + * already preloaded with the attributes of the ancestors specified in the {@link + * BlazeRule} annotation. + * @param environment The services Blaze provides to rule definitions. + * + * @return the {@link RuleClass} representing the rule. + */ + RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment environment); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleDefinitionEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleDefinitionEnvironment.java new file mode 100644 index 0000000..4dcd080 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleDefinitionEnvironment.java
@@ -0,0 +1,31 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.syntax.Label; + +/** + * Encapsulates the services available for implementors of the {@link RuleDefinition} + * interface. + */ +public interface RuleDefinitionEnvironment { + /** + * Parses the given string as a label and returns the label, by calling {@link + * Label#parseAbsolute}. Instead of throwing a {@link + * com.google.devtools.build.lib.syntax.Label.SyntaxException}, it throws an {@link + * IllegalArgumentException}, if the parsing fails. + */ + Label getLabel(String labelValue); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java b/src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java new file mode 100644 index 0000000..a4da4b4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java
@@ -0,0 +1,756 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * An object that encapsulates runfiles. Conceptually, the runfiles are a map of paths to files, + * forming a symlink tree. + * + * <p>In order to reduce memory consumption, this map is not explicitly stored here, but instead as + * a combination of four parts: artifacts placed at their root-relative paths, source tree symlinks, + * root symlinks (outside of the source tree), and artifacts included as parts of "pruning + * manifests" (see {@link PruningManifest}). + */ +@Immutable +public final class Runfiles { + private static final Function<Map.Entry<PathFragment, Artifact>, Artifact> TO_ARTIFACT = + new Function<Map.Entry<PathFragment, Artifact>, Artifact>() { + @Override + public Artifact apply(Map.Entry<PathFragment, Artifact> input) { + return input.getValue(); + } + }; + + private static final Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> + DUMMY_SYMLINK_EXPANDER = + new Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>>() { + @Override + public Map<PathFragment, Artifact> apply(Map<PathFragment, Artifact> input) { + return ImmutableMap.of(); + } + }; + + // It is important to declare this *after* the DUMMY_SYMLINK_EXPANDER to avoid NPEs + public static final Runfiles EMPTY = new Builder().build(); + + + /** + * The artifacts that should *always* be present in the runfiles directory. These are + * differentiated from the artifacts that may or may not be included by a pruning manifest + * (see {@link PruningManifest} below). + * + * <p>This collection may not include any middlemen. These artifacts will be placed at a location + * that corresponds to the root-relative path of each artifact. It's possible for several + * artifacts to have the same root-relative path, in which case the last one will win. + */ + private final NestedSet<Artifact> unconditionalArtifacts; + + /** + * A map of symlinks that should be present in the runfiles directory. In general, the symlink can + * be determined from the artifact by using the root-relative path, so this should only be used + * for cases where that isn't possible. + * + * <p>This may include runfiles symlinks from the root of the runfiles tree. + */ + private final NestedSet<Map.Entry<PathFragment, Artifact>> symlinks; + + /** + * A map of symlinks that should be present above the runfiles directory. These are useful for + * certain rule types like AppEngine apps which have root level config files outside of the + * regular source tree. + */ + private final NestedSet<Map.Entry<PathFragment, Artifact>> rootSymlinks; + + /** + * A function to generate extra manifest entries. + */ + private final Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> + manifestExpander; + + /** + * Defines a set of artifacts that may or may not be included in the runfiles directory and + * a manifest file that makes that determination. These are applied on top of any artifacts + * specified in {@link #unconditionalArtifacts}. + * + * <p>The incentive behind this is to enable execution-phase "pruning" of runfiles. Anything + * set in unconditionalArtifacts is hard-set in Blaze's analysis phase, and thus unchangeable in + * response to execution phase results. This isn't always convenient. For example, say we have an + * action that consumes a set of "possible" runtime dependencies for a source file, parses that + * file for "import a.b.c" statements, and outputs a manifest of the actual dependencies that are + * referenced and thus really needed. This can reduce the size of the runfiles set, but we can't + * use this information until the manifest output is available. + * + * <p>Only artifacts present in the candidate set AND the manifest output make it into the + * runfiles tree. The candidate set requirement guarantees that analysis-time dependencies are a + * superset of the pruned dependencies, so undeclared inclusions (which can break build + * correctness) aren't possible. + */ + public static class PruningManifest { + private final NestedSet<Artifact> candidateRunfiles; + private final Artifact manifestFile; + + /** + * Creates a new pruning manifest. + * + * @param candidateRunfiles set of possible artifacts that the manifest file may reference + * @param manifestFile the manifest file, expected to be a newline-separated list of + * source tree root-relative paths (i.e. "my/package/myfile.txt"). Anything that can't be + * resolved back to an entry in candidateRunfiles is ignored and will *not* make it into + * the runfiles tree. + */ + public PruningManifest(NestedSet<Artifact> candidateRunfiles, Artifact manifestFile) { + this.candidateRunfiles = candidateRunfiles; + this.manifestFile = manifestFile; + } + + public NestedSet<Artifact> getCandidateRunfiles() { + return candidateRunfiles; + } + + public Artifact getManifestFile() { + return manifestFile; + } + } + + /** + * The pruning manifests that should be applied to these runfiles. + */ + private final NestedSet<PruningManifest> pruningManifests; + + private Runfiles(NestedSet<Artifact> artifacts, + NestedSet<Map.Entry<PathFragment, Artifact>> symlinks, + NestedSet<Map.Entry<PathFragment, Artifact>> rootSymlinks, + NestedSet<PruningManifest> pruningManifests, + Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> expander) { + this.unconditionalArtifacts = Preconditions.checkNotNull(artifacts); + this.symlinks = Preconditions.checkNotNull(symlinks); + this.rootSymlinks = Preconditions.checkNotNull(rootSymlinks); + this.pruningManifests = Preconditions.checkNotNull(pruningManifests); + this.manifestExpander = Preconditions.checkNotNull(expander); + } + + /** + * Returns the artifacts that are unconditionally included in the runfiles (as opposed to + * pruning manifest candidates, which may or may not be included). + */ + @VisibleForTesting + public NestedSet<Artifact> getUnconditionalArtifacts() { + return unconditionalArtifacts; + } + + /** + * Returns the artifacts that are unconditionally included in the runfiles (as opposed to + * pruning manifest candidates, which may or may not be included). Middleman artifacts are + * excluded. + */ + public Iterable<Artifact> getUnconditionalArtifactsWithoutMiddlemen() { + return Iterables.filter(unconditionalArtifacts, Artifact.MIDDLEMAN_FILTER); + } + + /** + * Returns the collection of runfiles as artifacts, including both unconditional artifacts + * and pruning manifest candidates. + */ + @VisibleForTesting + public NestedSet<Artifact> getArtifacts() { + NestedSetBuilder<Artifact> allArtifacts = NestedSetBuilder.stableOrder(); + allArtifacts.addAll(unconditionalArtifacts.toCollection()); + for (PruningManifest manifest : getPruningManifests()) { + allArtifacts.addTransitive(manifest.getCandidateRunfiles()); + } + return allArtifacts.build(); + } + + /** + * Returns the collection of runfiles as artifacts, including both unconditional artifacts + * and pruning manifest candidates. Middleman artifacts are excluded. + */ + public Iterable<Artifact> getArtifactsWithoutMiddlemen() { + return Iterables.filter(getArtifacts(), Artifact.MIDDLEMAN_FILTER); + } + + /** + * Returns the symlinks. + */ + public NestedSet<Map.Entry<PathFragment, Artifact>> getSymlinks() { + return symlinks; + } + + /** + * Returns the symlinks as a map from path fragment to artifact. + */ + public Map<PathFragment, Artifact> getSymlinksAsMap() { + return entriesToMap(symlinks); + } + + /** + * @param eventHandler Used for throwing an error if we have an obscuring runlink. + * May be null, in which case obscuring symlinks are silently discarded. + * @param location Location for reporter. Ignored if reporter is null. + * @param workingManifest Manifest to be checked for obscuring symlinks. + * @return map of source file names mapped to their location on disk. + */ + public static Map<PathFragment, Artifact> filterListForObscuringSymlinks( + EventHandler eventHandler, Location location, Map<PathFragment, Artifact> workingManifest) { + Map<PathFragment, Artifact> newManifest = new HashMap<>(); + + outer: + for (Iterator<Entry<PathFragment, Artifact>> i = workingManifest.entrySet().iterator(); + i.hasNext(); ) { + Entry<PathFragment, Artifact> entry = i.next(); + PathFragment source = entry.getKey(); + Artifact symlink = entry.getValue(); + // drop nested entries; warn if this changes anything + int n = source.segmentCount(); + for (int j = 1; j < n; ++j) { + PathFragment prefix = source.subFragment(0, n - j); + Artifact ancestor = workingManifest.get(prefix); + if (ancestor != null) { + // This is an obscuring symlink, so just drop it and move on if there's no reporter. + if (eventHandler == null) { + continue outer; + } + PathFragment suffix = source.subFragment(n - j, n); + Path viaAncestor = ancestor.getPath().getRelative(suffix); + Path expected = symlink.getPath(); + if (!viaAncestor.equals(expected)) { + eventHandler.handle(Event.warn(location, "runfiles symlink " + source + " -> " + + expected + " obscured by " + prefix + " -> " + ancestor.getPath())); + } + continue outer; + } + } + newManifest.put(entry.getKey(), entry.getValue()); + } + return newManifest; + } + + /** + * Returns the symlinks as a map from PathFragment to Artifact, with PathFragments relativized + * and rooted at the specified points. + * @param root The root the PathFragment is computed relative to (before it is + * rooted again). May be null. + * @param eventHandler Used for throwing an error if we have an obscuring runlink. + * May be null, in which case obscuring symlinks are silently discarded. + * @param location Location for eventHandler warnings. Ignored if eventHandler is null. + * @return Pair of Maps from remote path fragment to artifact, the first of normal source tree + * entries, the second of any elements that live outside the source tree. + */ + public Pair<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> getRunfilesInputs( + PathFragment root, String workspaceSuffix, EventHandler eventHandler, Location location) + throws IOException { + Map<PathFragment, Artifact> manifest = getSymlinksAsMap(); + // Add unconditional artifacts (committed to inclusion on construction of runfiles). + for (Artifact artifact : getUnconditionalArtifactsWithoutMiddlemen()) { + addToManifest(manifest, artifact, root); + } + + // Add conditional artifacts (only included if they appear in a pruning manifest). + for (Runfiles.PruningManifest pruningManifest : getPruningManifests()) { + // This map helps us convert from source tree root-relative paths back to artifacts. + Map<String, Artifact> allowedRunfiles = new HashMap<>(); + for (Artifact artifact : pruningManifest.getCandidateRunfiles()) { + allowedRunfiles.put(artifact.getRootRelativePath().getPathString(), artifact); + } + BufferedReader reader = new BufferedReader( + new InputStreamReader(pruningManifest.getManifestFile().getPath().getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + Artifact artifact = allowedRunfiles.get(line); + if (artifact != null) { + addToManifest(manifest, artifact, root); + } + } + } + + manifest = filterListForObscuringSymlinks(eventHandler, location, manifest); + manifest.putAll(manifestExpander.apply(manifest)); + PathFragment path = new PathFragment(workspaceSuffix); + Map<PathFragment, Artifact> result = new HashMap<>(); + for (Map.Entry<PathFragment, Artifact> entry : manifest.entrySet()) { + result.put(path.getRelative(entry.getKey()), entry.getValue()); + } + return Pair.of(result, (Map<PathFragment, Artifact>) new HashMap<>(getRootSymlinksAsMap())); + } + + @VisibleForTesting + protected static void addToManifest(Map<PathFragment, Artifact> manifest, Artifact artifact, + PathFragment root) { + PathFragment rootRelativePath = root != null + ? artifact.getRootRelativePath().relativeTo(root) + : artifact.getRootRelativePath(); + manifest.put(rootRelativePath, artifact); + } + + /** + * Returns the root symlinks. + */ + public NestedSet<Map.Entry<PathFragment, Artifact>> getRootSymlinks() { + return rootSymlinks; + } + + /** + * Returns the root symlinks. + */ + public Map<PathFragment, Artifact> getRootSymlinksAsMap() { + return entriesToMap(rootSymlinks); + } + + /** + * Returns the unified map of path fragments to artifacts, taking both artifacts and symlinks into + * account. + */ + public Map<PathFragment, Artifact> asMapWithoutRootSymlinks() { + Map<PathFragment, Artifact> result = entriesToMap(symlinks); + // If multiple artifacts have the same root-relative path, the last one in the list will win. + // That is because the runfiles tree cannot contain the same artifact for different + // configurations, because it only uses root-relative paths. + for (Artifact artifact : Iterables.filter(unconditionalArtifacts, Artifact.MIDDLEMAN_FILTER)) { + result.put(artifact.getRootRelativePath(), artifact); + } + return result; + } + + /** + * Returns the pruning manifests specified for this runfiles tree. + */ + public NestedSet<PruningManifest> getPruningManifests() { + return pruningManifests; + } + + /** + * Returns the symlinks expander specified for this runfiles tree. + */ + public Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> getSymlinkExpander() { + return manifestExpander; + } + + /** + * Returns the unified map of path fragments to artifacts, taking into account artifacts, + * symlinks, and pruning manifest candidates. The returned set is guaranteed to be a (not + * necessarily strict) superset of the actual runfiles tree created at execution time. + */ + public NestedSet<Artifact> getAllArtifacts() { + if (isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + NestedSetBuilder<Artifact> allArtifacts = NestedSetBuilder.stableOrder(); + allArtifacts + .addTransitive(unconditionalArtifacts) + .addAll(Iterables.transform(symlinks, TO_ARTIFACT)) + .addAll(Iterables.transform(rootSymlinks, TO_ARTIFACT)); + for (PruningManifest manifest : getPruningManifests()) { + allArtifacts.addTransitive(manifest.getCandidateRunfiles()); + } + return allArtifacts.build(); + } + + /** + * Returns if there are no runfiles. + */ + public boolean isEmpty() { + return unconditionalArtifacts.isEmpty() && symlinks.isEmpty() && rootSymlinks.isEmpty() && + pruningManifests.isEmpty(); + } + + private static <K, V> Map<K, V> entriesToMap(Iterable<Map.Entry<K, V>> entrySet) { + Map<K, V> map = new LinkedHashMap<>(); + for (Map.Entry<K, V> entry : entrySet) { + map.put(entry.getKey(), entry.getValue()); + } + return map; + } + + /** + * Builder for Runfiles objects. + */ + public static final class Builder { + /** + * This must be COMPILE_ORDER because {@link #asMapWithoutRootSymlinks} overwrites earlier + * entries with later ones, so we want a post-order iteration. + */ + private NestedSetBuilder<Artifact> artifactsBuilder = + NestedSetBuilder.compileOrder(); + private NestedSetBuilder<Map.Entry<PathFragment, Artifact>> symlinksBuilder = + NestedSetBuilder.stableOrder(); + private NestedSetBuilder<Map.Entry<PathFragment, Artifact>> rootSymlinksBuilder = + NestedSetBuilder.stableOrder(); + private NestedSetBuilder<PruningManifest> pruningManifestsBuilder = + NestedSetBuilder.stableOrder(); + private Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> + manifestExpander = DUMMY_SYMLINK_EXPANDER; + + /** + * Builds a new Runfiles object. + */ + public Runfiles build() { + return new Runfiles(artifactsBuilder.build(), symlinksBuilder.build(), + rootSymlinksBuilder.build(), pruningManifestsBuilder.build(), + manifestExpander); + } + + /** + * Adds an artifact to the internal collection of artifacts. + */ + public Builder addArtifact(Artifact artifact) { + Preconditions.checkNotNull(artifact); + artifactsBuilder.add(artifact); + return this; + } + + /** + * Adds several artifacts to the internal collection. + */ + public Builder addArtifacts(Iterable<Artifact> artifacts) { + for (Artifact artifact : artifacts) { + addArtifact(artifact); + } + return this; + } + + + /** + * Use {@link #addTransitiveArtifacts} instead, to prevent increased memory use. + */ + @Deprecated + public Builder addArtifacts(NestedSet<Artifact> artifacts) { + // Do not delete this method, or else addArtifacts(Iterable) calls with a NestedSet argument + // will not be flagged. + Iterable<Artifact> it = artifacts; + addArtifacts(it); + return this; + } + /** + * Adds a nested set to the internal collection. + */ + public Builder addTransitiveArtifacts(NestedSet<Artifact> artifacts) { + artifactsBuilder.addTransitive(artifacts); + return this; + } + + /** + * Adds a symlink. + */ + public Builder addSymlink(PathFragment link, Artifact target) { + Preconditions.checkNotNull(link); + Preconditions.checkNotNull(target); + symlinksBuilder.add(Maps.immutableEntry(link, target)); + return this; + } + + /** + * Adds several symlinks. + */ + public Builder addSymlinks(Map<PathFragment, Artifact> symlinks) { + symlinksBuilder.addAll(symlinks.entrySet()); + return this; + } + + /** + * Adds several symlinks as a NestedSet. + */ + public Builder addSymlinks(NestedSet<Map.Entry<PathFragment, Artifact>> symlinks) { + symlinksBuilder.addTransitive(symlinks); + return this; + } + + /** + * Adds several root symlinks. + */ + public Builder addRootSymlinks(Map<PathFragment, Artifact> symlinks) { + rootSymlinksBuilder.addAll(symlinks.entrySet()); + return this; + } + + /** + * Adds several root symlinks as a NestedSet. + */ + public Builder addRootSymlinks(NestedSet<Map.Entry<PathFragment, Artifact>> symlinks) { + rootSymlinksBuilder.addTransitive(symlinks); + return this; + } + + /** + * Adds a pruning manifest. See {@link PruningManifest} for an explanation. + */ + public Builder addPruningManifest(PruningManifest manifest) { + pruningManifestsBuilder.add(manifest); + return this; + } + + /** + * Adds several pruning manifests as a NestedSet. See {@link PruningManifest} for an + * explanation. + */ + public Builder addPruningManifests(NestedSet<PruningManifest> manifests) { + pruningManifestsBuilder.addTransitive(manifests); + return this; + } + + /** + * Specify a function that can create additional manifest entries based on the input entries. + */ + public Builder setManifestExpander( + Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> expander) { + manifestExpander = Preconditions.checkNotNull(expander); + return this; + } + + /** + * Merges runfiles from a given runfiles support. + * + * @param runfilesSupport the runfiles support to be merged in + */ + public Builder merge(@Nullable RunfilesSupport runfilesSupport) { + return merge(runfilesSupport, null); + } + + /** + * Merges runfiles from a given runfiles support. + * + * <p>Sometimes a particular symlink from the runfiles support must not be included in runfiles. + * In such cases the path fragment denoting the symlink should be passed in as {@code + * ommittedAdditionalSymlink}. The symlink will then be filtered away from the set of additional + * symlinks of the target. + * + * @param runfilesSupport the runfiles support to be merged in + * @param omittedAdditionalSymlink the symlink to be omitted, or null if no filtering is needed + */ + public Builder merge(@Nullable RunfilesSupport runfilesSupport, + @Nullable final PathFragment omittedAdditionalSymlink) { + if (runfilesSupport == null) { + return this; + } + // TODO(bazel-team): We may be able to remove this now. + addArtifact(runfilesSupport.getRunfilesMiddleman()); + Runfiles runfiles = runfilesSupport.getRunfiles(); + if (omittedAdditionalSymlink == null) { + merge(runfiles); + } else { + artifactsBuilder.addTransitive(runfiles.getUnconditionalArtifacts()); + symlinksBuilder.addAll(Maps.filterKeys(entriesToMap(runfiles.getSymlinks()), + Predicates.not(Predicates.equalTo(omittedAdditionalSymlink))).entrySet()); + rootSymlinksBuilder.addTransitive(runfiles.getRootSymlinks()); + pruningManifestsBuilder.addTransitive(runfiles.getPruningManifests()); + if (manifestExpander == DUMMY_SYMLINK_EXPANDER) { + manifestExpander = runfiles.getSymlinkExpander(); + } else { + Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> otherExpander = + runfiles.getSymlinkExpander(); + Preconditions.checkState((otherExpander == DUMMY_SYMLINK_EXPANDER) + || manifestExpander.equals(otherExpander)); + } + } + return this; + } + + /** + * Adds the runfiles for a particular target and visits the transitive closure of "srcs", + * "deps" and "data", collecting all of their respective runfiles. + */ + public Builder addRunfiles(RuleContext ruleContext, + Function<TransitiveInfoCollection, Runfiles> mapping) { + Preconditions.checkNotNull(mapping); + Preconditions.checkNotNull(ruleContext); + addDataDeps(ruleContext); + addNonDataDeps(ruleContext, mapping); + return this; + } + + /** + * Adds the files specified by a mapping from the transitive info collection to the runfiles. + * + * <p>Dependencies in {@code srcs} and {@code deps} are considered. + */ + public Builder add(RuleContext ruleContext, + Function<TransitiveInfoCollection, Runfiles> mapping) { + Preconditions.checkNotNull(ruleContext); + Preconditions.checkNotNull(mapping); + for (TransitiveInfoCollection dep : getNonDataDeps(ruleContext)) { + Runfiles runfiles = mapping.apply(dep); + if (runfiles != null) { + merge(runfiles); + } + } + + return this; + } + + /** + * Collects runfiles from data dependencies of a target. + */ + public Builder addDataDeps(RuleContext ruleContext) { + addTargets(getPrerequisites(ruleContext, "data", Mode.DATA), + RunfilesProvider.DATA_RUNFILES); + return this; + } + + /** + * Collects runfiles from "srcs" and "deps" of a target. + */ + public Builder addNonDataDeps(RuleContext ruleContext, + Function<TransitiveInfoCollection, Runfiles> mapping) { + for (TransitiveInfoCollection target : getNonDataDeps(ruleContext)) { + addTargetExceptFileTargets(target, mapping); + } + return this; + } + + public Builder addTargets(Iterable<? extends TransitiveInfoCollection> targets, + Function<TransitiveInfoCollection, Runfiles> mapping) { + for (TransitiveInfoCollection target : targets) { + addTarget(target, mapping); + } + return this; + } + + public Builder addTarget(TransitiveInfoCollection target, + Function<TransitiveInfoCollection, Runfiles> mapping) { + return addTargetIncludingFileTargets(target, mapping); + } + + private Builder addTargetExceptFileTargets(TransitiveInfoCollection target, + Function<TransitiveInfoCollection, Runfiles> mapping) { + Runfiles runfiles = mapping.apply(target); + if (runfiles != null) { + merge(runfiles); + } + + return this; + } + + private Builder addTargetIncludingFileTargets(TransitiveInfoCollection target, + Function<TransitiveInfoCollection, Runfiles> mapping) { + if (target.getProvider(RunfilesProvider.class) == null + && mapping == RunfilesProvider.DATA_RUNFILES) { + // RuleConfiguredTarget implements RunfilesProvider, so this will only be called on + // FileConfiguredTarget instances. + // TODO(bazel-team): This is a terrible hack. We should be able to make this go away + // by implementing RunfilesProvider on FileConfiguredTarget. We'd need to be mindful + // of the memory use, though, since we have a whole lot of FileConfiguredTarget instances. + addTransitiveArtifacts(target.getProvider(FileProvider.class).getFilesToBuild()); + return this; + } + + return addTargetExceptFileTargets(target, mapping); + } + + /** + * Adds symlinks to given artifacts at their exec paths. + */ + public Builder addSymlinksToArtifacts(Iterable<Artifact> artifacts) { + for (Artifact artifact : artifacts) { + addSymlink(artifact.getExecPath(), artifact); + } + return this; + } + + /** + * Add the other {@link Runfiles} object transitively. + */ + public Builder merge(Runfiles runfiles) { + return merge(runfiles, true); + } + + /** + * Add the other {@link Runfiles} object transitively, but don't merge + * pruning manifests. + */ + public Builder mergeExceptPruningManifests(Runfiles runfiles) { + return merge(runfiles, false); + } + + /** + * Add the other {@link Runfiles} object transitively, with the option to include or exclude + * pruning manifests in the merge. + */ + private Builder merge(Runfiles runfiles, boolean includePruningManifests) { + artifactsBuilder.addTransitive(runfiles.getUnconditionalArtifacts()); + symlinksBuilder.addTransitive(runfiles.getSymlinks()); + rootSymlinksBuilder.addTransitive(runfiles.getRootSymlinks()); + if (includePruningManifests) { + pruningManifestsBuilder.addTransitive(runfiles.getPruningManifests()); + } + if (manifestExpander == DUMMY_SYMLINK_EXPANDER) { + manifestExpander = runfiles.getSymlinkExpander(); + } else { + Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> otherExpander = + runfiles.getSymlinkExpander(); + Preconditions.checkState((otherExpander == DUMMY_SYMLINK_EXPANDER) + || manifestExpander.equals(otherExpander)); + } + return this; + } + + private static Iterable<TransitiveInfoCollection> getNonDataDeps(RuleContext ruleContext) { + return Iterables.concat( + // TODO(bazel-team): This line shouldn't be here. Removing it requires that no rules have + // dependent rules in srcs (except for filegroups and such), but always in deps. + // TODO(bazel-team): DONT_CHECK is not optimal here. Rules that use split configs need to + // be changed not to call into here. + getPrerequisites(ruleContext, "srcs", Mode.DONT_CHECK), + getPrerequisites(ruleContext, "deps", Mode.DONT_CHECK)); + } + + /** + * For the specified attribute "attributeName" (which must be of type list(label)), resolves all + * the labels into ConfiguredTargets (for the same configuration as this one) and returns them + * as a list. + * + * <p>If the rule does not have the specified attribute, returns the empty list. + */ + private static Iterable<? extends TransitiveInfoCollection> getPrerequisites( + RuleContext ruleContext, String attributeName, Mode mode) { + if (ruleContext.getRule().isAttrDefined(attributeName, Type.LABEL_LIST)) { + return ruleContext.getPrerequisites(attributeName, mode); + } else { + return Collections.emptyList(); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RunfilesProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/RunfilesProvider.java new file mode 100644 index 0000000..2e26bbc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/RunfilesProvider.java
@@ -0,0 +1,91 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Runfiles a target contributes to targets that depend on it. + * + * <p>The set of runfiles contributed can be different if the dependency is through a + * <code>data</code> attribute (note that this is just a rough approximation of the reality -- + * rule implementations are free to request the data runfiles at any time) + */ +@Immutable +public final class RunfilesProvider implements TransitiveInfoProvider { + private final Runfiles defaultRunfiles; + private final Runfiles dataRunfiles; + + private RunfilesProvider(Runfiles defaultRunfiles, Runfiles dataRunfiles) { + this.defaultRunfiles = defaultRunfiles; + this.dataRunfiles = dataRunfiles; + } + + public Runfiles getDefaultRunfiles() { + return defaultRunfiles; + } + + public Runfiles getDataRunfiles() { + return dataRunfiles; + } + + /** + * Returns a function that gets the default runfiles from a {@link TransitiveInfoCollection} or + * the empty runfiles instance if it does not contain that provider. + */ + public static final Function<TransitiveInfoCollection, Runfiles> DEFAULT_RUNFILES = + new Function<TransitiveInfoCollection, Runfiles>() { + @Override + public Runfiles apply(TransitiveInfoCollection input) { + RunfilesProvider provider = input.getProvider(RunfilesProvider.class); + if (provider != null) { + return provider.getDefaultRunfiles(); + } + + return Runfiles.EMPTY; + } + }; + + /** + * Returns a function that gets the data runfiles from a {@link TransitiveInfoCollection} or the + * empty runfiles instance if it does not contain that provider. + * + * <p>These are usually used if the target is depended on through a {@code data} attribute. + */ + public static final Function<TransitiveInfoCollection, Runfiles> DATA_RUNFILES = + new Function<TransitiveInfoCollection, Runfiles>() { + @Override + public Runfiles apply(TransitiveInfoCollection input) { + RunfilesProvider provider = input.getProvider(RunfilesProvider.class); + if (provider != null) { + return provider.getDataRunfiles(); + } + + return Runfiles.EMPTY; + } + }; + + public static RunfilesProvider simple(Runfiles defaultRunfiles) { + return new RunfilesProvider(defaultRunfiles, defaultRunfiles); + } + + public static RunfilesProvider withData( + Runfiles defaultRunfiles, Runfiles dataRunfiles) { + return new RunfilesProvider(defaultRunfiles, dataRunfiles); + } + + public static final RunfilesProvider EMPTY = new RunfilesProvider( + Runfiles.EMPTY, Runfiles.EMPTY); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RunfilesSupport.java b/src/main/java/com/google/devtools/build/lib/analysis/RunfilesSupport.java new file mode 100644 index 0000000..aa7f429 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/RunfilesSupport.java
@@ -0,0 +1,382 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.SourceManifestAction.ManifestType; +import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.RunUnder; +import com.google.devtools.build.lib.collect.IterablesChain; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * This class manages the creation of the runfiles symlink farms. + * + * <p>For executables that might depend on the existence of files at run-time, we create a symlink + * farm: a directory which contains symlinks to the right locations for those runfiles. + * + * <p>The runfiles symlink farm serves two purposes. The first is to allow programs (and + * programmers) to refer to files using their workspace-relative paths, regardless of whether the + * files were source files or generated files, and regardless of which part of the package path they + * came from. The second purpose is to ensure that all run-time dependencies are explicitly declared + * in the BUILD files; programs may only use files which the build system knows that they depend on. + * + * <p>The symlink farm contains a MANIFEST file which describes its contents. The MANIFEST file + * lists the names and contents of all of the symlinks in the symlink farm. For efficiency, Blaze's + * dependency analysis ignores the actual symlinks and just looks at the MANIFEST file. It is an + * invariant that the MANIFEST file should accurately represent the contents of the symlinks + * whenever the MANIFEST file is present. build_runfile_links.py preserves this invariant (modulo + * bugs - currently it has a bug where it may fail to preserve that invariant if it gets + * interrupted). So the Blaze dependency analysis looks only at the MANIFEST file, rather than at + * the individual symlinks. + * + * <p>We create an Artifact for the MANIFEST file and a RunfilesAction Action to create it. This + * action does not depend on any other Artifacts. + * + * <p>When building an executable and running it, there are three things which must be built: the + * executable itself, the runfiles symlink farm (represented in the action graph by the Artifact for + * its MANIFEST), and the files pointed to by the symlinks in the symlink farm. To avoid redundancy + * in the dependency analysis, we create a Middleman Artifact which depends on all of these. Actions + * which will run an executable should depend on this Middleman Artifact. + */ +public class RunfilesSupport { + private static final String RUNFILES_DIR_EXT = ".runfiles"; + + private final Runfiles runfiles; + + private final Artifact runfilesInputManifest; + private final Artifact runfilesManifest; + private final Artifact runfilesMiddleman; + private final Artifact sourcesManifest; + private final Artifact owningExecutable; + private final boolean createSymlinks; + private final ImmutableList<String> args; + + /** + * Creates the RunfilesSupport helper with the given executable and runfiles. + * + * @param ruleContext the rule context to create the runfiles support for + * @param executable the executable for whose runfiles this runfiles support is responsible, may + * be null + * @param runfiles the runfiles + * @param appendingArgs to be added after the rule's args + */ + private RunfilesSupport(RuleContext ruleContext, Artifact executable, Runfiles runfiles, + List<String> appendingArgs, boolean createSymlinks) { + owningExecutable = executable; + this.createSymlinks = createSymlinks; + + // Adding run_under target to the runfiles manifest so it would become part + // of runfiles tree and would be executable everywhere. + RunUnder runUnder = ruleContext.getConfiguration().getRunUnder(); + if (runUnder != null && runUnder.getLabel() != null + && TargetUtils.isTestRule(ruleContext.getRule())) { + TransitiveInfoCollection runUnderTarget = + ruleContext.getPrerequisite(":run_under", Mode.DATA); + runfiles = new Runfiles.Builder() + .merge(getRunfiles(runUnderTarget)) + .merge(runfiles) + .build(); + } + this.runfiles = runfiles; + + Preconditions.checkState(!runfiles.isEmpty()); + + Map<PathFragment, Artifact> symlinks = getRunfilesSymlinks(); + if (executable != null && !symlinks.values().contains(executable)) { + throw new IllegalStateException("main program " + executable + " not included in runfiles"); + } + + runfilesInputManifest = createRunfilesInputManifestArtifact(ruleContext); + this.runfilesManifest = createRunfilesAction(ruleContext, runfiles); + this.runfilesMiddleman = createRunfilesMiddleman(ruleContext, runfiles.getAllArtifacts()); + sourcesManifest = createSourceManifest(ruleContext, runfiles); + args = ImmutableList.<String>builder() + .addAll(ruleContext.getTokenizedStringListAttr("args")) + .addAll(appendingArgs) + .build(); + } + + private RunfilesSupport(Runfiles runfiles, Artifact runfilesInputManifest, + Artifact runfilesManifest, Artifact runfilesMiddleman, Artifact sourcesManifest, + Artifact owningExecutable, boolean createSymlinks, ImmutableList<String> args) { + this.runfiles = runfiles; + this.runfilesInputManifest = runfilesInputManifest; + this.runfilesManifest = runfilesManifest; + this.runfilesMiddleman = runfilesMiddleman; + this.sourcesManifest = sourcesManifest; + this.owningExecutable = owningExecutable; + this.createSymlinks = createSymlinks; + this.args = args; + } + + /** + * Returns the executable owning this RunfilesSupport. Only use from Skylark. + */ + public Artifact getExecutable() { + return owningExecutable; + } + + /** + * Returns the exec path of the directory where the runfiles contained in this + * RunfilesSupport are generated. When the owning rule has no executable, + * returns null. + */ + public PathFragment getRunfilesDirectoryExecPath() { + if (owningExecutable == null) { + return null; + } + + PathFragment executablePath = owningExecutable.getExecPath(); + return executablePath.getParentDirectory().getChild( + executablePath.getBaseName() + RUNFILES_DIR_EXT); + } + + public Runfiles getRunfiles() { + return runfiles; + } + + /** + * For executable programs, the .runfiles_manifest file outside of the + * runfiles symlink farm; otherwise, returns null. + * + * <p>The MANIFEST file represents the contents of all of the symlinks in the + * symlink farm. For efficiency, Blaze's dependency analysis ignores the + * actual symlinks and just looks at the MANIFEST file. It is an invariant + * that the MANIFEST file should accurately represent the contents of the + * symlinks whenever the MANIFEST file is present. + */ + public Artifact getRunfilesInputManifest() { + return runfilesInputManifest; + } + + private Artifact createRunfilesInputManifestArtifact(ActionConstructionContext context) { + // The executable may be null for emptyRunfiles + PathFragment relativePath = (owningExecutable != null) + ? owningExecutable.getRootRelativePath() + : Util.getWorkspaceRelativePath(context.getRule()); + String basename = relativePath.getBaseName(); + PathFragment inputManifestPath = relativePath.replaceName(basename + ".runfiles_manifest"); + return context.getAnalysisEnvironment().getDerivedArtifact(inputManifestPath, + context.getConfiguration().getBinDirectory()); + } + + /** + * For executable programs, returns the MANIFEST file in the runfiles + * symlink farm, if blaze is run with --build_runfile_links; returns + * the .runfiles_manifest file outside of the symlink farm, if blaze + * is run with --nobuild_runfile_links. + * <p> + * Beware: In most cases {@link #getRunfilesInputManifest} is the more + * appropriate function. + */ + public Artifact getRunfilesManifest() { + return runfilesManifest; + } + + /** + * For executable programs, the root directory of the runfiles symlink farm; + * otherwise, returns null. + */ + public Path getRunfilesDirectory() { + return FileSystemUtils.replaceExtension(getRunfilesInputManifest().getPath(), RUNFILES_DIR_EXT); + } + + /** + * Returns the files pointed to by the symlinks in the runfiles symlink farm. This method is slow. + */ + @VisibleForTesting + public Collection<Artifact> getRunfilesSymlinkTargets() { + return getRunfilesSymlinks().values(); + } + + /** + * Returns the names of the symlinks in the runfiles symlink farm as a Set of PathFragments. This + * method is slow. + */ + // We should make this VisibleForTesting, but it is still used by TestHelper + public Set<PathFragment> getRunfilesSymlinkNames() { + return getRunfilesSymlinks().keySet(); + } + + /** + * Returns the names of the symlinks in the runfiles symlink farm as a Set of PathFragments. This + * method is slow. + */ + @VisibleForTesting + public Map<PathFragment, Artifact> getRunfilesSymlinks() { + return runfiles.asMapWithoutRootSymlinks(); + } + + /** + * Returns both runfiles artifacts and "conditional" artifacts that may be part of a + * Runfiles PruningManifest. This means the returned set may be an overapproximation of the + * actual set of runfiles (see {@link Runfiles.PruningManifest}). + */ + public Iterable<Artifact> getRunfilesArtifactsWithoutMiddlemen() { + return runfiles.getArtifactsWithoutMiddlemen(); + } + + /** + * Returns the middleman artifact that depends on getExecutable(), + * getRunfilesManifest(), and getRunfilesSymlinkTargets(). Anything which + * needs to actually run the executable should depend on this. + */ + public Artifact getRunfilesMiddleman() { + return runfilesMiddleman; + } + + /** + * Returns the Sources manifest. + * This may be null if the owningRule has no executable. + */ + public Artifact getSourceManifest() { + return sourcesManifest; + } + + private Artifact createRunfilesMiddleman(ActionConstructionContext context, + Iterable<Artifact> allRunfilesArtifacts) { + Iterable<Artifact> inputs = IterablesChain.<Artifact>builder() + .add(allRunfilesArtifacts) + .addElement(runfilesManifest) + .build(); + return context.getAnalysisEnvironment().getMiddlemanFactory().createRunfilesMiddleman( + context.getActionOwner(), owningExecutable, inputs, + context.getConfiguration().getMiddlemanDirectory()); + } + + /** + * Creates a runfiles action for all of the specified files, and returns the + * output artifact (the artifact for the MANIFEST file). + * + * <p>The "runfiles" action creates a symlink farm that links all the runfiles + * (which may come from different places, e.g. different package paths, + * generated files, etc.) into a single tree, so that programs can access them + * using the workspace-relative name. + */ + private Artifact createRunfilesAction(ActionConstructionContext context, Runfiles runfiles) { + // Compute the names of the runfiles directory and its MANIFEST file. + Artifact inputManifest = getRunfilesInputManifest(); + context.getAnalysisEnvironment().registerAction( + SourceManifestAction.forRunfiles( + ManifestType.SOURCE_SYMLINKS, context.getActionOwner(), inputManifest, runfiles)); + + if (!createSymlinks) { + // Just return the manifest if that's all the build calls for. + return inputManifest; + } + + PathFragment runfilesDir = FileSystemUtils.replaceExtension(inputManifest.getRootRelativePath(), + RUNFILES_DIR_EXT); + PathFragment outputManifestPath = runfilesDir.getRelative("MANIFEST"); + + BuildConfiguration config = context.getConfiguration(); + Artifact outputManifest = context.getAnalysisEnvironment().getDerivedArtifact( + outputManifestPath, config.getBinDirectory()); + context.getAnalysisEnvironment().registerAction(new SymlinkTreeAction( + context.getActionOwner(), inputManifest, outputManifest, /*filesetTree=*/false)); + return outputManifest; + } + + /** + * Creates an Artifact which writes the "sources only" manifest file. + * + * @param context the owner for the manifest action + * @param runfiles the runfiles + * @return the Artifact representing the file write action. + */ + private Artifact createSourceManifest(ActionConstructionContext context, Runfiles runfiles) { + // Put the sources only manifest next to the MANIFEST file but call it SOURCES. + PathFragment runfilesDir = getRunfilesDirectoryExecPath(); + if (runfilesDir != null) { + PathFragment sourcesManifestPath = runfilesDir.getRelative("SOURCES"); + Artifact sourceOnlyManifest = context.getAnalysisEnvironment().getDerivedArtifact( + sourcesManifestPath, context.getConfiguration().getBinDirectory()); + context.getAnalysisEnvironment().registerAction( + SourceManifestAction.forRunfiles( + ManifestType.SOURCES_ONLY, context.getActionOwner(), sourceOnlyManifest, runfiles)); + return sourceOnlyManifest; + } else { + return null; + } + } + + /** + * Helper method that returns a collection of artifacts that are necessary for the runfiles of the + * given target. Note that the runfile symlink tree is never built, so this may include artifacts + * that end up not being used (see {@link Runfiles}). + * + * @return the Runfiles object + */ + + private static Runfiles getRunfiles(TransitiveInfoCollection target) { + RunfilesProvider runfilesProvider = target.getProvider(RunfilesProvider.class); + if (runfilesProvider != null) { + return runfilesProvider.getDefaultRunfiles(); + } else { + return Runfiles.EMPTY; + } + } + + /** + * Returns the unmodifiable list of expanded and tokenized 'args' attribute + * values. + */ + public List<String> getArgs() { + return args; + } + + /** + * Creates and returns a RunfilesSupport object for the given rule and executable. Note that this + * method calls back into the passed in rule to obtain the runfiles. + */ + public static RunfilesSupport withExecutable(RuleContext ruleContext, Runfiles runfiles, + Artifact executable) { + return new RunfilesSupport(ruleContext, executable, runfiles, ImmutableList.<String>of(), + ruleContext.shouldCreateRunfilesSymlinks()); + } + + /** + * Creates and returns a RunfilesSupport object for the given rule and executable. Note that this + * method calls back into the passed in rule to obtain the runfiles. + */ + public static RunfilesSupport withExecutable(RuleContext ruleContext, Runfiles runfiles, + Artifact executable, boolean createSymlinks) { + return new RunfilesSupport(ruleContext, executable, runfiles, ImmutableList.<String>of(), + createSymlinks); + } + + /** + * Creates and returns a RunfilesSupport object for the given rule, executable, runfiles and args. + */ + public static RunfilesSupport withExecutable(RuleContext ruleContext, Runfiles runfiles, + Artifact executable, List<String> appendingArgs) { + return new RunfilesSupport(ruleContext, executable, runfiles, + ImmutableList.copyOf(appendingArgs), ruleContext.shouldCreateRunfilesSymlinks()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/SourceManifestAction.java b/src/main/java/com/google/devtools/build/lib/analysis/SourceManifestAction.java new file mode 100644 index 0000000..5aa3bdc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/SourceManifestAction.java
@@ -0,0 +1,404 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Action to create a manifest of input files for processing by a subsequent + * build step (e.g. runfiles symlinking or archive building). + * + * <p>The manifest's format is specifiable by {@link ManifestType}, in + * accordance with the needs of the calling functionality. + * + * <p>Note that this action carefully avoids building the manifest content in + * memory. + */ +public class SourceManifestAction extends AbstractFileWriteAction { + /** + * Action context that tells what workspace suffix we should use. + */ + public interface Context extends ActionContext { + PathFragment getRunfilesPrefix(); + } + + private static final String GUID = "07459553-a3d0-4d37-9d78-18ed942470f4"; + + /** + * Interface for defining manifest formatting and reporting specifics. + */ + @VisibleForTesting + interface ManifestWriter { + + /** + * Writes a single line of manifest output. + * + * @param manifestWriter the output stream + * @param rootRelativePath path of an entry relative to the manifest's root + * @param symlink (optional) symlink that resolves the above path + */ + void writeEntry(Writer manifestWriter, PathFragment rootRelativePath, + @Nullable Artifact symlink) throws IOException; + + /** + * Fulfills {@link #ActionMetadata.getMnemonic()} + */ + String getMnemonic(); + + /** + * Fulfills {@link #AbstractAction.getRawProgressMessage()} + */ + String getRawProgressMessage(); + } + + /** + * The strategy we use to write manifest entries. + */ + private final ManifestWriter manifestWriter; + + /** + * The runfiles for which to create the symlink tree. + */ + private final Runfiles runfiles; + + /** + * If non-null, the paths should be computed relative to this path fragment. + */ + private final PathFragment root; + + /** + * Creates a new AbstractSourceManifestAction instance using latin1 encoding + * to write the manifest file and with a specified root path for manifest entries. + * + * @param manifestWriter the strategy to use to write manifest entries + * @param owner the action owner + * @param output the file to which to write the manifest + * @param runfiles runfiles + * @param root the artifacts' root-relative path is relativized to this before writing it out + */ + private SourceManifestAction(ManifestWriter manifestWriter, ActionOwner owner, Artifact output, + Runfiles runfiles, PathFragment root) { + super(owner, getDependencies(runfiles), output, false); + this.manifestWriter = manifestWriter; + this.runfiles = runfiles; + this.root = root; + } + + @VisibleForTesting + public void writeOutputFile(OutputStream out, EventHandler eventHandler, String workspaceSuffix) + throws IOException { + writeFile(out, runfiles.getRunfilesInputs( + root, workspaceSuffix, eventHandler, getOwner().getLocation())); + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) + throws IOException { + final Pair<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> runfilesInputs = + runfiles.getRunfilesInputs(root, + executor.getContext(Context.class).getRunfilesPrefix().toString(), eventHandler, + getOwner().getLocation()); + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + writeFile(out, runfilesInputs); + } + }; + } + + /** + * Returns the input dependencies for this action. Note we don't need to create the symlink + * target Artifacts before we write the output manifest, so this Action does not have to + * depend on them. The only necessary dependencies are pruning manifests, which must be read + * to properly prune the tree. + */ + private static Collection<Artifact> getDependencies(Runfiles runfiles) { + ImmutableList.Builder<Artifact> builder = ImmutableList.builder(); + for (Runfiles.PruningManifest manifest : runfiles.getPruningManifests()) { + builder.add(manifest.getManifestFile()); + } + return builder.build(); + } + + /** + * Sort the entries in both the normal and root manifests and write the output + * file. + * + * @param out is the message stream to write errors to. + * @param output The actual mapping of the output manifest. + * @throws IOException + */ + private void writeFile(OutputStream out, + Pair<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> output) + throws IOException { + Writer manifestFile = new BufferedWriter(new OutputStreamWriter(out, ISO_8859_1)); + + Comparator<Map.Entry<PathFragment, Artifact>> fragmentComparator = + new Comparator<Map.Entry<PathFragment, Artifact>>() { + @Override + public int compare(Map.Entry<PathFragment, Artifact> path1, + Map.Entry<PathFragment, Artifact> path2) { + return path1.getKey().compareTo(path2.getKey()); + } + }; + + List<Map.Entry<PathFragment, Artifact>> sortedRootLinks = + new ArrayList<>(output.second.entrySet()); + Collections.sort(sortedRootLinks, fragmentComparator); + + List<Map.Entry<PathFragment, Artifact>> sortedManifest = + new ArrayList<>(output.first.entrySet()); + Collections.sort(sortedManifest, fragmentComparator); + + for (Map.Entry<PathFragment, Artifact> line : sortedRootLinks) { + manifestWriter.writeEntry(manifestFile, line.getKey(), line.getValue()); + } + + for (Map.Entry<PathFragment, Artifact> line : sortedManifest) { + manifestWriter.writeEntry(manifestFile, line.getKey(), line.getValue()); + } + manifestFile.flush(); + } + + @Override + public String getMnemonic() { + return manifestWriter.getMnemonic(); + } + + @Override + protected String getRawProgressMessage() { + return manifestWriter.getRawProgressMessage() + " for " + getOwner().getLabel(); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + Map<PathFragment, Artifact> symlinks = runfiles.getSymlinksAsMap(); + f.addInt(symlinks.size()); + for (Map.Entry<PathFragment, Artifact> symlink : symlinks.entrySet()) { + f.addPath(symlink.getKey()); + f.addPath(symlink.getValue().getPath()); + } + Map<PathFragment, Artifact> rootSymlinks = runfiles.getRootSymlinksAsMap(); + f.addInt(rootSymlinks.size()); + for (Map.Entry<PathFragment, Artifact> rootSymlink : rootSymlinks.entrySet()) { + f.addPath(rootSymlink.getKey()); + f.addPath(rootSymlink.getValue().getPath()); + } + + if (root != null) { + for (Artifact artifact : runfiles.getArtifactsWithoutMiddlemen()) { + f.addPath(artifact.getRootRelativePath().relativeTo(root)); + f.addPath(artifact.getPath()); + } + } else { + for (Artifact artifact : runfiles.getArtifactsWithoutMiddlemen()) { + f.addPath(artifact.getRootRelativePath()); + f.addPath(artifact.getPath()); + } + } + return f.hexDigestAndReset(); + } + + /** + * Supported manifest writing strategies. + */ + public static enum ManifestType implements ManifestWriter { + + /** + * Writes each line as: + * + * [rootRelativePath] [resolvingSymlink] + * + * <p>This strategy is suitable for creating an input manifest to a source view tree. Its + * output is a valid input to {@link com.google.devtools.build.lib.analysis.SymlinkTreeAction}. + */ + SOURCE_SYMLINKS { + @Override + public void writeEntry(Writer manifestWriter, PathFragment rootRelativePath, Artifact symlink) + throws IOException { + manifestWriter.append(rootRelativePath.getPathString()); + // This trailing whitespace is REQUIRED to process the single entry line correctly. + manifestWriter.append(' '); + if (symlink != null) { + manifestWriter.append(symlink.getPath().getPathString()); + } + manifestWriter.append('\n'); + } + + @Override + public String getMnemonic() { + return "SourceSymlinkManifest"; + } + + @Override + public String getRawProgressMessage() { + return "Creating source manifest"; + } + }, + + /** + * Writes each line as: + * + * [rootRelativePath] + * + * <p>This strategy is suitable for an input into a packaging system (notably .par) that + * consumes a list of all source files but needs that list to be constant with respect to + * how the user has their client laid out on local disk. + */ + SOURCES_ONLY { + @Override + public void writeEntry(Writer manifestWriter, PathFragment rootRelativePath, Artifact symlink) + throws IOException { + manifestWriter.append(rootRelativePath.getPathString()); + manifestWriter.append('\n'); + manifestWriter.flush(); + } + + @Override + public String getMnemonic() { + return "PackagingSourcesManifest"; + } + + @Override + public String getRawProgressMessage() { + return "Creating file sources list"; + } + } + } + + /** Creates an action for the given runfiles. */ + public static SourceManifestAction forRunfiles(ManifestType manifestType, ActionOwner owner, + Artifact output, Runfiles runfiles) { + return new SourceManifestAction(manifestType, owner, output, runfiles, null); + } + + /** + * Builder class to construct {@link SourceManifestAction} instances. + */ + public static final class Builder { + private final ManifestWriter manifestWriter; + private final ActionOwner owner; + private final Artifact output; + private PathFragment top; + private final Runfiles.Builder runfilesBuilder = new Runfiles.Builder(); + + public Builder(ManifestType manifestType, ActionOwner owner, Artifact output) { + manifestWriter = manifestType; + this.owner = owner; + this.output = output; + } + + @VisibleForTesting + Builder(ManifestWriter manifestWriter, ActionOwner owner, Artifact output) { + this.manifestWriter = manifestWriter; + this.owner = owner; + this.output = output; + } + + public SourceManifestAction build() { + return new SourceManifestAction(manifestWriter, owner, output, runfilesBuilder.build(), top); + } + + /** + * Sets the path fragment which is used to relativize the artifacts' root + * relative paths further. Most likely, you don't need this. + */ + public Builder setTopLevel(PathFragment top) { + this.top = top; + return this; + } + + /** + * Adds a set of symlinks from the artifacts' root-relative paths to the + * artifacts themselves. + */ + public Builder addSymlinks(Iterable<Artifact> artifacts) { + runfilesBuilder.addArtifacts(artifacts); + return this; + } + + /** + * Adds a map of symlinks. + */ + public Builder addSymlinks(Map<PathFragment, Artifact> symlinks) { + runfilesBuilder.addSymlinks(symlinks); + return this; + } + + /** + * Adds a single symlink. + */ + public Builder addSymlink(PathFragment link, Artifact target) { + runfilesBuilder.addSymlink(link, target); + return this; + } + + /** + * <p>Adds a mapping of Artifacts to the directory above the normal symlink + * forest base. + */ + public Builder addRootSymlinks(Map<PathFragment, Artifact> rootSymlinks) { + runfilesBuilder.addRootSymlinks(rootSymlinks); + return this; + } + + /** + * Set an expander function for the symlinks. + */ + @VisibleForTesting + Builder setSymlinksExpander( + Function<Map<PathFragment, Artifact>, Map<PathFragment, Artifact>> expander) { + runfilesBuilder.setManifestExpander(expander); + return this; + } + + /** + * Adds a runfiles pruning manifest. + */ + @VisibleForTesting + Builder addPruningManifest(Runfiles.PruningManifest manifest) { + runfilesBuilder.addPruningManifest(manifest); + return this; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/SymlinkTreeAction.java b/src/main/java/com/google/devtools/build/lib/analysis/SymlinkTreeAction.java new file mode 100644 index 0000000..2dc0d4a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/SymlinkTreeAction.java
@@ -0,0 +1,109 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.util.Fingerprint; + +/** + * Action responsible for the symlink tree creation. + * Used to generate runfiles and fileset symlink farms. + */ +public class SymlinkTreeAction extends AbstractAction { + + private static final String GUID = "63412bda-4026-4c8e-a3ad-7deb397728d4"; + + private final Artifact inputManifest; + private final Artifact outputManifest; + private final boolean filesetTree; + + /** + * Creates SymlinkTreeAction instance. + * + * @param owner action owner + * @param inputManifest exec path to the input runfiles manifest + * @param outputManifest exec path to the generated symlink tree manifest + * (must have "MANIFEST" base name). Symlink tree root + * will be set to the artifact's parent directory. + * @param filesetTree true if this is fileset symlink tree, + * false if this is a runfiles symlink tree. + */ + public SymlinkTreeAction(ActionOwner owner, Artifact inputManifest, Artifact outputManifest, + boolean filesetTree) { + super(owner, ImmutableList.of(inputManifest), ImmutableList.of(outputManifest)); + Preconditions.checkArgument(outputManifest.getPath().getBaseName().equals("MANIFEST")); + this.inputManifest = inputManifest; + this.outputManifest = outputManifest; + this.filesetTree = filesetTree; + } + + public Artifact getInputManifest() { + return inputManifest; + } + + public Artifact getOutputManifest() { + return outputManifest; + } + + public boolean isFilesetTree() { + return filesetTree; + } + + @Override + public String getMnemonic() { + return "SymlinkTree"; + } + + @Override + protected String getRawProgressMessage() { + return (filesetTree ? "Creating Fileset tree " : "Creating runfiles tree ") + + outputManifest.getExecPath().getParentDirectory().getPathString(); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addInt(filesetTree ? 1 : 0); + return f.hexDigestAndReset(); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + // Return null here to indicate that resources would be managed manually + // during action execution. + return null; + } + + @Override + public String describeStrategy(Executor executor) { + return "local"; // Symlink tree is always generated locally. + } + + @Override + public void execute( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + actionExecutionContext.getExecutor().getContext(SymlinkTreeActionContext.class) + .createSymlinks(this, actionExecutionContext); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/SymlinkTreeActionContext.java b/src/main/java/com/google/devtools/build/lib/analysis/SymlinkTreeActionContext.java new file mode 100644 index 0000000..fe61056 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/SymlinkTreeActionContext.java
@@ -0,0 +1,31 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +/** + * Action context for symlink tree actions (an action that creates a tree of symlinks). + */ +public interface SymlinkTreeActionContext extends ActionContext { + + /** + * Creates the symlink tree. + */ + void createSymlinks(SymlinkTreeAction action, + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TargetAndConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/TargetAndConfiguration.java new file mode 100644 index 0000000..cc0feb6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/TargetAndConfiguration.java
@@ -0,0 +1,104 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * Refers to the pair of a target and a configuration and certain additional information. Not the + * same as {@link ConfiguredTarget} -- that also contains the result of the analysis phase. + */ +public class TargetAndConfiguration { + private final Target target; + @Nullable private final BuildConfiguration configuration; + + public TargetAndConfiguration(Target target, @Nullable BuildConfiguration configuration) { + this.target = Preconditions.checkNotNull(target); + this.configuration = configuration; + } + + public TargetAndConfiguration(ConfiguredTarget configuredTarget) { + this.target = Preconditions.checkNotNull(configuredTarget).getTarget(); + this.configuration = configuredTarget.getConfiguration(); + } + + // The node name in the graph. The name should be unique. + // It is not suitable for user display. + public String getName() { + return target.getLabel() + " " + + (configuration == null ? "null" : configuration.shortCacheKey()); + } + + public static final Function<TargetAndConfiguration, String> NAME_FUNCTION = + new Function<TargetAndConfiguration, String>() { + @Override + public String apply(TargetAndConfiguration node) { + return node.getName(); + } + }; + + public static final Function<TargetAndConfiguration, ConfiguredTargetKey> + TO_LABEL_AND_CONFIGURATION = new Function<TargetAndConfiguration, ConfiguredTargetKey>() { + @Override + public ConfiguredTargetKey apply(TargetAndConfiguration input) { + return new ConfiguredTargetKey(input.getLabel(), input.getConfiguration()); + } + }; + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + if (!(that instanceof TargetAndConfiguration)) { + return false; + } + + TargetAndConfiguration thatNode = (TargetAndConfiguration) that; + return thatNode.target.getLabel().equals(this.target.getLabel()) && + thatNode.configuration == this.configuration; + } + + @Override + public int hashCode() { + return Objects.hash(target.getLabel(), configuration); + } + + @Override + public String toString() { + return target.getLabel() + " (" + configuration + ")"; + } + + public Target getTarget() { + return target; + } + + public Label getLabel() { + return target.getLabel(); + } + + @Nullable + public BuildConfiguration getConfiguration() { + return configuration; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java new file mode 100644 index 0000000..fcfec55 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java
@@ -0,0 +1,75 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * This event is fired as soon as a target is either built or fails. + */ +public final class TargetCompleteEvent implements SkyValue { + + private final ConfiguredTarget target; + private final NestedSet<Label> rootCauses; + + private TargetCompleteEvent(ConfiguredTarget target, NestedSet<Label> rootCauses) { + this.target = target; + this.rootCauses = (rootCauses == null) + ? NestedSetBuilder.<Label>emptySet(Order.STABLE_ORDER) + : rootCauses; + } + + /** + * Construct a successful target completion event. + */ + public static TargetCompleteEvent createSuccessful(ConfiguredTarget ct) { + return new TargetCompleteEvent(ct, null); + } + + /** + * Construct a target completion event for a failed target, with the given non-empty root causes. + */ + public static TargetCompleteEvent createFailed(ConfiguredTarget ct, NestedSet<Label> rootCauses) { + Preconditions.checkArgument(!Iterables.isEmpty(rootCauses)); + return new TargetCompleteEvent(ct, rootCauses); + } + + /** + * Returns the target associated with the event. + */ + public ConfiguredTarget getTarget() { + return target; + } + + /** + * Determines whether the target has failed or succeeded. + */ + public boolean failed() { + return !rootCauses.isEmpty(); + } + + /** + * Get the root causes of the target. May be empty. + */ + public Iterable<Label> getRootCauses() { + return rootCauses; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TargetContext.java b/src/main/java/com/google/devtools/build/lib/analysis/TargetContext.java new file mode 100644 index 0000000..9c95db3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/TargetContext.java
@@ -0,0 +1,96 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.packages.PackageSpecification; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.List; + +import javax.annotation.Nullable; + +/** + * A helper class for building {@link ConfiguredTarget} instances, in particular for non-rule ones. + * For {@link RuleConfiguredTarget} instances, use {@link RuleContext} instead, + * which is a subclass of this class. + * + * <p>The class is intended to be sub-classed by RuleContext, in order to share the code. However, + * it's not intended for sub-classing beyond that, and the constructor is intentionally package + * private to enforce that. + */ +public class TargetContext { + + private final AnalysisEnvironment env; + private final Target target; + private final BuildConfiguration configuration; + /** + * This list only contains prerequisites that are not declared in rule attributes, with the + * exception of visibility (i.e., visibility is represented here, even though it is a rule + * attribute in case of a rule). Rule attributes are handled by the {@link RuleContext} subclass. + */ + private final List<ConfiguredTarget> directPrerequisites; + private final NestedSet<PackageSpecification> visibility; + + /** + * The constructor is intentionally package private. + */ + TargetContext(AnalysisEnvironment env, Target target, BuildConfiguration configuration, + List<ConfiguredTarget> directPrerequisites, + NestedSet<PackageSpecification> visibility) { + this.env = env; + this.target = target; + this.configuration = configuration; + this.directPrerequisites = directPrerequisites; + this.visibility = visibility; + } + + public AnalysisEnvironment getAnalysisEnvironment() { + return env; + } + + public Target getTarget() { + return target; + } + + public Label getLabel() { + return target.getLabel(); + } + + /** + * Returns the configuration for this target. This may return null if the target is supposed to be + * configuration-independent (like an input file, or a visibility rule). However, this is + * guaranteed to be non-null for rules and for output files. + */ + @Nullable + public BuildConfiguration getConfiguration() { + return configuration; + } + + public NestedSet<PackageSpecification> getVisibility() { + return visibility; + } + + TransitiveInfoCollection findDirectPrerequisite(Label label, BuildConfiguration config) { + for (ConfiguredTarget prerequisite : directPrerequisites) { + if (prerequisite.getLabel().equals(label) && (prerequisite.getConfiguration() == config)) { + return prerequisite; + } + } + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TempsProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/TempsProvider.java new file mode 100644 index 0000000..109992e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/TempsProvider.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.Collection; + +/** + * A {@link TransitiveInfoProvider} for rule classes that save extra files when + * {@code --save_temps} is in effect. + */ +@Immutable +public final class TempsProvider implements TransitiveInfoProvider { + + private final ImmutableList<Artifact> temps; + + public TempsProvider(ImmutableList<Artifact> temps) { + this.temps = temps; + } + + /** + * Return the extra artifacts to save when {@code --save_temps} is in effect. + */ + public Collection<Artifact> getTemps() { + return temps; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactContext.java b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactContext.java new file mode 100644 index 0000000..c9b8e51 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactContext.java
@@ -0,0 +1,104 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.Objects; +import java.util.Set; + +/** + * Contains options which control the set of artifacts to build for top-level targets. + */ +@Immutable +public final class TopLevelArtifactContext { + + public static final TopLevelArtifactContext DEFAULT = new TopLevelArtifactContext( + "", /*compileOnly=*/false, /*compilationPrerequisitesOnly*/false, + /*runTestsExclusively=*/false, /*outputGroups=*/ImmutableSet.<String>of(), + /*shouldRunTests=*/false); + + private final String buildCommand; + private final boolean compileOnly; + private final boolean compilationPrerequisitesOnly; + private final boolean runTestsExclusively; + private final ImmutableSet<String> outputGroups; + private final boolean shouldRunTests; + + public TopLevelArtifactContext(String buildCommand, boolean compileOnly, + boolean compilationPrerequisitesOnly, boolean runTestsExclusively, + ImmutableSet<String> outputGroups, boolean shouldRunTests) { + this.buildCommand = buildCommand; + this.compileOnly = compileOnly; + this.compilationPrerequisitesOnly = compilationPrerequisitesOnly; + this.runTestsExclusively = runTestsExclusively; + this.outputGroups = outputGroups; + this.shouldRunTests = shouldRunTests; + } + + /** Returns the build command as a string. */ + public String buildCommand() { + return buildCommand; + } + + /** Returns the value of the --compile_only flag. */ + public boolean compileOnly() { + return compileOnly; + } + + /** Returns the value of the --compilation_prerequisites_only flag. */ + public boolean compilationPrerequisitesOnly() { + return compilationPrerequisitesOnly; + } + + /** Whether to run tests in exclusive mode. */ + public boolean runTestsExclusively() { + return runTestsExclusively; + } + + /** Returns the value of the --output_groups flag. */ + public Set<String> outputGroups() { + return outputGroups; + } + + /** Whether the top-level request command may run tests. */ + public boolean shouldRunTests() { + return shouldRunTests; + } + + // TopLevelArtifactContexts are stored in maps in BuildView, + // so equals() and hashCode() need to work. + @Override + public boolean equals(Object other) { + if (other instanceof TopLevelArtifactContext) { + TopLevelArtifactContext otherContext = (TopLevelArtifactContext) other; + return buildCommand.equals(otherContext.buildCommand) + && compileOnly == otherContext.compileOnly + && compilationPrerequisitesOnly == otherContext.compilationPrerequisitesOnly + && runTestsExclusively == otherContext.runTestsExclusively + && outputGroups.equals(otherContext.outputGroups) + && shouldRunTests == otherContext.shouldRunTests; + } else { + return false; + } + } + + @Override + public int hashCode() { + return Objects.hash(buildCommand, compileOnly, compilationPrerequisitesOnly, + runTestsExclusively, outputGroups, shouldRunTests); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactHelper.java b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactHelper.java new file mode 100644 index 0000000..3c025f4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactHelper.java
@@ -0,0 +1,158 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.test.TestProvider; + +/** + * A small static class containing utility methods for handling the inclusion of + * extra top-level artifacts into the build. + */ +public final class TopLevelArtifactHelper { + + private TopLevelArtifactHelper() { + // Prevent instantiation. + } + + /** Returns command-specific artifacts which may exist for a given target and build command. */ + public static final Iterable<Artifact> getCommandArtifacts(TransitiveInfoCollection target, + String buildCommand) { + TopLevelArtifactProvider provider = target.getProvider(TopLevelArtifactProvider.class); + if (provider != null + && provider.getCommandsForExtraArtifacts().contains(buildCommand.toLowerCase())) { + return provider.getArtifactsForCommand(); + } else { + return ImmutableList.of(); + } + } + + /** + * Utility function to form a list of all test output Artifacts of the given targets to test. + */ + public static ImmutableCollection<Artifact> getAllArtifactsToTest( + Iterable<? extends TransitiveInfoCollection> targets) { + if (targets == null) { + return ImmutableList.of(); + } + ImmutableList.Builder<Artifact> allTestArtifacts = ImmutableList.builder(); + for (TransitiveInfoCollection target : targets) { + allTestArtifacts.addAll(TestProvider.getTestStatusArtifacts(target)); + } + return allTestArtifacts.build(); + } + + /** + * Utility function to form a NestedSet of all top-level Artifacts of the given targets. + */ + public static NestedSet<Artifact> getAllArtifactsToBuild( + Iterable<? extends TransitiveInfoCollection> targets, TopLevelArtifactContext options) { + NestedSetBuilder<Artifact> allArtifacts = NestedSetBuilder.stableOrder(); + for (TransitiveInfoCollection target : targets) { + allArtifacts.addTransitive(getAllArtifactsToBuild(target, options)); + } + return allArtifacts.build(); + } + + /** + * Returns all artifacts to build if this target is requested as a top-level target. The resulting + * set includes the temps and either the files to compile, if + * {@code options.compileOnly() == true}, or the files to run. + * + * <p>Calls to this method should generally return quickly; however, the runfiles computation can + * be lazy, in which case it can be expensive on the first call. Subsequent calls may or may not + * return the same {@code Iterable} instance. + */ + public static NestedSet<Artifact> getAllArtifactsToBuild(TransitiveInfoCollection target, + TopLevelArtifactContext options) { + NestedSetBuilder<Artifact> allArtifacts = NestedSetBuilder.stableOrder(); + TempsProvider tempsProvider = target.getProvider(TempsProvider.class); + if (tempsProvider != null) { + allArtifacts.addAll(tempsProvider.getTemps()); + } + + TopLevelArtifactProvider topLevelArtifactProvider = + target.getProvider(TopLevelArtifactProvider.class); + if (topLevelArtifactProvider != null) { + for (String outputGroup : options.outputGroups()) { + NestedSet<Artifact> results = topLevelArtifactProvider.getOutputGroup(outputGroup); + if (results != null) { + allArtifacts.addTransitive(results); + } + } + } + + if (options.compileOnly()) { + FilesToCompileProvider provider = target.getProvider(FilesToCompileProvider.class); + if (provider != null) { + allArtifacts.addAll(provider.getFilesToCompile()); + } + } else if (options.compilationPrerequisitesOnly()) { + CompilationPrerequisitesProvider provider = + target.getProvider(CompilationPrerequisitesProvider.class); + if (provider != null) { + allArtifacts.addTransitive(provider.getCompilationPrerequisites()); + } + } else { + FilesToRunProvider filesToRunProvider = target.getProvider(FilesToRunProvider.class); + boolean hasRunfilesSupport = false; + if (filesToRunProvider != null) { + allArtifacts.addAll(filesToRunProvider.getFilesToRun()); + hasRunfilesSupport = filesToRunProvider.getRunfilesSupport() != null; + } + + if (!hasRunfilesSupport) { + RunfilesProvider runfilesProvider = + target.getProvider(RunfilesProvider.class); + if (runfilesProvider != null) { + allArtifacts.addTransitive(runfilesProvider.getDefaultRunfiles().getAllArtifacts()); + } + } + + AlwaysBuiltArtifactsProvider forcedArtifacts = target.getProvider( + AlwaysBuiltArtifactsProvider.class); + if (forcedArtifacts != null) { + allArtifacts.addTransitive(forcedArtifacts.getArtifactsToAlwaysBuild()); + } + } + + allArtifacts.addAll(getCommandArtifacts(target, options.buildCommand())); + allArtifacts.addAll(getCoverageArtifacts(target, options)); + return allArtifacts.build(); + } + + private static Iterable<Artifact> getCoverageArtifacts(TransitiveInfoCollection target, + TopLevelArtifactContext topLevelOptions) { + if (!topLevelOptions.compileOnly() && !topLevelOptions.compilationPrerequisitesOnly() + && topLevelOptions.shouldRunTests()) { + // Add baseline code coverage artifacts if we are collecting code coverage. We do that only + // when running tests. + // It might be slightly faster to first check if any configuration has coverage enabled. + if (target.getConfiguration() != null + && target.getConfiguration().isCodeCoverageEnabled()) { + BaselineCoverageArtifactsProvider provider = + target.getProvider(BaselineCoverageArtifactsProvider.class); + if (provider != null) { + return provider.getBaselineCoverageArtifacts(); + } + } + } + return ImmutableList.of(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactProvider.java new file mode 100644 index 0000000..e2a2d57 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactProvider.java
@@ -0,0 +1,61 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * ConfiguredTargets implementing this interface can provide command-specific + * and unconditional extra artifacts to the build. + */ +@Immutable +public final class TopLevelArtifactProvider implements TransitiveInfoProvider { + + private final ImmutableList<String> commandsForExtraArtifacts; + private final ImmutableList<Artifact> artifactsForCommand; + private final ImmutableMap<String, NestedSet<Artifact>> outputGroups; + + public TopLevelArtifactProvider(ImmutableList<String> commandsForExtraArtifacts, + ImmutableList<Artifact> artifactsForCommand) { + this.commandsForExtraArtifacts = commandsForExtraArtifacts; + this.artifactsForCommand = artifactsForCommand; + this.outputGroups = ImmutableMap.<String, NestedSet<Artifact>>of(); + } + + public TopLevelArtifactProvider(String key, NestedSet<Artifact> artifactsToBuild) { + this.commandsForExtraArtifacts = ImmutableList.of(); + this.artifactsForCommand = ImmutableList.of(); + this.outputGroups = ImmutableMap.<String, NestedSet<Artifact>>of(key, artifactsToBuild); + } + + /** Returns the commands (in lowercase) that this provider should provide artifacts for. */ + public ImmutableList<String> getCommandsForExtraArtifacts() { + return commandsForExtraArtifacts; + } + + /** Returns the extra artifacts for the commands. */ + public ImmutableList<Artifact> getArtifactsForCommand() { + return artifactsForCommand; + } + + /** Returns artifacts that are to be built for every command. */ + public NestedSet<Artifact> getOutputGroup(String outputGroupName) { + return outputGroups.get(outputGroupName); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoCollection.java b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoCollection.java new file mode 100644 index 0000000..82396ee --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoCollection.java
@@ -0,0 +1,118 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.collect.UnmodifiableIterator; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkModule; + +import javax.annotation.Nullable; + +/** + * Objects that implement this interface bundle multiple {@link TransitiveInfoProvider} interfaces. + * + * <p>This interface (together with {@link TransitiveInfoProvider} is the cornerstone of the data + * model of the analysis phase. + * + * <p>The computation a configured target does is allowed to depend on the following things: + * <ul> + * <li>The associated Target (which will usually be a Rule) + * <li>Its own configuration (the configured target does not have access to other configurations, + * e.g. the host configuration, though) + * <li>The transitive info providers and labels of its direct dependencies. + * </ul> + * + * <p>And these are the only inputs. Notably, a configured target is not supposed to access + * other configured targets, the transitive info collections of configured targets it does not + * directly depend on, the actions created by anyone else or the contents of any input file. We + * strive to make it impossible for configured targets to do these things. + * + * <p>A configured target is expected to produce the following data during its analysis: + * <ul> + * <li>A number of Artifacts and Actions generating them + * <li>A set of {@link TransitiveInfoProvider}s that it passes on to the targets directly dependent + * on it + * </ul> + * + * <p>The information that can be passed on to dependent targets by way of + * {@link TransitiveInfoProvider} is subject to constraints (which are detailed in the + * documentation of that class). + * + * <p>Configured targets are currently allowed to create artifacts at any exec path. It would be + * better if they could be constrained to a subtree based on the label of the configured target, + * but this is currently not feasible because multiple rules violate this constraint and the + * output format is part of its interface. + * + * <p>In principle, multiple configured targets should not create actions with conflicting + * outputs. There are still a few exceptions to this rule that are slated to be eventually + * removed, we have provisions to handle this case (Action instances that share at least one + * output file are required to be exactly the same), but this does put some pressure on the design + * and we are eventually planning to eliminate this option. + * + * <p>These restrictions together make it possible to: + * <ul> + * <li>Correctly cache the analysis phase; by tightly constraining what a configured target is + * allowed to access and what it is not, we can know when it needs to invalidate a particular + * one and when it can reuse an already existing one. + * <li>Serialize / deserialize individual configured targets at will, making it possible for + * example to swap out part of the analysis state if there is memory pressure or to move them in + * persistent storage so that the state can be reconstructed at a different time or in a + * different process. The stretch goal is to eventually facilitate cross-uses caching of this + * information. + * </ul> + * + * <p>Implementations of build rules should <b>not</b> hold on to references to the + * {@link TransitiveInfoCollection}s representing their direct prerequisites in order to reduce + * their memory footprint (otherwise, the referenced object could refer one of its direct + * dependencies in turn, thereby making the size of the objects reachable from a single instance + * unbounded). + * + * @see TransitiveInfoProvider + */ +@SkylarkModule(name = "target", doc = "A BUILD target.") +public interface TransitiveInfoCollection extends Iterable<TransitiveInfoProvider> { + + /** + * Returns the transitive information provider requested, or null if the provider is not found. + * The provider has to be a TransitiveInfoProvider Java class. + */ + @Nullable <P extends TransitiveInfoProvider> P getProvider(Class<P> provider); + + /** + * Returns the label associated with this prerequisite. + */ + Label getLabel(); + + /** + * <p>Returns the {@link BuildConfiguration} for which this transitive info collection is defined. + * Configuration is defined for all configured targets with exception of {@link + * InputFileConfiguredTarget} and {@link PackageGroupConfiguredTarget} for which it is always + * <b>null</b>.</p> + */ + @Nullable BuildConfiguration getConfiguration(); + + /** + * Returns the transitive information requested or null, if the information is not found. + * The transitive information has to have been added using the Skylark framework. + */ + @Nullable Object get(String providerKey); + + /** + * Returns an unmodifiable iterator over the transitive info providers in the collections. + */ + @Override + UnmodifiableIterator<TransitiveInfoProvider> iterator(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoProvider.java new file mode 100644 index 0000000..37ec191 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoProvider.java
@@ -0,0 +1,60 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +/** + * This marker interface must be extended by every interface that represents + * rolled-up data about the transitive closure of a configured target. + * + * TransitiveInfoProviders need to be serializable, and for that reason they must conform to + * the following restrictions: + * + * <ul> + * <li>The provider interface must directly extend {@code TransitiveInfoProvider}. + * <li>Every method must return immutable data.</li> + * <li>Every method must return the same object if called multiple times with the same + * arguments.</li> + * <li>Overloading a method name multiple times is forbidden.</li> + * <li>The return type of a method must satisfy one of the following conditions: + * <ul> + * <li>It must be from the set of {String, Integer, int, Boolean, bool, Label, PathFragment, + * Artifact}, OR</li> + * <li>it must be an ImmutableList/List/Collection/Iterable of T, where T is either + * one of the types above with a default serializer or T implements ValueSerializer), OR</li> + * <li>it must be serializable (TBD)</li> + * </ul> + * <li>If the method takes arguments, it must declare a custom serializer (TBD).</li> + * </ul> + * + * <p>Some typical uses of this interface are: + * <ul> + * <li>The set of Python source files in the transitive closure of this rule + * <li>The set of declared C++ header files in the transitive closure + * <li>The files that need to be built when the target is mentioned on the command line + * </ul> + * + * <p>Note that if implemented naively, this would result in the memory requirements + * being O(n^2): in a long dependency chain, if every target adds one single artifact, storing the + * transitive closures of every rule would take 1+2+3+...+n-1+n = O(n^2) memory. + * + * <p>In order to avoid this, we introduce the concept of nested sets. A nested set is an immutable + * data structure that can contain direct members and other nested sets (recursively). Nested sets + * are iterable and can be flattened into ordered sets, where the order depends on which + * implementation of NestedSet you pick. + * + * @see TransitiveInfoCollection + */ +public interface TransitiveInfoProvider { +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/Util.java b/src/main/java/com/google/devtools/build/lib/analysis/Util.java new file mode 100644 index 0000000..ee10bf0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/Util.java
@@ -0,0 +1,64 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Utility methods for use by ConfiguredTarget implementations. + */ +public abstract class Util { + + private Util() {} + + //---------- Label and Target related methods + + /** + * Returns the workspace-relative path of the specified target (file or rule). + * + * <p>For example, "//foo/bar:wiz" and "//foo:bar/wiz" both result in "foo/bar/wiz". + */ + public static PathFragment getWorkspaceRelativePath(Target target) { + return getWorkspaceRelativePath(target.getLabel()); + } + + /** + * Returns the workspace-relative path of the specified target (file or rule). + * + * <p>For example, "//foo/bar:wiz" and "//foo:bar/wiz" both result in "foo/bar/wiz". + */ + public static PathFragment getWorkspaceRelativePath(Label label) { + return label.getPackageFragment().getRelative(label.getName()); + } + + /** + * Returns the workspace-relative path of the specified target (file or rule), + * prepending a prefix and appending a suffix. + * + * <p>For example, "//foo/bar:wiz" and "//foo:bar/wiz" both result in "foo/bar/wiz". + */ + public static PathFragment getWorkspaceRelativePath(Target target, String prefix, String suffix) { + return target.getLabel().getPackageFragment().getRelative(prefix + target.getName() + suffix); + } + + /** + * Checks if a PathFragment contains a '-'. + */ + public static boolean containsHyphen(PathFragment path) { + return path.getPathString().indexOf('-') >= 0; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ViewCreationFailedException.java b/src/main/java/com/google/devtools/build/lib/analysis/ViewCreationFailedException.java new file mode 100644 index 0000000..ae7dfc5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/ViewCreationFailedException.java
@@ -0,0 +1,31 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis; + +/** + * An exception indicating that there was a problem during the view + * construction (loading and analysis phases) for one or more targets, that the + * configured target graph could not be successfully constructed, and that + * a build cannot be started. + */ +public class ViewCreationFailedException extends Exception { + + public ViewCreationFailedException(String message) { + super(message); + } + + public ViewCreationFailedException(String message, Throwable cause) { + super(message + ": " + cause.getMessage(), cause); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/VisibilityProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/VisibilityProvider.java new file mode 100644 index 0000000..a438145 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/VisibilityProvider.java
@@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.packages.PackageSpecification; + +/** + * Provider class for configured targets that have a visibility. + */ +public interface VisibilityProvider extends TransitiveInfoProvider { + + /** + * Returns the visibility specification. + */ + NestedSet<PackageSpecification> getVisibility(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/VisibilityProviderImpl.java b/src/main/java/com/google/devtools/build/lib/analysis/VisibilityProviderImpl.java new file mode 100644 index 0000000..01dd06a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/VisibilityProviderImpl.java
@@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.packages.PackageSpecification; + +/** + * Visibility provider implementation. + */ +@Immutable +public final class VisibilityProviderImpl implements VisibilityProvider { + private final NestedSet<PackageSpecification> visibility; + + public VisibilityProviderImpl(NestedSet<PackageSpecification> visibility) { + this.visibility = visibility; + } + + @Override + public NestedSet<PackageSpecification> getVisibility() { + return visibility; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/WorkspaceStatusAction.java b/src/main/java/com/google/devtools/build/lib/analysis/WorkspaceStatusAction.java new file mode 100644 index 0000000..5b3bd27 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/WorkspaceStatusAction.java
@@ -0,0 +1,185 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis; + +import com.google.common.base.Splitter; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * An action writing the workspace status files. + * + * <p>These files represent information about the environment the build was run in. They are used + * by language-specific build info factories to make the data in them available for individual + * languages (e.g. by turning them into .h files for C++) + * + * <p>The format of these files a list of key-value pairs, one for each line. The key and the value + * are separated by a space. + * + * <p>There are two of these files: volatile and stable. Changes in the volatile file do not + * cause rebuilds if no other file is changed. This is useful for frequently-changing information + * that does not significantly affect the build, e.g. the current time. + */ +public abstract class WorkspaceStatusAction extends AbstractAction { + + /** + * The type of a workspace status action key. + */ + public enum KeyType { + INTEGER, + STRING, + VERBATIM, + } + + /** + * Language for keys that should be present in the build info for every language. + */ + // TODO(bazel-team): Once this is released, migrate the only place in the depot to use + // the BUILD_USERNAME, BUILD_HOSTNAME and BUILD_DIRECTORY keys instead of BUILD_INFO. Then + // language-specific build info keys can be removed. + public static final String ALL_LANGUAGES = "*"; + + /** + * Action context required by the actions that write language-specific workspace status artifacts. + */ + public static interface Context extends ActionContext { + ImmutableMap<String, Key> getStableKeys(); + ImmutableMap<String, Key> getVolatileKeys(); + } + + /** + * A key in the workspace status info file. + */ + public static class Key { + private final KeyType type; + + /** + * Should be set to ALL_LANGUAGES if the key should be present in the build info of every + * language. + */ + private final String language; + private final String defaultValue; + private final String redactedValue; + + private Key(KeyType type, String language, String defaultValue, String redactedValue) { + this.type = type; + this.language = language; + this.defaultValue = defaultValue; + this.redactedValue = redactedValue; + } + + public KeyType getType() { + return type; + } + + public boolean isInLanguage(String language) { + return this.language.equals(ALL_LANGUAGES) || this.language.equals(language); + } + + public String getDefaultValue() { + return defaultValue; + } + + public String getRedactedValue() { + return redactedValue; + } + + public static Key forLanguage( + String language, KeyType type, String defaultValue, String redactedValue) { + return new Key(type, language, defaultValue, redactedValue); + } + + public static Key of(KeyType type, String defaultValue, String redactedValue) { + return new Key(type, ALL_LANGUAGES, defaultValue, redactedValue); + } + } + + /** + * Parses the output of the workspace status action. + * + * <p>The output is a text file with each line representing a workspace status info key. + * The key is the part of the line before the first space and should consist of the characters + * [A-Z_] (although this is not checked). Everything after the first space is the value. + */ + public static Map<String, String> parseValues(Path file) throws IOException { + HashMap<String, String> result = new HashMap<>(); + Splitter lineSplitter = Splitter.on(" ").limit(2); + for (String line : Splitter.on("\n").split( + new String(FileSystemUtils.readContentAsLatin1(file)))) { + List<String> items = ImmutableList.copyOf(lineSplitter.split(line)); + if (items.size() != 2) { + continue; + } + + result.put(items.get(0), items.get(1)); + } + + return ImmutableMap.copyOf(result); + } + + /** + * Factory for {@link WorkspaceStatusAction}. + */ + public interface Factory { + /** + * Creates the workspace status action. + * + * <p>If the objects returned for two builds are equals, the workspace status action can be + * be reused between them. Note that this only applies to the action object itself (the action + * will be unconditionally re-executed on every build) + */ + WorkspaceStatusAction createWorkspaceStatusAction( + ArtifactFactory artifactFactory, ArtifactOwner artifactOwner, Supplier<UUID> buildId); + + /** + * Creates a dummy workspace status map. Used in cases where the build failed, so that part of + * the workspace status is nevertheless available. + */ + Map<String, String> createDummyWorkspaceStatus(); + } + + protected WorkspaceStatusAction(ActionOwner owner, + Iterable<Artifact> inputs, + Iterable<Artifact> outputs) { + super(owner, inputs, outputs); + } + + /** + * The volatile status artifact containing items that may change even if nothing changed + * between the two builds, e.g. current time. + */ + public abstract Artifact getVolatileStatus(); + + /** + * The stable status artifact containing items that change only if information relevant to the + * build changes, e.g. the name of the user running the build or the hostname. + */ + public abstract Artifact getStableStatus(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/AbstractFileWriteAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/AbstractFileWriteAction.java new file mode 100644 index 0000000..187fc48 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/AbstractFileWriteAction.java
@@ -0,0 +1,142 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.actions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.syntax.Label; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Abstract Action to write to a file. + */ +public abstract class AbstractFileWriteAction extends AbstractAction { + + protected final boolean makeExecutable; + + /** + * Creates a new AbstractFileWriteAction instance. + * + * @param owner the action owner. + * @param inputs the Artifacts that this Action depends on + * @param output the Artifact that will be created by executing this Action. + * @param makeExecutable iff true will change the output file to be + * executable. + */ + public AbstractFileWriteAction(ActionOwner owner, + Iterable<Artifact> inputs, Artifact output, boolean makeExecutable) { + // There is only one output, and it is primary. + super(owner, inputs, ImmutableList.of(output)); + this.makeExecutable = makeExecutable; + } + + public boolean makeExecutable() { + return makeExecutable; + } + + @Override + public final void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + try { + getStrategy(actionExecutionContext.getExecutor()).exec(actionExecutionContext.getExecutor(), + this, actionExecutionContext.getFileOutErr(), actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException( + "Writing file for rule '" + Label.print(getOwner().getLabel()) + "'", + actionExecutionContext.getExecutor().getVerboseFailures(), this); + } + afterWrite(actionExecutionContext.getExecutor()); + } + + /** + * Produce a DeterministicWriter that can write the file to an OutputStream deterministically. + * + * @param eventHandler destination for warning messages. (Note that errors should + * still be indicated by throwing an exception; reporter.error() will + * not cause action execution to fail.) + * @param executor the Executor. + * @throws IOException if the content cannot be written to the output stream + */ + public abstract DeterministicWriter newDeterministicWriter(EventHandler eventHandler, + Executor executor) throws IOException, InterruptedException, ExecException; + + /** + * This hook is called after the File has been successfully written to disk. + * + * @param executor the Executor. + */ + protected void afterWrite(Executor executor) { + } + + // We're mainly doing I/O, so estimate very low CPU usage, e.g. 1%. Just a guess. + private static final ResourceSet DEFAULT_FILEWRITE_LOCAL_ACTION_RESOURCE_SET = + new ResourceSet(/*memoryMb=*/0.0, /*cpuUsage=*/0.01, /*ioUsage=*/0.2); + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return executor.getContext(FileWriteActionContext.class).estimateResourceConsumption(this); + } + + public ResourceSet estimateResourceConsumptionLocal() { + return DEFAULT_FILEWRITE_LOCAL_ACTION_RESOURCE_SET; + } + + @Override + public String getMnemonic() { + return "FileWrite"; + } + + @Override + protected String getRawProgressMessage() { + return "Writing " + (makeExecutable ? "script " : "file ") + + Iterables.getOnlyElement(getOutputs()).prettyPrint(); + } + + /** + * Whether the file write can be generated remotely. If the file is consumed in Blaze + * unconditionally, it doesn't make sense to run remotely. + */ + public boolean isRemotable() { + return true; + } + + @Override + public final String describeStrategy(Executor executor) { + return executor.getContext(FileWriteActionContext.class).strategyLocality(this); + } + + private FileWriteActionContext getStrategy(Executor executor) { + return executor.getContext(FileWriteActionContext.class); + } + + /** + * A deterministic writer writes bytes to an output stream. The same byte stream is written + * on every invocation of writeOutputFile(). + */ + public interface DeterministicWriter { + public void writeOutputFile(OutputStream out) throws IOException; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/ActionConstructionContext.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/ActionConstructionContext.java new file mode 100644 index 0000000..b7461e5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/ActionConstructionContext.java
@@ -0,0 +1,37 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.actions; + +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Rule; + +/** + * A temporary interface to allow migration from RuleConfiguredTarget to RuleContext. It bundles + * the items commonly needed to construct action instances. + */ +public interface ActionConstructionContext { + /** The rule for which the actions are constructed. */ + Rule getRule(); + + /** Returns the action owner that should be used for actions. */ + ActionOwner getActionOwner(); + + /** Returns the {@link BuildConfiguration} for which the given rule is analyzed. */ + BuildConfiguration getConfiguration(); + + /** The current analysis environment. */ + AnalysisEnvironment getAnalysisEnvironment(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/BinaryFileWriteAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/BinaryFileWriteAction.java new file mode 100644 index 0000000..b9a2ea5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/BinaryFileWriteAction.java
@@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.io.ByteSource; +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Action to write a binary file. + */ +public final class BinaryFileWriteAction extends AbstractFileWriteAction { + + private static final String GUID = "eeee07fe-4b40-11e4-82d6-eba0b4f713e2"; + + private final ByteSource source; + + /** + * Creates a new BinaryFileWriteAction instance without inputs. + * + * @param owner the action owner. + * @param output the Artifact that will be created by executing this Action. + * @param source a source of bytes that will be written to the file. + * @param makeExecutable iff true will change the output file to be executable. + */ + public BinaryFileWriteAction( + ActionOwner owner, Artifact output, ByteSource source, boolean makeExecutable) { + super(owner, /*inputs=*/Artifact.NO_ARTIFACTS, output, makeExecutable); + this.source = Preconditions.checkNotNull(source); + } + + @VisibleForTesting + public ByteSource getSource() { + return source; + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) { + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + try (InputStream in = source.openStream()) { + ByteStreams.copy(in, out); + } + out.flush(); + } + }; + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addString(String.valueOf(makeExecutable)); + + try (InputStream in = source.openStream()) { + byte[] buffer = new byte[512]; + int amountRead; + while ((amountRead = in.read(buffer)) != -1) { + f.addBytes(buffer, 0, amountRead); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + + return f.hexDigestAndReset(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/CommandLine.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/CommandLine.java new file mode 100644 index 0000000..ddadc25 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/CommandLine.java
@@ -0,0 +1,123 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.actions; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.collect.CollectionUtils; + +/** + * A representation of a command line to be executed by a SpawnAction. + */ +public abstract class CommandLine { + /** + * Returns the command line. + */ + public abstract Iterable<String> arguments(); + + /** + * Returns whether the command line represents a shell command with the given shell executable. + * This is used to give better error messages. + * + * <p>By default, this method returns false. + */ + public boolean isShellCommand() { + return false; + } + + /** + * A default implementation of a command line backed by a copy of the given list of arguments. + */ + static CommandLine ofInternal(Iterable<String> arguments, final boolean isShellCommand) { + final Iterable<String> immutableArguments = CollectionUtils.makeImmutable(arguments); + return new CommandLine() { + @Override + public Iterable<String> arguments() { + return immutableArguments; + } + + @Override + public boolean isShellCommand() { + return isShellCommand; + } + }; + } + + /** + * Returns a {@link CommandLine} backed by a copy of the given list of arguments. + */ + public static CommandLine of(Iterable<String> arguments, final boolean isShellCommand) { + final Iterable<String> immutableArguments = CollectionUtils.makeImmutable(arguments); + return new CommandLine() { + @Override + public Iterable<String> arguments() { + return immutableArguments; + } + + @Override + public boolean isShellCommand() { + return isShellCommand; + } + }; + } + + /** + * Returns a {@link CommandLine} that is constructed by prepending the {@code executableArgs} to + * {@code commandLine}. + */ + static CommandLine ofMixed(final ImmutableList<String> executableArgs, + final CommandLine commandLine, final boolean isShellCommand) { + Preconditions.checkState(!executableArgs.isEmpty()); + return new CommandLine() { + @Override + public Iterable<String> arguments() { + return Iterables.concat(executableArgs, commandLine.arguments()); + } + + @Override + public boolean isShellCommand() { + return isShellCommand; + } + }; + } + + /** + * Returns a {@link CommandLine} with {@link CharSequence} arguments. This can be useful to create + * memory efficient command lines with {@link com.google.devtools.build.lib.util.LazyString}s. + */ + public static CommandLine ofCharSequences(final ImmutableList<CharSequence> arguments) { + return new CommandLine() { + @Override + public Iterable<String> arguments() { + ImmutableList.Builder<String> builder = ImmutableList.builder(); + for (CharSequence arg : arguments) { + builder.add(arg.toString()); + } + return builder.build(); + } + }; + } + + /** + * This helps when debugging Blaze code that uses {@link CommandLine}s, as you can see their + * content directly in the variable inspector. + */ + @Override + public String toString() { + return Joiner.on(' ').join(arguments()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/CustomCommandLine.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/CustomCommandLine.java new file mode 100644 index 0000000..d358f0b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/CustomCommandLine.java
@@ -0,0 +1,358 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.actions; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.List; + +/** + * A customizable, serializable class for building memory efficient command lines. + */ +@Immutable +public final class CustomCommandLine extends CommandLine { + + private abstract static class ArgvFragment { + abstract void eval(ImmutableList.Builder<String> builder); + } + + // It's better to avoid anonymous classes if we want to serialize command lines + + private static final class ObjectArg extends ArgvFragment { + private final Object arg; + + private ObjectArg(Object arg) { + this.arg = arg; + } + + @Override + void eval(ImmutableList.Builder<String> builder) { + builder.add(arg.toString()); + } + } + + private static final class JoinExecPathsArg extends ArgvFragment { + + private final String delimiter; + private final Iterable<Artifact> artifacts; + + private JoinExecPathsArg(String delimiter, Iterable<Artifact> artifacts) { + this.delimiter = delimiter; + this.artifacts = CollectionUtils.makeImmutable(artifacts); + } + + @Override + void eval(ImmutableList.Builder<String> builder) { + builder.add(Artifact.joinExecPaths(delimiter, artifacts)); + } + } + + private static final class PathWithTemplateArg extends ArgvFragment { + + private final String template; + private final PathFragment[] paths; + + private PathWithTemplateArg(String template, PathFragment... paths) { + this.template = template; + this.paths = paths; + } + + @Override + void eval(ImmutableList.Builder<String> builder) { + // PathFragment.toString() uses getPathString() + builder.add(String.format(template, (Object[]) paths)); + } + } + + // TODO(bazel-team): CustomArgv and CustomMultiArgv is going to be difficult to expose + // in Skylark. Maybe we can get rid of them by refactoring JavaCompileAction. It also + // raises immutability / serialization issues. + /** + * Custom Java code producing a String argument. Usage of this class is discouraged. + */ + public abstract static class CustomArgv extends ArgvFragment { + + @Override + void eval(ImmutableList.Builder<String> builder) { + builder.add(argv()); + } + + public abstract String argv(); + } + + /** + * Custom Java code producing a List of String arguments. Usage of this class is discouraged. + */ + public abstract static class CustomMultiArgv extends ArgvFragment { + + @Override + void eval(ImmutableList.Builder<String> builder) { + builder.addAll(argv()); + } + + public abstract Iterable<String> argv(); + } + + private static final class JoinPathsArg extends ArgvFragment { + + private final String delimiter; + private final Iterable<PathFragment> paths; + + private JoinPathsArg(String delimiter, Iterable<PathFragment> paths) { + this.delimiter = delimiter; + this.paths = CollectionUtils.makeImmutable(paths); + } + + @Override + void eval(ImmutableList.Builder<String> builder) { + builder.add(Joiner.on(delimiter).join(paths)); + } + } + + /** + * Arguments that intersperse strings between the items in a sequence. There are two forms of + * interspersing, and either may be used by this implementation: + * <ul> + * <li>before each - a string is added before each item in a sequence. e.g. + * {@code -f foo -f bar -f baz} + * <li>format each - a format string is used to format each item in a sequence. e.g. + * {@code -I/foo -I/bar -I/baz} for the format {@code "-I%s"} + * </ul> + * + * <p>This class could be used both with both the "before" and "format" features at the same + * time, but this is probably more confusion than it is worth. If you need this functionality, + * consider using "before" only but storing the strings pre-formated in a {@link NestedSet}. + */ + private static final class InterspersingArgs extends ArgvFragment { + private final Iterable<?> sequence; + private final String beforeEach; + private final String formatEach; + + /** + * Do not call from outside this class because this does not guarantee that {@code sequence} is + * immutable. + */ + private InterspersingArgs(Iterable<?> sequence, String beforeEach, String formatEach) { + this.sequence = sequence; + this.beforeEach = beforeEach; + this.formatEach = formatEach; + } + + static InterspersingArgs fromStrings( + Iterable<?> sequence, String beforeEach, String formatEach) { + return new InterspersingArgs( + CollectionUtils.makeImmutable(sequence), beforeEach, formatEach); + } + + static InterspersingArgs fromExecPaths( + Iterable<Artifact> sequence, String beforeEach, String formatEach) { + return new InterspersingArgs( + Artifact.toExecPaths(CollectionUtils.makeImmutable(sequence)), beforeEach, formatEach); + } + + @Override + void eval(ImmutableList.Builder<String> builder) { + for (Object item : sequence) { + if (item == null) { + continue; + } + + if (beforeEach != null) { + builder.add(beforeEach); + } + String arg = item.toString(); + if (formatEach != null) { + arg = String.format(formatEach, arg); + } + builder.add(arg); + } + } + } + + /** + * A Builder class for CustomCommandLine with the appropriate methods. + * + * <p>{@link Iterable} instances passed to {@code add*} methods will be stored internally as + * collections that are known to be immutable copies. This means that any {@link Iterable} that is + * not a {@link NestedSet} or {@link ImmutableList} may be copied. + * + * <p>{@code addFormatEach*} methods take an {@link Iterable} but use these as arguments to + * {@link String#format(String, Object...)} with a certain constant format string. For instance, + * if {@code format} is {@code "-I%s"}, then the final arguments may be + * {@code -Ifoo -Ibar -Ibaz} + * + * <p>{@code addBeforeEach*} methods take an {@link Iterable} but insert a certain {@link String} + * once before each element in the string, meaning the total number of elements added is twice the + * length of the {@link Iterable}. For instance: {@code -f foo -f bar -f baz} + */ + public static final class Builder { + + private final List<ArgvFragment> arguments = new ArrayList<>(); + + public Builder add(CharSequence arg) { + if (arg != null) { + arguments.add(new ObjectArg(arg)); + } + return this; + } + + public Builder add(Label arg) { + if (arg != null) { + arguments.add(new ObjectArg(arg)); + } + return this; + } + + public Builder add(String arg, Iterable<String> args) { + if (arg != null && args != null) { + arguments.add(new ObjectArg(arg)); + arguments.add(InterspersingArgs.fromStrings(args, /*beforeEach=*/null, "%s")); + } + return this; + } + + public Builder add(Iterable<String> args) { + if (args != null) { + arguments.add(InterspersingArgs.fromStrings(args, /*beforeEach=*/null, "%s")); + } + return this; + } + + public Builder addExecPath(String arg, Artifact artifact) { + if (arg != null && artifact != null) { + arguments.add(new ObjectArg(arg)); + arguments.add(new ObjectArg(artifact.getExecPath())); + } + return this; + } + + public Builder addExecPaths(String arg, Iterable<Artifact> artifacts) { + if (arg != null && artifacts != null) { + arguments.add(new ObjectArg(arg)); + arguments.add(InterspersingArgs.fromExecPaths(artifacts, /*beforeEach=*/null, "%s")); + } + return this; + } + + public Builder addExecPaths(Iterable<Artifact> artifacts) { + if (artifacts != null) { + arguments.add(InterspersingArgs.fromExecPaths(artifacts, /*beforeEach=*/null, "%s")); + } + return this; + } + + public Builder addJoinExecPaths(String arg, String delimiter, Iterable<Artifact> artifacts) { + if (arg != null && artifacts != null) { + arguments.add(new ObjectArg(arg)); + arguments.add(new JoinExecPathsArg(delimiter, artifacts)); + } + return this; + } + + public Builder addPath(PathFragment path) { + if (path != null) { + arguments.add(new ObjectArg(path)); + } + return this; + } + + public Builder addPaths(String template, PathFragment... path) { + if (template != null && path != null) { + arguments.add(new PathWithTemplateArg(template, path)); + } + return this; + } + + public Builder addJoinPaths(String delimiter, Iterable<PathFragment> paths) { + if (delimiter != null && paths != null) { + arguments.add(new JoinPathsArg(delimiter, paths)); + } + return this; + } + + public Builder addBeforeEachPath(String repeated, Iterable<PathFragment> paths) { + if (repeated != null && paths != null) { + arguments.add(InterspersingArgs.fromStrings(paths, repeated, "%s")); + } + return this; + } + + public Builder addBeforeEach(String repeated, Iterable<String> strings) { + if (repeated != null && strings != null) { + arguments.add(InterspersingArgs.fromStrings(strings, repeated, "%s")); + } + return this; + } + + public Builder addBeforeEachExecPath(String repeated, Iterable<Artifact> artifacts) { + if (repeated != null && artifacts != null) { + arguments.add(InterspersingArgs.fromExecPaths(artifacts, repeated, "%s")); + } + return this; + } + + public Builder addFormatEach(String format, Iterable<String> strings) { + if (format != null && strings != null) { + arguments.add(InterspersingArgs.fromStrings(strings, /*beforeEach=*/null, format)); + } + return this; + } + + public Builder add(CustomArgv arg) { + if (arg != null) { + arguments.add(arg); + } + return this; + } + + public Builder add(CustomMultiArgv arg) { + if (arg != null) { + arguments.add(arg); + } + return this; + } + + public CustomCommandLine build() { + return new CustomCommandLine(arguments); + } + } + + public static Builder builder() { + return new Builder(); + } + + private final ImmutableList<ArgvFragment> arguments; + + private CustomCommandLine(List<ArgvFragment> arguments) { + this.arguments = ImmutableList.copyOf(arguments); + } + + @Override + public Iterable<String> arguments() { + ImmutableList.Builder<String> builder = ImmutableList.builder(); + for (ArgvFragment arg : arguments) { + arg.eval(builder); + } + return builder.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/ExecutableSymlinkAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/ExecutableSymlinkAction.java new file mode 100644 index 0000000..376d9b8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/ExecutableSymlinkAction.java
@@ -0,0 +1,66 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.actions; + +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; + +/** + * Action to create an executable symbolic link. It includes additional + * validation that symlink target is indeed an executable file. + */ +public final class ExecutableSymlinkAction extends SymlinkAction { + + public ExecutableSymlinkAction(ActionOwner owner, Artifact input, Artifact output) { + super(owner, input, output, "Symlinking " + owner.getLabel()); + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException { + Path inputPath = actionExecutionContext.getExecutor().getExecRoot().getRelative( + getInputPath()); + try { + // Validate that input path is a file with the executable bit is set. + if (!inputPath.isFile()) { + throw new ActionExecutionException( + "'" + Iterables.getOnlyElement(getInputs()).prettyPrint() + "' is not a file", this, + false); + } + if (!inputPath.isExecutable()) { + throw new ActionExecutionException("failed to create symbolic link '" + + Iterables.getOnlyElement(getOutputs()).prettyPrint() + + "': file '" + Iterables.getOnlyElement(getInputs()).prettyPrint() + + "' is not executable", this, false); + } + } catch (IOException e) { + throw new ActionExecutionException("failed to create symbolic link '" + + Iterables.getOnlyElement(getOutputs()).prettyPrint() + + "' to the '" + Iterables.getOnlyElement(getInputs()).prettyPrint() + + "' due to I/O error: " + e.getMessage(), e, this, false); + } + + super.execute(actionExecutionContext); + } + + @Override + public String getMnemonic() { return "ExecutableSymlink"; } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java new file mode 100644 index 0000000..1128617 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java
@@ -0,0 +1,145 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.actions; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collection; + +/** + * Action to write to a file. + * <p>TODO(bazel-team): Choose a better name to distinguish this class from + * {@link BinaryFileWriteAction}. + */ +public class FileWriteAction extends AbstractFileWriteAction { + + private static final String GUID = "332877c7-ca9f-4731-b387-54f620408522"; + + /** + * We keep it as a CharSequence for memory-efficiency reasons. The toString() + * method of the object represents the content of the file. + * + * <p>For example, this allows us to keep a {@code List<Artifact>} wrapped + * in a {@code LazyString} instead of the string representation of the concatenation. + * This saves memory because the Artifacts are shared objects but the + * resulting String is not. + */ + private final CharSequence fileContents; + + /** + * Creates a new FileWriteAction instance without inputs using UTF8 encoding. + * + * @param owner the action owner. + * @param output the Artifact that will be created by executing this Action. + * @param fileContents the contents to be written to the file. + * @param makeExecutable iff true will change the output file to be + * executable. + */ + public FileWriteAction(ActionOwner owner, Artifact output, CharSequence fileContents, + boolean makeExecutable) { + this(owner, Artifact.NO_ARTIFACTS, output, fileContents, makeExecutable); + } + + /** + * Creates a new FileWriteAction instance using UTF8 encoding. + * + * @param owner the action owner. + * @param inputs the Artifacts that this Action depends on + * @param output the Artifact that will be created by executing this Action. + * @param fileContents the contents to be written to the file. + * @param makeExecutable iff true will change the output file to be + * executable. + */ + public FileWriteAction(ActionOwner owner, Collection<Artifact> inputs, + Artifact output, CharSequence fileContents, boolean makeExecutable) { + super(owner, inputs, output, makeExecutable); + this.fileContents = fileContents; + } + + /** + * Creates a new FileWriteAction instance using UTF8 encoding. + * + * @param owner the action owner. + * @param inputs the Artifacts that this Action depends on + * @param output the Artifact that will be created by executing this Action. + * @param makeExecutable iff true will change the output file to be + * executable. + */ + protected FileWriteAction(ActionOwner owner, Collection<Artifact> inputs, + Artifact output, boolean makeExecutable) { + this(owner, inputs, output, "", makeExecutable); + } + + public String getFileContents() { + return fileContents.toString(); + } + + /** + * Create a DeterministicWriter for the content of the output file as provided by + * {@link #getFileContents()}. + */ + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, + Executor executor) { + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + out.write(getFileContents().getBytes(UTF_8)); + } + }; + } + + /** + * Computes the Action key for this action by computing the fingerprint for + * the file contents. + */ + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addString(String.valueOf(makeExecutable)); + f.addString(getFileContents()); + return f.hexDigestAndReset(); + } + + /** + * Creates a FileWriteAction to write contents to the resulting artifact + * fileName in the genfiles root underneath the package path. + * + * @param ruleContext the ruleContext that will own the action of creating this file. + * @param fileName name of the file to create. + * @param contents data to write to file. + * @param executable flags that file should be marked executable. + * @return Artifact describing the file to create. + */ + public static Artifact createFile(RuleContext ruleContext, + String fileName, CharSequence contents, boolean executable) { + Artifact scriptFileArtifact = ruleContext.getAnalysisEnvironment().getDerivedArtifact( + ruleContext.getTarget().getLabel().getPackageFragment().getRelative(fileName), + ruleContext.getConfiguration().getGenfilesDirectory()); + ruleContext.registerAction(new FileWriteAction( + ruleContext.getActionOwner(), scriptFileArtifact, contents, executable)); + return scriptFileArtifact; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteActionContext.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteActionContext.java new file mode 100644 index 0000000..7e10331 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteActionContext.java
@@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.actions; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.util.io.FileOutErr; + +/** + * The action context for {@link AbstractFileWriteAction} instances (technically instances of + * subclasses). + */ +public interface FileWriteActionContext extends ActionContext { + + /** + * Performs all the setup and then calls back into the action to write the data. + */ + void exec(Executor executor, AbstractFileWriteAction action, FileOutErr outErr, + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException; + + /** + * Returns the estimated resource consumption of the action. + */ + ResourceSet estimateResourceConsumption(AbstractFileWriteAction action); + + /** + * Returns where the action actually runs. + */ + String strategyLocality(AbstractFileWriteAction action); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileHelper.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileHelper.java new file mode 100644 index 0000000..e7869e4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileHelper.java
@@ -0,0 +1,158 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.actions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ParameterFile; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.List; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * A command-line implementation that wraps another command line and puts the arguments in a + * parameter file if necessary + * + * <p>The Linux kernel has a limit for the command line length, and that can be easily reached + * if, for example, a command is listing all its inputs on the command line. + */ +@Immutable +public final class ParamFileHelper { + + /** + * Returns a params file artifact or null for a given command description. + * + * <p>Returns null if parameter files are not to be used according to paramFileInfo, or if the + * command line is short enough that a parameter file is not needed. + * + * <p>Make sure to add the returned artifact (if not null) as an input of the corresponding + * action. + * + * @param executableArgs leading arguments that should never be wrapped in a parameter file + * @param arguments arguments to the command (in addition to executableArgs), OR + * @param commandLine a {@link CommandLine} that provides the arguments (in addition to + * executableArgs) + * @param paramFileInfo parameter file information + * @param configuration the configuration + * @param analysisEnvironment the analysis environment + * @param outputs outputs of the action (used to construct a filename for the params file) + */ + public static Artifact getParamsFile( + List<String> executableArgs, + @Nullable Iterable<String> arguments, + @Nullable CommandLine commandLine, + @Nullable ParamFileInfo paramFileInfo, + BuildConfiguration configuration, + AnalysisEnvironment analysisEnvironment, + Iterable<Artifact> outputs) { + if (paramFileInfo == null || + getParamFileSize(executableArgs, arguments, commandLine) + < configuration.getMinParamFileSize()) { + return null; + } + + PathFragment paramFilePath = ParameterFile.derivePath( + Iterables.getFirst(outputs, null).getRootRelativePath()); + return analysisEnvironment.getDerivedArtifact(paramFilePath, configuration.getBinDirectory()); + } + + /** + * Creates a command line using an external params file. + * + * <p>Call this with the result of {@link #getParamsFile} if it is not null. + * + * @param executableArgs leading arguments that should never be wrapped in a parameter file + * @param arguments arguments to the command (in addition to executableArgs), OR + * @param commandLine a {@link CommandLine} that provides the arguments (in addition to + * executableArgs) + * @param isShellCommand true if this is a shell command + * @param owner owner of the action + * @param paramFileInfo parameter file information + */ + public static CommandLine createWithParamsFile( + List<String> executableArgs, + @Nullable Iterable<String> arguments, + @Nullable CommandLine commandLine, + boolean isShellCommand, + ActionOwner owner, + List<Action> requiredActions, + ParamFileInfo paramFileInfo, + Artifact parameterFile) { + Preconditions.checkNotNull(parameterFile); + if (commandLine != null && arguments != null && !Iterables.isEmpty(arguments)) { + throw new IllegalStateException("must provide either commandLine or arguments: " + arguments); + } + + CommandLine paramFileContents = + (commandLine != null) ? commandLine : CommandLine.ofInternal(arguments, false); + requiredActions.add(new ParameterFileWriteAction(owner, parameterFile, paramFileContents, + paramFileInfo.getFileType(), paramFileInfo.getCharset())); + + String pathWithFlag = paramFileInfo.getFlag() + parameterFile.getExecPathString(); + Iterable<String> commandArgv = Iterables.concat(executableArgs, ImmutableList.of(pathWithFlag)); + return CommandLine.ofInternal(commandArgv, isShellCommand); + } + + /** + * Creates a command line without using a params file. + * + * <p>Call this if {@link #getParamsFile} returns null. + * + * @param executableArgs leading arguments that should never be wrapped in a parameter file + * @param arguments arguments to the command (in addition to executableArgs), OR + * @param commandLine a {@link CommandLine} that provides the arguments (in addition to + * executableArgs) + * @param isShellCommand true if this is a shell command + */ + public static CommandLine createWithoutParamsFile(List<String> executableArgs, + Iterable<String> arguments, CommandLine commandLine, boolean isShellCommand) { + if (commandLine == null) { + Iterable<String> commandArgv = Iterables.concat(executableArgs, arguments); + return CommandLine.ofInternal(commandArgv, isShellCommand); + } + + if (executableArgs.isEmpty()) { + return commandLine; + } + + return CommandLine.ofMixed(ImmutableList.copyOf(executableArgs), commandLine, isShellCommand); + } + + /** + * Estimates the params file size for the given arguments. + */ + private static int getParamFileSize( + List<String> executableArgs, Iterable<String> arguments, CommandLine commandLine) { + Iterable<String> actualArguments = (commandLine != null) ? commandLine.arguments() : arguments; + return getParamFileSize(executableArgs) + getParamFileSize(actualArguments); + } + + private static int getParamFileSize(Iterable<String> args) { + int size = 0; + for (String s : args) { + size += s.length() + 1; + } + return size; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileInfo.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileInfo.java new file mode 100644 index 0000000..ae00181 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileInfo.java
@@ -0,0 +1,79 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.actions; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; + +import java.nio.charset.Charset; +import java.util.Objects; + +import javax.annotation.concurrent.Immutable; + +/** + * An object that encapsulates how a params file should be constructed: what is the filetype, + * what charset to use and what prefix (typically "@") to use. + */ +@Immutable +public final class ParamFileInfo { + private final ParameterFileType fileType; + private final Charset charset; + private final String flag; + + public ParamFileInfo(ParameterFileType fileType, Charset charset, String flag) { + this.fileType = Preconditions.checkNotNull(fileType); + this.charset = Preconditions.checkNotNull(charset); + this.flag = Preconditions.checkNotNull(flag); + } + + /** + * Returns the file type. + */ + public ParameterFileType getFileType() { + return fileType; + } + + /** + * Returns the charset. + */ + public Charset getCharset() { + return charset; + } + + /** + * Returns the prefix for the params filename on the command line (typically "@"). + */ + public String getFlag() { + return flag; + } + + @Override + public int hashCode() { + return Objects.hash(charset, flag, fileType); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof ParamFileInfo)) { + return false; + } + ParamFileInfo other = (ParamFileInfo) obj; + return fileType.equals(other.fileType) && charset.equals(other.charset) + && flag.equals(other.flag); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/ParameterFileWriteAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParameterFileWriteAction.java new file mode 100644 index 0000000..fd10e2b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParameterFileWriteAction.java
@@ -0,0 +1,122 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.ShellEscaper; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.Charset; + +/** + * Action to write a parameter file for a {@link CommandLine}. + */ +public final class ParameterFileWriteAction extends AbstractFileWriteAction { + + private static final String GUID = "45f678d8-e395-401e-8446-e795ccc6361f"; + + private final CommandLine commandLine; + private final ParameterFileType type; + private final Charset charset; + + /** + * Creates a new instance. + * + * @param owner the action owner + * @param output the Artifact that will be created by executing this Action + * @param commandLine the contents to be written to the file + * @param type the type of the file + * @param charset the charset of the file + */ + public ParameterFileWriteAction(ActionOwner owner, Artifact output, CommandLine commandLine, + ParameterFileType type, Charset charset) { + super(owner, ImmutableList.<Artifact>of(), output, false); + this.commandLine = commandLine; + this.type = type; + this.charset = charset; + } + + /** + * Returns the list of options written to the parameter file. Don't use this + * method outside tests - the list is often huge, resulting in significant + * garbage collection overhead. + */ + @VisibleForTesting + public Iterable<String> getContents() { + return commandLine.arguments(); + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) { + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + switch (type) { + case SHELL_QUOTED : + writeContentQuoted(out); + break; + case UNQUOTED : + writeContentUnquoted(out); + break; + default : + throw new AssertionError(); + } + } + }; + } + + /** + * Writes the arguments from the list into the parameter file. + */ + private void writeContentUnquoted(OutputStream outputStream) throws IOException { + OutputStreamWriter out = new OutputStreamWriter(outputStream, charset); + for (String line : commandLine.arguments()) { + out.write(line); + out.write('\n'); + } + out.flush(); + } + + /** + * Writes the arguments from the list into the parameter file with shell + * quoting (if required). + */ + private void writeContentQuoted(OutputStream outputStream) throws IOException { + OutputStreamWriter out = new OutputStreamWriter(outputStream, charset); + for (String line : ShellEscaper.escapeAll(commandLine.arguments())) { + out.write(line); + out.write('\n'); + } + out.flush(); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addString(String.valueOf(makeExecutable)); + f.addStrings(commandLine.arguments()); + return f.hexDigestAndReset(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java new file mode 100644 index 0000000..f51c917 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java
@@ -0,0 +1,874 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.actions; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.CharMatcher; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.actions.extra.SpawnInfo; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.collect.IterablesChain; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.protobuf.GeneratedMessage.GeneratedExtension; + +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.CheckReturnValue; + +/** + * An Action representing an arbitrary subprocess to be forked and exec'd. + */ +public class SpawnAction extends AbstractAction { + private static class ExtraActionInfoSupplier<T> { + private final GeneratedExtension<ExtraActionInfo, T> extension; + private final T value; + + private ExtraActionInfoSupplier( + GeneratedExtension<ExtraActionInfo, T> extension, + T value) { + this.extension = extension; + this.value = value; + } + + void extend(ExtraActionInfo.Builder builder) { + builder.setExtension(extension, value); + } + } + + private static final String GUID = "ebd6fce3-093e-45ee-adb6-bf513b602f0d"; + + private final CommandLine argv; + + private final String progressMessage; + private final String mnemonic; + // entries are (directory for remote execution, Artifact) + private final Map<PathFragment, Artifact> inputManifests; + + private final ResourceSet resourceSet; + private final ImmutableMap<String, String> environment; + private final ImmutableMap<String, String> executionInfo; + + private final ExtraActionInfoSupplier<?> extraActionInfoSupplier; + + /** + * Constructs a SpawnAction using direct initialization arguments. + * <p> + * All collections provided must not be subsequently modified. + * + * @param owner the owner of the Action. + * @param inputs the set of all files potentially read by this action; must + * not be subsequently modified. + * @param outputs the set of all files written by this action; must not be + * subsequently modified. + * @param resourceSet the resources consumed by executing this Action + * @param environment the map of environment variables. + * @param argv the command line to execute. This is merely a list of options + * to the executable, and is uninterpreted by the build tool for the + * purposes of dependency checking; typically it may include the names + * of input and output files, but this is not necessary. + * @param progressMessage the message printed during the progression of the build + * @param mnemonic the mnemonic that is reported in the master log. + */ + public SpawnAction(ActionOwner owner, + Iterable<Artifact> inputs, Iterable<Artifact> outputs, + ResourceSet resourceSet, + CommandLine argv, + Map<String, String> environment, + String progressMessage, + String mnemonic) { + this(owner, inputs, outputs, + resourceSet, argv, ImmutableMap.copyOf(environment), + ImmutableMap.<String, String>of(), progressMessage, + ImmutableMap.<PathFragment, Artifact>of(), mnemonic, null); + } + + /** + * Constructs a SpawnAction using direct initialization arguments. + * + * <p>All collections provided must not be subsequently modified. + * + * @param owner the owner of the Action. + * @param inputs the set of all files potentially read by this action; must + * not be subsequently modified. + * @param outputs the set of all files written by this action; must not be + * subsequently modified. + * @param resourceSet the resources consumed by executing this Action + * @param environment the map of environment variables. + * @param executionInfo out-of-band information for scheduling the spawn. + * @param argv the argv array (including argv[0]) of arguments to pass. This + * is merely a list of options to the executable, and is uninterpreted + * by the build tool for the purposes of dependency checking; typically + * it may include the names of input and output files, but this is not + * necessary. + * @param progressMessage the message printed during the progression of the build + * @param inputManifests entries in inputs that are symlink manifest files. + * These are passed to remote execution in the environment rather than as inputs. + * @param mnemonic the mnemonic that is reported in the master log. + */ + public SpawnAction(ActionOwner owner, + Iterable<Artifact> inputs, Iterable<Artifact> outputs, + ResourceSet resourceSet, + CommandLine argv, + ImmutableMap<String, String> environment, + ImmutableMap<String, String> executionInfo, + String progressMessage, + ImmutableMap<PathFragment, Artifact> inputManifests, + String mnemonic, + ExtraActionInfoSupplier<?> extraActionInfoSupplier) { + super(owner, inputs, outputs); + this.resourceSet = resourceSet; + this.executionInfo = executionInfo; + this.environment = environment; + this.argv = argv; + this.progressMessage = progressMessage; + this.inputManifests = inputManifests; + this.mnemonic = mnemonic; + this.extraActionInfoSupplier = extraActionInfoSupplier; + } + + /** + * Returns the (immutable) list of all arguments, including the command name, argv[0]. + */ + @VisibleForTesting + public List<String> getArguments() { + return ImmutableList.copyOf(argv.arguments()); + } + + /** + * Returns command argument, argv[0]. + */ + @VisibleForTesting + public String getCommandFilename() { + return Iterables.getFirst(argv.arguments(), null); + } + + /** + * Returns the (immutable) list of arguments, excluding the command name, + * argv[0]. + */ + @VisibleForTesting + public List<String> getRemainingArguments() { + return ImmutableList.copyOf(Iterables.skip(argv.arguments(), 1)); + } + + @VisibleForTesting + public boolean isShellCommand() { + return argv.isShellCommand(); + } + + /** + * Executes the action without handling ExecException errors. + * + * <p>Called by {@link #execute}. + */ + protected void internalExecute( + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException { + getContext(actionExecutionContext.getExecutor()).exec(getSpawn(), actionExecutionContext); + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + try { + internalExecute(actionExecutionContext); + } catch (ExecException e) { + String failMessage = progressMessage; + if (isShellCommand()) { + // The possible reasons it could fail are: shell executable not found, shell + // exited non-zero, or shell died from signal. The first is impossible + // and the second two aren't very interesting, so in the interests of + // keeping the noise-level down, we don't print a reason why, just the + // command that failed. + // + // 0=shell executable, 1=shell command switch, 2=command + failMessage = "error executing shell command: " + "'" + + truncate(Iterables.get(argv.arguments(), 2), 200) + "'"; + } + throw e.toActionExecutionException(failMessage, executor.getVerboseFailures(), this); + } + } + + /** + * Returns s, truncated to no more than maxLen characters, appending an + * ellipsis if truncation occurred. + */ + private static String truncate(String s, int maxLen) { + return s.length() > maxLen + ? s.substring(0, maxLen - "...".length()) + "..." + : s; + } + + /** + * Returns a Spawn that is representative of the command that this Action + * will execute. This function must not modify any state. + */ + public Spawn getSpawn() { + return new ActionSpawn(); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addStrings(argv.arguments()); + f.addString(getMnemonic()); + f.addInt(inputManifests.size()); + for (Map.Entry<PathFragment, Artifact> input : inputManifests.entrySet()) { + f.addString(input.getKey().getPathString() + "/"); + f.addPath(input.getValue().getExecPath()); + } + f.addStringMap(getEnvironment()); + f.addStringMap(getExecutionInfo()); + return f.hexDigestAndReset(); + } + + @Override + public String describeKey() { + StringBuilder message = new StringBuilder(); + message.append(getProgressMessage()); + message.append('\n'); + for (Map.Entry<String, String> entry : getEnvironment().entrySet()) { + message.append(" Environment variable: "); + message.append(ShellEscaper.escapeString(entry.getKey())); + message.append('='); + message.append(ShellEscaper.escapeString(entry.getValue())); + message.append('\n'); + } + for (String argument : ShellEscaper.escapeAll(argv.arguments())) { + message.append(" Argument: "); + message.append(argument); + message.append('\n'); + } + return message.toString(); + } + + @Override + public final String getMnemonic() { + return mnemonic; + } + + @Override + protected String getRawProgressMessage() { + return progressMessage; + } + + @Override + public ExtraActionInfo.Builder getExtraActionInfo() { + ExtraActionInfo.Builder builder = super.getExtraActionInfo(); + if (extraActionInfoSupplier == null) { + Spawn spawn = getSpawn(); + SpawnInfo spawnInfo = spawn.getExtraActionInfo(); + + return builder + .setExtension(SpawnInfo.spawnInfo, spawnInfo); + } else { + extraActionInfoSupplier.extend(builder); + return builder; + } + } + + /** + * Returns the environment in which to run this action. + */ + public Map<String, String> getEnvironment() { + return environment; + } + + /** + * Returns the out-of-band execution data for this action. + */ + public Map<String, String> getExecutionInfo() { + return executionInfo; + } + + @Override + public String describeStrategy(Executor executor) { + return getContext(executor).strategyLocality(getMnemonic(), isRemotable()); + } + + protected SpawnActionContext getContext(Executor executor) { + return executor.getSpawnActionContext(getMnemonic()); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + SpawnActionContext context = getContext(executor); + if (context.isRemotable(getMnemonic(), isRemotable())) { + return ResourceSet.ZERO; + } + return resourceSet; + } + + /** + * Returns true if this can be run remotely. + */ + public final boolean isRemotable() { + // TODO(bazel-team): get rid of this method. + return !executionInfo.containsKey("local"); + } + + /** + * The Spawn which this SpawnAction will execute. + */ + private class ActionSpawn extends BaseSpawn { + + private final List<Artifact> filesets = new ArrayList<>(); + + public ActionSpawn() { + super(ImmutableList.copyOf(argv.arguments()), + ImmutableMap.<String, String>of(), + executionInfo, + inputManifests, + SpawnAction.this, + resourceSet); + for (Artifact input : getInputs()) { + if (input.isFileset()) { + filesets.add(input); + } + } + } + + @Override + public ImmutableMap<String, String> getEnvironment() { + return ImmutableMap.copyOf(SpawnAction.this.getEnvironment()); + } + + @Override + public ImmutableList<Artifact> getFilesetManifests() { + return ImmutableList.copyOf(filesets); + } + + @Override + public Iterable<? extends ActionInput> getInputFiles() { + // Remove Fileset directories in inputs list. Instead, these are + // included as manifests in getEnvironment(). + List<Artifact> inputs = Lists.newArrayList(getInputs()); + inputs.removeAll(filesets); + inputs.removeAll(inputManifests.values()); + return inputs; + // Also expand middleman artifacts. + } + } + + /** + * Builder class to construct {@link SpawnAction} instances. + */ + public static class Builder { + + private final NestedSetBuilder<Artifact> inputsBuilder = + NestedSetBuilder.stableOrder(); + private final List<Artifact> outputs = new ArrayList<>(); + private final Map<PathFragment, Artifact> inputManifests = new LinkedHashMap<>(); + private ResourceSet resourceSet = AbstractAction.DEFAULT_RESOURCE_SET; + private ImmutableMap<String, String> environment = ImmutableMap.of(); + private ImmutableMap<String, String> executionInfo = ImmutableMap.of(); + private boolean isShellCommand = false; + private boolean useDefaultShellEnvironment = false; + private PathFragment executable; + // executableArgs does not include the executable itself. + private List<String> executableArgs; + private final IterablesChain.Builder<String> argumentsBuilder = IterablesChain.builder(); + private CommandLine commandLine; + + private String progressMessage; + private ParamFileInfo paramFileInfo = null; + private String mnemonic = "Unknown"; + private ExtraActionInfoSupplier<?> extraActionInfoSupplier = null; + + /** + * Creates a SpawnAction builder. + */ + public Builder() {} + + /** + * Creates a builder that is a copy of another builder. + */ + public Builder(Builder other) { + this.inputsBuilder.addTransitive(other.inputsBuilder.build()); + this.outputs.addAll(other.outputs); + this.inputManifests.putAll(other.inputManifests); + this.resourceSet = other.resourceSet; + this.environment = other.environment; + this.executionInfo = other.executionInfo; + this.isShellCommand = other.isShellCommand; + this.useDefaultShellEnvironment = other.useDefaultShellEnvironment; + this.executable = other.executable; + this.executableArgs = (other.executableArgs != null) + ? Lists.newArrayList(other.executableArgs) + : null; + this.argumentsBuilder.add(other.argumentsBuilder.build()); + this.commandLine = other.commandLine; + this.progressMessage = other.progressMessage; + this.paramFileInfo = other.paramFileInfo; + this.mnemonic = other.mnemonic; + } + + /** + * Builds the SpawnAction using the passed in action configuration and returns it and all + * dependent actions. The first item of the returned array is always the SpawnAction itself. + * + * <p>This method makes a copy of all the collections, so it is safe to reuse the builder after + * this method returns. + * + * <p>This is annotated with @CheckReturnValue, which causes a compiler error when you call this + * method and ignore its return value. This is because some time ago, calling .build() had the + * side-effect of registering it with the RuleContext that was passed in to the constructor. + * This logic was removed, but if people don't notice and still rely on the side-effect, things + * may break. + * + * @return the SpawnAction and any actions required by it, with the first item always being the + * SpawnAction itself. + */ + @CheckReturnValue + public Action[] build(ActionConstructionContext context) { + return build(context.getActionOwner(), context.getAnalysisEnvironment(), + context.getConfiguration()); + } + + @VisibleForTesting @CheckReturnValue + public Action[] build(ActionOwner owner, AnalysisEnvironment analysisEnvironment, + BuildConfiguration configuration) { + if (isShellCommand && executable == null) { + executable = configuration.getShExecutable(); + } + Preconditions.checkNotNull(executable); + Preconditions.checkNotNull(executableArgs); + + if (useDefaultShellEnvironment) { + this.environment = configuration.getDefaultShellEnvironment(); + } + + ImmutableList<String> argv = ImmutableList.<String>builder() + .add(executable.getPathString()) + .addAll(executableArgs) + .build(); + + Iterable<String> arguments = argumentsBuilder.build(); + + Artifact paramsFile = ParamFileHelper.getParamsFile(argv, arguments, commandLine, + paramFileInfo, configuration, analysisEnvironment, outputs); + + List<Action> actions = new ArrayList<>(); + CommandLine actualCommandLine; + if (paramsFile != null) { + actualCommandLine = ParamFileHelper.createWithParamsFile(argv, arguments, commandLine, + isShellCommand, owner, actions, paramFileInfo, paramsFile); + } else { + actualCommandLine = ParamFileHelper.createWithoutParamsFile(argv, arguments, commandLine, + isShellCommand); + } + + Iterable<Artifact> actualInputs = collectActualInputs(paramsFile); + + actions.add(0, new SpawnAction(owner, actualInputs, ImmutableList.copyOf(outputs), + resourceSet, actualCommandLine, environment, executionInfo, progressMessage, + ImmutableMap.copyOf(inputManifests), mnemonic, extraActionInfoSupplier)); + return actions.toArray(new Action[actions.size()]); + } + + private Iterable<Artifact> collectActualInputs(Artifact parameterFile) { + if (parameterFile != null) { + inputsBuilder.add(parameterFile); + } + return inputsBuilder.build(); + } + + /** + * Adds an input to this action. + */ + public Builder addInput(Artifact artifact) { + inputsBuilder.add(artifact); + return this; + } + + /** + * Adds inputs to this action. + */ + public Builder addInputs(Iterable<Artifact> artifacts) { + inputsBuilder.addAll(artifacts); + return this; + } + + /** + * Adds transitive inputs to this action. + */ + public Builder addTransitiveInputs(NestedSet<Artifact> artifacts) { + inputsBuilder.addTransitive(artifacts); + return this; + } + + public Builder addInputManifest(Artifact artifact, PathFragment remote) { + inputManifests.put(remote, artifact); + return this; + } + + public Builder addOutput(Artifact artifact) { + outputs.add(artifact); + return this; + } + + public Builder addOutputs(Iterable<Artifact> artifacts) { + Iterables.addAll(outputs, artifacts); + return this; + } + + public Builder setResources(ResourceSet resourceSet) { + this.resourceSet = resourceSet; + return this; + } + + /** + * Sets the map of environment variables. + */ + public Builder setEnvironment(Map<String, String> environment) { + this.environment = ImmutableMap.copyOf(environment); + this.useDefaultShellEnvironment = false; + return this; + } + + /** + * Sets the map of execution info. + */ + public Builder setExecutionInfo(Map<String, String> info) { + this.executionInfo = ImmutableMap.copyOf(info); + return this; + } + + /** + * Sets the environment to the configurations default shell environment, + * see {@link BuildConfiguration#getDefaultShellEnvironment}. + */ + public Builder useDefaultShellEnvironment() { + this.environment = null; + this.useDefaultShellEnvironment = true; + return this; + } + + /** + * Sets the executable path; the path is interpreted relative to the + * execution root. + * + * <p>Calling this method overrides any previous values set via calls to + * {@link #setExecutable(Artifact)}, {@link #setJavaExecutable}, or + * {@link #setShellCommand(String)}. + */ + public Builder setExecutable(PathFragment executable) { + this.executable = executable; + this.executableArgs = Lists.newArrayList(); + this.isShellCommand = false; + return this; + } + + /** + * Sets the executable as an artifact. + * + * <p>Calling this method overrides any previous values set via calls to + * {@link #setExecutable(Artifact)}, {@link #setJavaExecutable}, or + * {@link #setShellCommand(String)}. + */ + public Builder setExecutable(Artifact executable) { + return setExecutable(executable.getExecPath()); + } + + /** + * Sets the executable as a configured target. Automatically adds the files + * to run to the inputs and uses the executable of the target as the + * executable. + * + * <p>Calling this method overrides any previous values set via calls to + * {@link #setExecutable(Artifact)}, {@link #setJavaExecutable}, or + * {@link #setShellCommand(String)}. + */ + public Builder setExecutable(TransitiveInfoCollection executable) { + FilesToRunProvider provider = executable.getProvider(FilesToRunProvider.class); + Preconditions.checkArgument(provider != null); + return setExecutable(provider); + } + + /** + * Sets the executable as a configured target. Automatically adds the files + * to run to the inputs and uses the executable of the target as the + * executable. + * + * <p>Calling this method overrides any previous values set via calls to + * {@link #setExecutable}, {@link #setJavaExecutable}, or + * {@link #setShellCommand(String)}. + */ + public Builder setExecutable(FilesToRunProvider executableProvider) { + Preconditions.checkArgument(executableProvider.getExecutable() != null, + "The target does not have an executable"); + setExecutable(executableProvider.getExecutable().getExecPath()); + return addTool(executableProvider); + } + + private Builder setJavaExecutable(PathFragment javaExecutable, Artifact deployJar, + List<String> jvmArgs, String... launchArgs) { + this.executable = javaExecutable; + this.executableArgs = Lists.newArrayList(); + executableArgs.add("-Xverify:none"); + executableArgs.addAll(jvmArgs); + for (String arg : launchArgs) { + executableArgs.add(arg); + } + inputsBuilder.add(deployJar); + this.isShellCommand = false; + return this; + } + + /** + * Sets the executable to be a java class executed from the given deploy + * jar. The deploy jar is automatically added to the action inputs. + * + * <p>Calling this method overrides any previous values set via calls to + * {@link #setExecutable}, {@link #setJavaExecutable}, or + * {@link #setShellCommand(String)}. + */ + public Builder setJavaExecutable(PathFragment javaExecutable, + Artifact deployJar, String javaMainClass, List<String> jvmArgs) { + return setJavaExecutable(javaExecutable, deployJar, jvmArgs, "-cp", + deployJar.getExecPathString(), javaMainClass); + } + + /** + * Sets the executable to be a jar executed from the given deploy jar. The deploy jar is + * automatically added to the action inputs. + * + * <p>This method is similar to {@link #setJavaExecutable} but it assumes that the Jar artifact + * declares a main class. + * + * <p>Calling this method overrides any previous values set via calls to {@link #setExecutable}, + * {@link #setJavaExecutable}, or {@link #setShellCommand(String)}. + */ + public Builder setJarExecutable(PathFragment javaExecutable, + Artifact deployJar, List<String> jvmArgs) { + return setJavaExecutable(javaExecutable, deployJar, jvmArgs, "-jar", + deployJar.getExecPathString()); + } + + /** + * Sets the executable to be the shell and adds the given command as the + * command to be executed. + * + * <p>Note that this will not clear the arguments, so any arguments will + * be passed in addition to the command given here. + * + * <p>Calling this method overrides any previous values set via calls to + * {@link #setExecutable(Artifact)}, {@link #setJavaExecutable}, or + * {@link #setShellCommand(String)}. + */ + public Builder setShellCommand(String command) { + this.executable = null; + // 0=shell command switch, 1=command + this.executableArgs = Lists.newArrayList("-c", command); + this.isShellCommand = true; + return this; + } + + /** + * Sets the executable to be the shell and adds the given interned commands as the + * commands to be executed. + */ + public Builder setShellCommand(Iterable<String> command) { + this.executable = new PathFragment(Iterables.getFirst(command, null)); + // The first item of the commands is the shell executable that should be used. + this.executableArgs = ImmutableList.copyOf(Iterables.skip(command, 1)); + this.isShellCommand = true; + return this; + } + + /** + * Adds an executable and its runfiles, so it can be called from a shell command. + */ + public Builder addTool(FilesToRunProvider tool) { + addInputs(tool.getFilesToRun()); + if (tool.getRunfilesManifest() != null) { + addInputManifest(tool.getRunfilesManifest(), + BaseSpawn.runfilesForFragment(tool.getExecutable().getExecPath())); + } + return this; + } + + /** + * Appends the arguments to the list of executable arguments. + */ + public Builder addExecutableArguments(String... arguments) { + Preconditions.checkState(executableArgs != null); + executableArgs.addAll(Arrays.asList(arguments)); + return this; + } + + /** + * Add multiple arguments in the order they are returned by the collection + * to the list of executable arguments. + */ + public Builder addExecutableArguments(Iterable<String> arguments) { + Preconditions.checkState(executableArgs != null); + Iterables.addAll(executableArgs, arguments); + return this; + } + + /** + * Appends the argument to the list of command-line arguments. + */ + public Builder addArgument(String argument) { + Preconditions.checkState(commandLine == null); + argumentsBuilder.addElement(argument); + return this; + } + + /** + * Appends the arguments to the list of command-line arguments. + */ + public Builder addArguments(String... arguments) { + Preconditions.checkState(commandLine == null); + argumentsBuilder.add(ImmutableList.copyOf(arguments)); + return this; + } + + /** + * Add multiple arguments in the order they are returned by the collection. + */ + public Builder addArguments(Iterable<String> arguments) { + Preconditions.checkState(commandLine == null); + argumentsBuilder.add(CollectionUtils.makeImmutable(arguments)); + return this; + } + + /** + * Appends the argument both to the inputs and to the list of command-line + * arguments. + */ + public Builder addInputArgument(Artifact argument) { + Preconditions.checkState(commandLine == null); + addInput(argument); + addArgument(argument.getExecPathString()); + return this; + } + + /** + * Appends the arguments both to the inputs and to the list of command-line + * arguments. + */ + public Builder addInputArguments(Iterable<Artifact> arguments) { + for (Artifact argument : arguments) { + addInputArgument(argument); + } + return this; + } + + /** + * Appends the argument both to the ouputs and to the list of command-line + * arguments. + */ + public Builder addOutputArgument(Artifact argument) { + Preconditions.checkState(commandLine == null); + outputs.add(argument); + argumentsBuilder.addElement(argument.getExecPathString()); + return this; + } + + /** + * Sets a delegate to compute the command line at a later time. This method + * cannot be used in conjunction with the {@link #addArgument} or {@link + * #addArguments} methods. + * + * <p>The main intention of this method is to save memory by allowing + * client-controlled sharing between actions and configured targets. + * Objects passed to this method MUST be immutable. + */ + public Builder setCommandLine(CommandLine commandLine) { + Preconditions.checkState(argumentsBuilder.isEmpty()); + this.commandLine = commandLine; + return this; + } + + public Builder setProgressMessage(String progressMessage) { + this.progressMessage = progressMessage; + return this; + } + + public Builder setMnemonic(String mnemonic) { + Preconditions.checkArgument( + !mnemonic.isEmpty() && CharMatcher.JAVA_LETTER_OR_DIGIT.matchesAllOf(mnemonic), + "mnemonic must only contain letters and/or digits, and have non-zero length, was: \"%s\"", + mnemonic); + this.mnemonic = mnemonic; + return this; + } + + public <T> Builder setExtraActionInfo( + GeneratedExtension<ExtraActionInfo, T> extension, T value) { + this.extraActionInfoSupplier = new ExtraActionInfoSupplier<T>(extension, value); + return this; + } + + /** + * Enable use of a parameter file and set the encoding to ISO-8859-1 (latin1). + * + * <p>In order to use parameter files, at least one output artifact must be specified. + */ + public Builder useParameterFile(ParameterFileType parameterFileType) { + return useParameterFile(parameterFileType, ISO_8859_1, "@"); + } + + /** + * Enable or disable the use of a parameter file, set the encoding to the given value, and + * specify the argument prefix to use in passing the parameter file name to the tool. + * + * <p>The default argument prefix is "@". In order to use parameter files, at least one output + * artifact must be specified. + */ + public Builder useParameterFile( + ParameterFileType parameterFileType, Charset charset, String flagPrefix) { + paramFileInfo = new ParamFileInfo(parameterFileType, charset, flagPrefix); + return this; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkAction.java new file mode 100644 index 0000000..2bbb3e2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkAction.java
@@ -0,0 +1,130 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.actions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; + +/** + * Action to create a symbolic link. + */ +public class SymlinkAction extends AbstractAction { + + private static final String GUID = "349675b5-437c-4da8-891a-7fb98fba6ab5"; + + private final PathFragment inputPath; + private final Artifact output; + private final String progressMessage; + + /** + * Creates a new SymlinkAction instance. + * + * @param owner the action owner. + * @param input the Artifact that will be the src of the symbolic link. + * @param output the Artifact that will be created by executing this Action. + * @param progressMessage the progress message. + */ + public SymlinkAction(ActionOwner owner, Artifact input, Artifact output, + String progressMessage) { + // These actions typically have only one input and one output, which + // become the sole and primary in their respective lists. + this(owner, input.getExecPath(), input, output, progressMessage); + } + + /** + * Creates a new SymlinkAction instance, where the inputPath + * may be different than that input artifact's path. This is + * only useful when dealing with runfiles trees where + * link target is a directory. + * + * @param owner the action owner. + * @param inputPath the Path that will be the src of the symbolic link. + * @param input the Artifact that is required to build the inputPath. + * @param output the Artifact that will be created by executing this Action. + * @param progressMessage the progress message. + */ + public SymlinkAction(ActionOwner owner, PathFragment inputPath, Artifact input, + Artifact output, String progressMessage) { + super(owner, ImmutableList.of(input), ImmutableList.of(output)); + this.inputPath = Preconditions.checkNotNull(inputPath); + this.output = Preconditions.checkNotNull(output); + this.progressMessage = progressMessage; + } + + public PathFragment getInputPath() { + return inputPath; + } + + public Path getOutputPath() { + return output.getPath(); + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException { + try { + getOutputPath().createSymbolicLink( + actionExecutionContext.getExecutor().getExecRoot().getRelative(inputPath)); + } catch (IOException e) { + throw new ActionExecutionException("failed to create symbolic link '" + + Iterables.getOnlyElement(getOutputs()).prettyPrint() + + "' to the '" + Iterables.getOnlyElement(getInputs()).prettyPrint() + + "' due to I/O error: " + e.getMessage(), e, this, false); + } + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return new ResourceSet(/*memoryMb=*/0, /*cpuUsage=*/0, /*ioUsage=*/0.0); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + // We don't normally need to add inputs to the key. In this case, however, the inputPath can be + // different from the actual input artifact. + f.addPath(inputPath); + return f.hexDigestAndReset(); + } + + @Override + public String getMnemonic() { + return "Symlink"; + } + + @Override + protected String getRawProgressMessage() { + return progressMessage; + } + + @Override + public String describeStrategy(Executor executor) { + return "local"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/TemplateExpansionAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/TemplateExpansionAction.java new file mode 100644 index 0000000..b2c83fb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/TemplateExpansionAction.java
@@ -0,0 +1,335 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.actions; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.ResourceFileLoader; +import com.google.devtools.build.lib.util.StringUtilities; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collection; +import java.util.List; + +/** + * Action to expand a template and write the expanded content to a file. + */ +public class TemplateExpansionAction extends AbstractFileWriteAction { + + private static final String GUID = "786c1fe0-dca8-407a-b108-e1ecd6d1bc7f"; + + /** + * A pair of a string to be substituted and a string to substitute it with. + * For simplicity, these are called key and value. All implementations must + * be immutable, and always return the identical key. The returned values + * must be the same, though they need not be the same object. + * + * <p>It should be assumed that the {@link #getKey} invocation is cheap, and + * that the {@link #getValue} invocation is expensive. + */ + public abstract static class Substitution { + private Substitution() { + } + + public abstract String getKey(); + public abstract String getValue(); + + /** + * Returns an immutable Substitution instance for the given key and value. + */ + public static Substitution of(final String key, final String value) { + return new Substitution() { + @Override + public String getKey() { + return key; + } + + @Override + public String getValue() { + return value; + } + }; + } + + /** + * Returns an immutable Substitution instance for the key and list of values. The + * values will be joined by spaces before substitution. + */ + public static Substitution ofSpaceSeparatedList(final String key, final List<?> value) { + return new Substitution() { + @Override + public String getKey() { + return key; + } + + @Override + public String getValue() { + return Joiner.on(" ").join(value); + } + }; + } + } + + /** + * A substitution with a fixed key, and a computed value. The computed value + * must not change over the lifetime of an instance, though the {@link + * #getValue} method may return different String objects. + * + * <p>It should be assumed that the {@link #getKey} invocation is cheap, and + * that the {@link #getValue} invocation is expensive. + */ + public abstract static class ComputedSubstitution extends Substitution { + private final String key; + + public ComputedSubstitution(String key) { + this.key = key; + } + + @Override + public String getKey() { + return key; + } + } + + /** + * A template that contains text content, or alternatively throws an {@link + * IOException}. + */ + public abstract static class Template { + + /** + * We only allow subclasses in this file. + */ + private Template() { + } + + /** + * Returns the text content of the template. + */ + protected abstract String getContent() throws IOException; + + /** + * Returns a string that is used for the action key. This must change if + * the getContent method returns something different, but is not allowed to + * throw an exception. + */ + protected abstract String getKey(); + + /** + * Loads a template from the given resource. The resource is looked up + * relative to the given class. If the resource cannot be loaded, the returned + * template throws an {@link IOException} when {@link #getContent} is + * called. This makes it safe to use this method in a constant initializer. + */ + public static Template forResource(final Class<?> relativeToClass, final String templateName) { + try { + String content = ResourceFileLoader.loadResource(relativeToClass, templateName); + return forString(content); + } catch (final IOException e) { + return new Template() { + @Override + protected String getContent() throws IOException { + throw new IOException("failed to load resource file '" + templateName + + "' due to I/O error: " + e.getMessage(), e); + } + + @Override + protected String getKey() { + return "ERROR: " + e.getMessage(); + } + }; + } + } + + /** + * Returns a template for the given text string. + */ + public static Template forString(final String templateText) { + return new Template() { + @Override + protected String getContent() { + return templateText; + } + + @Override + protected String getKey() { + return templateText; + } + }; + } + + /** + * Returns a template that loads the given artifact. It is important that + * the artifact is also an input for the action, or this won't work. + * Therefore this method is private, and you should use the corresponding + * {@link TemplateExpansionAction} constructor. + */ + private static Template forArtifact(final Artifact templateArtifact) { + return new Template() { + @Override + protected String getContent() throws IOException { + Path templatePath = templateArtifact.getPath(); + try { + return new String(FileSystemUtils.readContentAsLatin1(templatePath)); + } catch (IOException e) { + throw new IOException("failed to load template file '" + templatePath.getPathString() + + "' due to I/O error: " + e.getMessage(), e); + } + } + + @Override + protected String getKey() { + // This isn't strictly necessary, because the action inputs are automatically considered. + return "ARTIFACT: " + templateArtifact.getExecPathString(); + } + }; + } + } + + private final Template template; + private final List<Substitution> substitutions; + + /** + * Creates a new TemplateExpansionAction instance. + * + * @param owner the action owner. + * @param inputs the Artifacts that this Action depends on + * @param output the Artifact that will be created by executing this Action. + * @param template the template that will be expanded by this Action. + * @param substitutions the substitutions that will be applied to the + * template. All substitutions will be applied in order. + * @param makeExecutable iff true will change the output file to be + * executable. + */ + private TemplateExpansionAction(ActionOwner owner, + Collection<Artifact> inputs, + Artifact output, + Template template, + List<Substitution> substitutions, + boolean makeExecutable) { + super(owner, inputs, output, makeExecutable); + this.template = template; + this.substitutions = ImmutableList.copyOf(substitutions); + } + + /** + * Creates a new TemplateExpansionAction instance for an artifact template. + * + * @param owner the action owner. + * @param templateArtifact the Artifact that will be read as the text template + * file + * @param output the Artifact that will be created by executing this Action. + * @param substitutions the substitutions that will be applied to the + * template. All substitutions will be applied in order. + * @param makeExecutable iff true will change the output file to be + * executable. + */ + public TemplateExpansionAction(ActionOwner owner, + Artifact templateArtifact, + Artifact output, + List<Substitution> substitutions, + boolean makeExecutable) { + this(owner, ImmutableList.of(templateArtifact), output, Template.forArtifact(templateArtifact), + substitutions, makeExecutable); + } + + /** + * Creates a new TemplateExpansionAction instance without inputs. + * + * @param owner the action owner. + * @param output the Artifact that will be created by executing this Action. + * @param template the template + * @param substitutions the substitutions that will be applied to the + * template. All substitutions will be applied in order. + * @param makeExecutable iff true will change the output file to be + * executable. + */ + public TemplateExpansionAction(ActionOwner owner, + Artifact output, + Template template, + List<Substitution> substitutions, + boolean makeExecutable) { + this(owner, Artifact.NO_ARTIFACTS, output, template, substitutions, makeExecutable); + } + + /** + * Expands the template by applying all substitutions. + * @param template + * @return the expanded text. + */ + private String expandTemplate(String template) { + for (Substitution entry : substitutions) { + template = StringUtilities.replaceAllLiteral(template, entry.getKey(), entry.getValue()); + } + return template; + } + + @VisibleForTesting + public String getFileContents() throws IOException { + return expandTemplate(template.getContent()); + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, + Executor executor) throws IOException { + final byte[] bytes = getFileContents().getBytes(UTF_8); + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + out.write(bytes); + } + }; + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addString(String.valueOf(makeExecutable)); + f.addString(template.getKey()); + f.addInt(substitutions.size()); + for (Substitution entry : substitutions) { + f.addString(entry.getKey()); + f.addString(entry.getValue()); + } + return f.hexDigestAndReset(); + } + + @Override + public String getMnemonic() { + return "TemplateExpand"; + } + + @Override + protected String getRawProgressMessage() { + return "Expanding template " + Iterables.getOnlyElement(getOutputs()).prettyPrint(); + } + + public List<Substitution> getSubstitutions() { + return substitutions; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/buildinfo/BuildInfoCollection.java b/src/main/java/com/google/devtools/build/lib/analysis/buildinfo/BuildInfoCollection.java new file mode 100644 index 0000000..54067a0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/buildinfo/BuildInfoCollection.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.buildinfo; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; + +import java.util.List; + +/** + * A collection of build-info files for both stamped and unstamped modes. + */ +public final class BuildInfoCollection { + private final ImmutableList<Action> actions; + private final ImmutableList<Artifact> stampedBuildInfo; + private final ImmutableList<Artifact> redactedBuildInfo; + + public BuildInfoCollection(List<? extends Action> actions, List<Artifact> stampedBuildInfo, + List<Artifact> redactedBuildInfo) { + this.actions = ImmutableList.copyOf(actions); + this.stampedBuildInfo = ImmutableList.copyOf(stampedBuildInfo); + this.redactedBuildInfo = ImmutableList.copyOf(redactedBuildInfo); + } + + public ImmutableList<Action> getActions() { + return actions; + } + + public ImmutableList<Artifact> getStampedBuildInfo() { + return stampedBuildInfo; + } + + public ImmutableList<Artifact> getRedactedBuildInfo() { + return redactedBuildInfo; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/buildinfo/BuildInfoFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/buildinfo/BuildInfoFactory.java new file mode 100644 index 0000000..c6ec4d7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/buildinfo/BuildInfoFactory.java
@@ -0,0 +1,99 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.buildinfo; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.Serializable; + +/** + * A factory for language-specific build-info files. Use this to translate the build-info into + * target-independent language-specific files. The generated actions are registered into the action + * graph on every build, but only executed if anything depends on them. + */ +public interface BuildInfoFactory extends Serializable { + /** + * Type of the build-data artifact. + */ + public enum BuildInfoType { + /** + * Ignore changes to this file for the purposes of determining whether an action needs to be + * re-executed. I.e., the action is only re-executed if at least one other input has changed. + */ + NO_REBUILD, + + /** + * Changes to this file trigger re-execution of actions, similar to source file changes. + */ + FORCE_REBUILD_IF_CHANGED; + } + + /** + * Context for the creation of build-info artifacts. + */ + public interface BuildInfoContext { + Artifact getBuildInfoArtifact(PathFragment rootRelativePath, Root root, BuildInfoType type); + Root getBuildDataDirectory(); + } + + /** + * Build-info key for lookup from the {@link + * com.google.devtools.build.lib.analysis.AnalysisEnvironment}. + */ + public static final class BuildInfoKey implements Serializable { + private final String name; + + public BuildInfoKey(String name) { + this.name = name; + } + + @Override + public String toString() { + return name; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof BuildInfoKey)) { + return false; + } + return name.equals(((BuildInfoKey) o).name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + } + + /** + * Create actions and artifacts for language-specific build-info files. + */ + BuildInfoCollection create(BuildInfoContext context, BuildConfiguration config, + Artifact buildInfo, Artifact buildChangelist); + + /** + * Returns the key for the information created by this factory. + */ + BuildInfoKey getKey(); + + /** + * Returns false if this build info factory is disabled based on the configuration (usually by + * checking if all required configuration fragments are present). + */ + boolean isEnabled(BuildConfiguration config); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BinTools.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BinTools.java new file mode 100644 index 0000000..6d2477d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BinTools.java
@@ -0,0 +1,191 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.config; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.vfs.Dirent; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Symlinks; + +import java.io.IOException; + +/** + * Initializes the <execRoot>/_bin/ directory that contains auxiliary tools used during action + * execution (alarm, etc). The main purpose of this is to make sure that those tools are accessible + * using relative paths from the execution root. + */ +public final class BinTools { + private final BlazeDirectories directories; + private final Path binDir; // the working bin directory under execRoot + private final ImmutableList<String> embeddedTools; + + private BinTools(BlazeDirectories directories, ImmutableList<String> tools) { + this.directories = directories; + this.binDir = directories.getExecRoot().getRelative("_bin"); + this.embeddedTools = tools; + } + + /** + * Creates an instance with the list of embedded tools obtained from scanning the directory + * into which said binaries were extracted by the launcher. + */ + public static BinTools forProduction(BlazeDirectories directories) throws IOException { + ImmutableList.Builder<String> builder = ImmutableList.builder(); + scanDirectoryRecursively(builder, directories.getEmbeddedBinariesRoot(), ""); + return new BinTools(directories, builder.build()); + } + + /** + * Creates an empty instance for testing. + */ + @VisibleForTesting + public static BinTools empty(BlazeDirectories directories) { + return new BinTools(directories, ImmutableList.<String>of()); + } + + /** + * Creates an instance for testing without actually symlinking the tools. + * + * <p>Used for tests that need a set of embedded tools to be present, but not the actual files. + */ + @VisibleForTesting + public static BinTools forUnitTesting(BlazeDirectories directories, Iterable<String> tools) { + return new BinTools(directories, ImmutableList.copyOf(tools)); + } + + /** + * Populates the _bin directory by symlinking the necessary files from the given + * srcDir, and returns the corresponding BinTools. + */ + @VisibleForTesting + public static BinTools forIntegrationTesting( + BlazeDirectories directories, String srcDir, Iterable<String> tools) + throws IOException { + Path srcPath = directories.getOutputBase().getFileSystem().getPath(srcDir); + for (String embedded : tools) { + Path runfilesPath = srcPath.getRelative(embedded); + if (!runfilesPath.isFile()) { + // The file isn't there - nothing to symlink! + // + // Note: This path is usually taken by the tests using the in-memory + // file system. They can't run the embedded scripts anyhow, so there isn't + // much point in creating a symlink to a non-existent binary here. + continue; + } + Path outputPath = directories.getExecRoot().getChild("_bin").getChild(embedded); + if (outputPath.exists()) { + outputPath.delete(); + } + FileSystemUtils.createDirectoryAndParents(outputPath.getParentDirectory()); + outputPath.createSymbolicLink(runfilesPath); + } + + return new BinTools(directories, ImmutableList.copyOf(tools)); + } + + private static void scanDirectoryRecursively( + ImmutableList.Builder<String> result, Path root, String relative) throws IOException { + for (Dirent dirent : root.readdir(Symlinks.NOFOLLOW)) { + String childRelative = relative.isEmpty() + ? dirent.getName() + : relative + "/" + dirent.getName(); + switch (dirent.getType()) { + case FILE: + result.add(childRelative); + break; + + case DIRECTORY: + scanDirectoryRecursively(result, root.getChild(dirent.getName()), childRelative); + break; + + default: + // Nothing to do here -- we ignore symlinks, since they should not be present in the + // embedded binaries tree. + break; + } + } + } + + public PathFragment getExecPath(String embedPath) { + Preconditions.checkState(embeddedTools.contains(embedPath), "%s not in %s", embedPath, + embeddedTools); + return new PathFragment("_bin").getRelative(new PathFragment(embedPath).getBaseName()); + } + + public Artifact getEmbeddedArtifact(String embedPath, ArtifactFactory artifactFactory) { + return artifactFactory.getDerivedArtifact(getExecPath(embedPath)); + } + + public ImmutableList<Artifact> getAllEmbeddedArtifacts(ArtifactFactory artifactFactory) { + ImmutableList.Builder<Artifact> builder = ImmutableList.builder(); + for (String embeddedTool : embeddedTools) { + builder.add(getEmbeddedArtifact(embeddedTool, artifactFactory)); + } + return builder.build(); + } + + /** + * Initializes the build tools not available at absolute paths. Note that + * these must be constant across all configurations. + */ + public void setupBuildTools() throws ExecException { + try { + FileSystemUtils.createDirectoryAndParents(binDir); + } catch (IOException e) { + throw new EnvironmentalExecException("could not create directory '" + binDir + "'", e); + } + + for (String embeddedPath : embeddedTools) { + setupTool(embeddedPath); + } + } + + private void setupTool(String embeddedPath) throws ExecException { + Path sourcePath = directories.getEmbeddedBinariesRoot().getRelative(embeddedPath); + Path linkPath = binDir.getRelative(new PathFragment(embeddedPath).getBaseName()); + linkTool(sourcePath, linkPath); + } + + private void linkTool(Path sourcePath, Path linkPath) throws ExecException { + if (linkPath.getFileSystem().supportsSymbolicLinks()) { + try { + if (!linkPath.isSymbolicLink()) { + // ensureSymbolicLink() does not handle the case where there is already + // a file with the same name, so we need to handle it here. + linkPath.delete(); + } + FileSystemUtils.ensureSymbolicLink(linkPath, sourcePath); + } catch (IOException e) { + throw new EnvironmentalExecException("failed to link '" + sourcePath + "'", e); + } + } else { + // For file systems that do not support linking, copy. + try { + FileSystemUtils.copyTool(sourcePath, linkPath); + } catch (IOException e) { + throw new EnvironmentalExecException("failed to copy '" + sourcePath + "'" , e); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java new file mode 100644 index 0000000..8e80211 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
@@ -0,0 +1,1944 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.config; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.PackageRootResolver; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.ViewCreationFailedException; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection.Transitions; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Attribute.Configurator; +import com.google.devtools.build.lib.packages.Attribute.SplitTransition; +import com.google.devtools.build.lib.packages.Attribute.Transition; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.PackageGroup; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.rules.test.TestActionBuilder; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.RegexFilter; +import com.google.devtools.build.lib.util.StringUtilities; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunction.Environment; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.EnumConverter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.TriState; + +import java.io.IOException; +import java.io.Serializable; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Instances of BuildConfiguration represent a collection of context + * information which may affect a build (for example: the target platform for + * compilation, or whether or not debug tables are required). In fact, all + * "environmental" information (e.g. from the tool's command-line, as opposed + * to the BUILD file) that can affect the output of any build tool should be + * explicitly represented in the BuildConfiguration instance. + * + * <p>A single build may require building tools to run on a variety of + * platforms: when compiling a server application for production, we must build + * the build tools (like compilers) to run on the host platform, but cross-compile + * the application for the production environment. + * + * <p>There is always at least one BuildConfiguration instance in any build: + * the one representing the host platform. Additional instances may be created, + * in a cross-compilation build, for example. + * + * <p>Instances of BuildConfiguration are canonical: + * <pre>c1.equals(c2) <=> c1==c2.</pre> + */ +@SkylarkModule(name = "configuration", + doc = "Data required for the analysis of a target that comes from targets that " + + "depend on it and not targets that it depends on.") +public final class BuildConfiguration implements Serializable { + + /** + * An interface for language-specific configurations. + */ + public abstract static class Fragment implements Serializable { + /** + * Returns a human-readable name of the configuration fragment. + */ + public abstract String getName(); + + /** + * Validates the options for this Fragment. Issues warnings for the + * use of deprecated options, and warnings or errors for any option settings + * that conflict. + */ + @SuppressWarnings("unused") + public void reportInvalidOptions(EventHandler reporter, BuildOptions buildOptions) { + } + + /** + * Adds mapping of names to values of "Make" variables defined by this configuration. + */ + @SuppressWarnings("unused") + public void addGlobalMakeVariables(ImmutableMap.Builder<String, String> globalMakeEnvBuilder) { + } + + /** + * Collects all labels that should be implicitly loaded from labels that were specified as + * options, keyed by the name to be displayed to the user if something goes wrong. + * The resulting set only contains labels that were derived from command-line options; the + * intention is that it can be used to sanity-check that the command-line options actually + * contain these in their transitive closure. + */ + @SuppressWarnings("unused") + public void addImplicitLabels(Multimap<String, Label> implicitLabels) { + } + + /** + * Returns a string that identifies the configuration fragment. + */ + public abstract String cacheKey(); + + /** + * The fragment may use this hook to perform I/O and read data into memory that is used during + * analysis. During the analysis phase disk I/O operations are disallowed. + * + * <p>This hook is only called for the top-level configuration after the loading phase is + * complete. + */ + @SuppressWarnings("unused") + public void prepareHook(Path execPath, ArtifactFactory artifactFactory, + PathFragment genfilesPath, PackageRootResolver resolver) + throws ViewCreationFailedException { + } + + /** + * Adds all the roots from this fragment. + */ + @SuppressWarnings("unused") + public void addRoots(List<Root> roots) { + } + + /** + * Returns a (key, value) mapping to insert into the subcommand environment for coverage. + */ + public Map<String, String> getCoverageEnvironment() { + return ImmutableMap.<String, String>of(); + } + + /* + * Returns the command-line "Make" variable overrides. + */ + public ImmutableMap<String, String> getCommandLineDefines() { + return ImmutableMap.of(); + } + + /** + * Returns all the coverage labels for the fragment. + */ + public ImmutableList<Label> getCoverageLabels() { + return ImmutableList.of(); + } + + /** + * Returns the coverage report generator tool labels. + */ + public ImmutableList<Label> getCoverageReportGeneratorLabels() { + return ImmutableList.of(); + } + + /** + * Returns a fragment of the output directory name for this configuration. The output + * directory for the whole configuration contains all the short names by all fragments. + */ + @Nullable + public String getOutputDirectoryName() { + return null; + } + + /** + * This will be added to the name of the configuration, but not to the output directory name. + */ + @Nullable + public String getConfigurationNameSuffix() { + return null; + } + + /** + * The platform name is a concatenation of fragment platform names. + */ + public String getPlatformName() { + return ""; + } + + /** + * Return false if incremental build is not possible for some reason. + */ + public boolean supportsIncrementalBuild() { + return true; + } + + /** + * Return true if the fragment performs static linking. This information is needed for + * lincence checking. + */ + public boolean performsStaticLink() { + return false; + } + + /** + * Fragments should delete temporary directories they create for their inner mechanisms. + * This is only called for target configuration. + */ + @SuppressWarnings("unused") + public void prepareForExecutionPhase() throws IOException { + } + + /** + * Add items to the shell environment. + */ + @SuppressWarnings("unused") + public void setupShellEnvironment(ImmutableMap.Builder<String, String> builder) { + } + + /** + * Add mappings from generally available tool names (like "sh") to their paths + * that actions can access. + */ + @SuppressWarnings("unused") + public void defineExecutables(ImmutableMap.Builder<String, PathFragment> builder) { + } + + /** + * Returns { 'option name': 'alternative default' } entries for options where the + * "real default" should be something besides the default specified in the {@link Option} + * declaration. + */ + public Map<String, Object> lateBoundOptionDefaults() { + return ImmutableMap.of(); + } + + /** + * Declares dependencies on any relevant Skyframe values (for example, relevant FileValues). + * + * @param env the skyframe environment + */ + public void declareSkyframeDependencies(Environment env) { + } + } + + /** + * A converter from strings to Labels. + */ + public static class LabelConverter implements Converter<Label> { + @Override + public Label convert(String input) throws OptionsParsingException { + try { + // Check if the input starts with '/'. We don't check for "//" so that + // we get a better error message if the user accidentally tries to use + // an absolute path (starting with '/') for a label. + if (!input.startsWith("/")) { + input = "//" + input; + } + return Label.parseAbsolute(input); + } catch (SyntaxException e) { + throw new OptionsParsingException(e.getMessage()); + } + } + + @Override + public String getTypeDescription() { + return "a build target label"; + } + } + + public static class PluginOptionConverter implements Converter<Map.Entry<String, String>> { + @Override + public Map.Entry<String, String> convert(String input) throws OptionsParsingException { + int index = input.indexOf('='); + if (index == -1) { + throw new OptionsParsingException("Plugin option not in the plugin=option format"); + } + String option = input.substring(0, index); + String value = input.substring(index + 1); + return Maps.immutableEntry(option, value); + } + + @Override + public String getTypeDescription() { + return "An option for a plugin"; + } + } + + public static class RunsPerTestConverter extends PerLabelOptions.PerLabelOptionsConverter { + @Override + public PerLabelOptions convert(String input) throws OptionsParsingException { + try { + return parseAsInteger(input); + } catch (NumberFormatException ignored) { + return parseAsRegex(input); + } + } + + private PerLabelOptions parseAsInteger(String input) + throws NumberFormatException, OptionsParsingException { + int numericValue = Integer.parseInt(input); + if (numericValue <= 0) { + throw new OptionsParsingException("'" + input + "' should be >= 1"); + } else { + RegexFilter catchAll = new RegexFilter(Collections.singletonList(".*"), + Collections.<String>emptyList()); + return new PerLabelOptions(catchAll, Collections.singletonList(input)); + } + } + + private PerLabelOptions parseAsRegex(String input) throws OptionsParsingException { + PerLabelOptions testRegexps = super.convert(input); + if (testRegexps.getOptions().size() != 1) { + throw new OptionsParsingException( + "'" + input + "' has multiple runs for a single pattern"); + } + String runsPerTest = Iterables.getOnlyElement(testRegexps.getOptions()); + try { + int numericRunsPerTest = Integer.parseInt(runsPerTest); + if (numericRunsPerTest <= 0) { + throw new OptionsParsingException("'" + input + "' has a value < 1"); + } + } catch (NumberFormatException e) { + throw new OptionsParsingException("'" + input + "' has a non-numeric value", e); + } + return testRegexps; + } + + @Override + public String getTypeDescription() { + return "a positive integer or test_regex@runs. This flag may be passed more than once"; + } + } + + /** + * Values for the --strict_*_deps option + */ + public static enum StrictDepsMode { + /** Silently allow referencing transitive dependencies. */ + OFF, + /** Warn about transitive dependencies being used directly. */ + WARN, + /** Fail the build when transitive dependencies are used directly. */ + ERROR, + /** Transition to strict by default. */ + STRICT, + /** When no flag value is specified on the command line. */ + DEFAULT + } + + /** + * Converter for the --strict_*_deps option. + */ + public static class StrictDepsConverter extends EnumConverter<StrictDepsMode> { + public StrictDepsConverter() { + super(StrictDepsMode.class, "strict dependency checking level"); + } + } + + /** + * Options that affect the value of a BuildConfiguration instance. + * + * <p>(Note: any client that creates a view will also need to declare + * BuildView.Options, which affect the <i>mechanism</i> of view construction, + * even if they don't affect the value of the BuildConfiguration instances.) + * + * <p>IMPORTANT: when adding new options, be sure to consider whether those + * values should be propagated to the host configuration or not (see + * {@link ConfigurationFactory#getConfiguration}. + * + * <p>ALSO IMPORTANT: all option types MUST define a toString method that + * gives identical results for semantically identical option values. The + * simplest way to ensure that is to return the input string. + */ + public static class Options extends FragmentOptions implements Cloneable { + public String getCpu() { + return cpu; + } + + @Option(name = "cpu", + defaultValue = "null", + category = "semantics", + help = "The target CPU.") + public String cpu; + + @Option(name = "min_param_file_size", + defaultValue = "32768", + category = "undocumented", + help = "Minimum command line length before creating a parameter file.") + public int minParamFileSize; + + @Option(name = "experimental_extended_sanity_checks", + defaultValue = "false", + category = "undocumented", + help = "Enables internal validation checks to make sure that configured target " + + "implementations only access things they should. Causes a performance hit.") + public boolean extendedSanityChecks; + + @Option(name = "experimental_allow_runtime_deps_on_neverlink", + defaultValue = "true", + category = "undocumented", + help = "Flag to help transition from allowing to disallowing runtime_deps on neverlink" + + " Java archives. The depot needs to be cleaned up to roll this out by default.") + public boolean allowRuntimeDepsOnNeverLink; + + @Option(name = "strict_filesets", + defaultValue = "false", + category = "semantics", + help = "If this option is enabled, filesets crossing package boundaries are reported " + + "as errors. It does not work when check_fileset_dependencies_recursively is " + + "disabled.") + public boolean strictFilesets; + + // Plugins are build using the host config. To avoid cycles we just don't propagate + // this option to the host config. If one day we decide to use plugins when building + // host tools, we can improve this by (for example) creating a compiler configuration that is + // used only for building plugins. + @Option(name = "plugin", + converter = LabelConverter.class, + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Plugins to use in the build. Currently works with java_plugin.") + public List<Label> pluginList; + + @Option(name = "plugin_copt", + converter = PluginOptionConverter.class, + allowMultiple = true, + category = "flags", + defaultValue = ":", + help = "Plugin options") + public List<Map.Entry<String, String>> pluginCoptList; + + @Option(name = "stamp", + defaultValue = "true", + category = "semantics", + help = "Stamp binaries with the date, username, hostname, workspace information, etc.") + public boolean stampBinaries; + + // TODO(bazel-team): delete from OSS tree + @Option(name = "instrumentation_filter", + converter = RegexFilter.RegexFilterConverter.class, + defaultValue = "-javatests,-_test$", + category = "semantics", + help = "When coverage is enabled, only rules with names included by the " + + "specified regex-based filter will be instrumented. Rules prefixed " + + "with '-' are excluded instead. By default, rules containing " + + "'javatests' or ending with '_test' will not be instrumented.") + public RegexFilter instrumentationFilter; + + @Option(name = "show_cached_analysis_results", + defaultValue = "true", + category = "undocumented", + help = "Bazel reruns a static analysis only if it detects changes in the analysis " + + "or its dependencies. If this option is enabled, Bazel will show the analysis' " + + "results, even if it did not rerun the analysis. If this option is disabled, " + + "Bazel will show analysis results only if it reran the analysis.") + public boolean showCachedAnalysisResults; + + @Option(name = "host_cpu", + defaultValue = "null", + category = "semantics", + help = "The host CPU.") + public String hostCpu; + + @Option(name = "compilation_mode", + abbrev = 'c', + converter = CompilationMode.Converter.class, + defaultValue = "fastbuild", + category = "semantics", // Should this be "flags"? + help = "Specify the mode the binary will be built in. " + + "Values: 'fastbuild', 'dbg', 'opt'.") + public CompilationMode compilationMode; + + /** + * This option is used internally to set the short name (see {@link + * #getShortName()}) of the <i>host</i> configuration to a constant, so + * that the output files for the host are completely independent of those + * for the target, no matter what options are in force (k8/piii, opt/dbg, + * etc). + */ + @Option(name = "configuration short name", // (Spaces => can't be specified on command line.) + defaultValue = "null", + category = "undocumented") + public String shortName; + + @Option(name = "platform_suffix", + defaultValue = "null", + category = "misc", + help = "Specifies a suffix to be added to the configuration directory.") + public String platformSuffix; + + @Option(name = "test_env", + converter = Converters.OptionalAssignmentConverter.class, + allowMultiple = true, + defaultValue = "", + category = "testing", + help = "Specifies additional environment variables to be injected into the test runner " + + "environment. Variables can be either specified by name, in which case its value " + + "will be read from the Bazel client environment, or by the name=value pair. " + + "This option can be used multiple times to specify several variables. " + + "Used only by the 'bazel test' command." + ) + public List<Map.Entry<String, String>> testEnvironment; + + @Option(name = "collect_code_coverage", + defaultValue = "false", + category = "testing", + help = "If specified, Bazel will instrument code (using offline instrumentation where " + + "possible) and will collect coverage information during tests. Only targets that " + + " match --instrumentation_filter will be affected. Usually this option should " + + " not be specified directly - 'bazel coverage' command should be used instead." + ) + public boolean collectCodeCoverage; + + @Option(name = "microcoverage", + defaultValue = "false", + category = "testing", + help = "If specified with coverage, Blaze will collect microcoverage (per test method " + + "coverage) information during tests. Only targets that match " + + "--instrumentation_filter will be affected. Usually this option should not be " + + "specified directly - 'blaze coverage --microcoverage' command should be used " + + "instead." + ) + public boolean collectMicroCoverage; + + @Option(name = "cache_test_results", + defaultValue = "auto", + category = "testing", + abbrev = 't', // it's useful to toggle this on/off quickly + help = "If 'auto', Bazel will only rerun a test if any of the following conditions apply: " + + "(1) Bazel detects changes in the test or its dependencies " + + "(2) the test is marked as external " + + "(3) multiple test runs were requested with --runs_per_test" + + "(4) the test failed" + + "If 'yes', the caching behavior will be the same as 'auto' except that " + + "it may cache test failures and test runs with --runs_per_test." + + "If 'no', all tests will be always executed.") + public TriState cacheTestResults; + + @Deprecated + @Option(name = "test_result_expiration", + defaultValue = "-1", // No expiration by defualt. + category = "testing", + help = "This option is deprecated and has no effect.") + public int testResultExpiration; + + @Option(name = "test_sharding_strategy", + defaultValue = "explicit", + category = "testing", + converter = TestActionBuilder.ShardingStrategyConverter.class, + help = "Specify strategy for test sharding: " + + "'explicit' to only use sharding if the 'shard_count' BUILD attribute is present. " + + "'disabled' to never use test sharding. " + + "'experimental_heuristic' to enable sharding on remotely executed tests without an " + + "explicit 'shard_count' attribute which link in a supported framework. Considered " + + "experimental.") + public TestActionBuilder.TestShardingStrategy testShardingStrategy; + + @Option(name = "runs_per_test", + allowMultiple = true, + defaultValue = "1", + category = "testing", + converter = RunsPerTestConverter.class, + help = "Specifies number of times to run each test. If any of those attempts " + + "fail for any reason, the whole test would be considered failed. " + + "Normally the value specified is just an integer. Example: --runs_per_test=3 " + + "will run all tests 3 times. " + + "Alternate syntax: regex_filter@runs_per_test. Where runs_per_test stands for " + + "an integer value and regex_filter stands " + + "for a list of include and exclude regular expression patterns (Also see " + + "--instrumentation_filter). Example: " + + "--runs_per_test=//foo/.*,-//foo/bar/.*@3 runs all tests in //foo/ " + + "except those under foo/bar three times. " + + "This option can be passed multiple times. ") + public List<PerLabelOptions> runsPerTest; + + @Option(name = "build_runfile_links", + defaultValue = "true", + category = "strategy", + help = "If true, build runfiles symlink forests for all targets. " + + "If false, write only manifests when possible.") + public boolean buildRunfiles; + + @Option(name = "test_arg", + allowMultiple = true, + defaultValue = "", + category = "testing", + help = "Specifies additional options and arguments that should be passed to the test " + + "executable. Can be used multiple times to specify several arguments. " + + "If multiple tests are executed, each of them will receive identical arguments. " + + "Used only by the 'bazel test' command." + ) + public List<String> testArguments; + + @Option(name = "test_filter", + allowMultiple = false, + defaultValue = "null", + category = "testing", + help = "Specifies a filter to forward to the test framework. Used to limit " + + "the tests run. Note that this does not affect which targets are built.") + public String testFilter; + + @Option(name = "check_fileset_dependencies_recursively", + defaultValue = "true", + category = "semantics", + help = "If false, fileset targets will, whenever possible, create " + + "symlinks to directories instead of creating one symlink for each " + + "file inside the directory. Disabling this will significantly " + + "speed up fileset builds, but targets that depend on filesets will " + + "not be rebuilt if files are added, removed or modified in a " + + "subdirectory which has not been traversed.") + public boolean checkFilesetDependenciesRecursively; + + @Option(name = "run_under", + category = "run", + defaultValue = "null", + converter = RunUnderConverter.class, + help = "Prefix to insert in front of command before running. " + + "Examples:\n" + + "\t--run_under=valgrind\n" + + "\t--run_under=strace\n" + + "\t--run_under='strace -c'\n" + + "\t--run_under='valgrind --quiet --num-callers=20'\n" + + "\t--run_under=//package:target\n" + + "\t--run_under='//package:target --options'\n") + public RunUnder runUnder; + + @Option(name = "distinct_host_configuration", + defaultValue = "true", + category = "strategy", + help = "Build all the tools used during the build for a distinct configuration from " + + "that used for the target program. By default, the same configuration is used " + + "for host and target programs, but this may cause undesirable rebuilds of tool " + + "such as the protocol compiler (and then everything downstream) whenever a minor " + + "change is made to the target configuration, such as setting the linker options. " + + "When this flag is specified, a distinct configuration will be used to build the " + + "tools, preventing undesired rebuilds. However, certain libraries will then " + + "need to be compiled twice, once for each configuration, which may cause some " + + "builds to be slower. As a rule of thumb, this option is likely to benefit " + + "users that make frequent changes in configuration (e.g. opt/dbg). " + + "Please read the user manual for the full explanation.") + public boolean useDistinctHostConfiguration; + + @Option(name = "check_visibility", + defaultValue = "true", + category = "checking", + help = "If disabled, visibility errors are demoted to warnings.") + public boolean checkVisibility; + + // Moved from viewOptions to here because license information is very expensive to serialize. + // Having it here allows us to skip computation of transitive license information completely + // when the setting is disabled. + @Option(name = "check_licenses", + defaultValue = "false", + category = "checking", + help = "Check that licensing constraints imposed by dependent packages " + + "do not conflict with distribution modes of the targets being built. " + + "By default, licenses are not checked.") + public boolean checkLicenses; + + @Option(name = "experimental_enforce_constraints", + defaultValue = "true", + category = "undocumented", + help = "Checks the environments each target is compatible with and reports errors if any " + + "target has dependencies that don't support the same environments") + public boolean enforceConstraints; + + @Option(name = "experimental_action_listener", + allowMultiple = true, + defaultValue = "", + category = "experimental", + converter = LabelConverter.class, + help = "Use action_listener to attach an extra_action to existing build actions.") + public List<Label> actionListeners; + + @Option(name = "is host configuration", + defaultValue = "false", + category = "undocumented", + help = "Shows whether these options are set for host configuration.") + public boolean isHost; + + @Option(name = "experimental_proto_header_modules", + defaultValue = "false", + category = "undocumented", + help = "Enables compilation of C++ header modules for proto libraries.") + public boolean protoHeaderModules; + + @Option(name = "features", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "The given features will be enabled or disabled by default for all packages. " + + "Specifying -<feature> will disable the feature globally. " + + "Negative features always override positive ones. " + + "This flag is used to enable rolling out default feature changes without a " + + "Blaze release.") + public List<String> defaultFeatures; + + @Override + public FragmentOptions getHost(boolean fallback) { + Options host = (Options) getDefault(); + + host.shortName = "host"; + host.compilationMode = CompilationMode.OPT; + host.isHost = true; + + if (fallback) { + // In the fallback case, we have already tried the target options and they didn't work, so + // now we try the default options; the hostCpu field has the default value, because we use + // getDefault() above. + host.cpu = computeHostCpu(host.hostCpu); + } else { + host.cpu = computeHostCpu(hostCpu); + } + + // === Runfiles === + // Ideally we could force this the other way, and skip runfiles construction + // for host tools which are never run locally, but that's probably a very + // small optimization. + host.buildRunfiles = true; + + // === Linkstamping === + // Disable all link stamping for the host configuration, to improve action + // cache hit rates for tools. + host.stampBinaries = false; + + // === Visibility === + host.checkVisibility = checkVisibility; + + // === Licenses === + host.checkLicenses = checkLicenses; + + // === Allow runtime_deps to depend on neverlink Java libraries. + host.allowRuntimeDepsOnNeverLink = allowRuntimeDepsOnNeverLink; + + // === Pass on C++ compiler features. + host.defaultFeatures = ImmutableList.copyOf(defaultFeatures); + + return host; + } + + private static String computeHostCpu(String explicitHostCpu) { + if (explicitHostCpu != null) { + return explicitHostCpu; + } + switch (OS.getCurrent()) { + case DARWIN: + return "darwin"; + default: + return "k8"; + } + } + + @Override + public void addAllLabels(Multimap<String, Label> labelMap) { + labelMap.putAll("action_listener", actionListeners); + labelMap.putAll("plugins", pluginList); + if ((runUnder != null) && (runUnder.getLabel() != null)) { + labelMap.put("RunUnder", runUnder.getLabel()); + } + } + } + + /** + * A list of build configurations that only contains the null element. + */ + private static final List<BuildConfiguration> NULL_LIST = + Collections.unmodifiableList(Arrays.asList(new BuildConfiguration[] { null })); + + private final String cacheKey; + private final String shortCacheKey; + + private Transitions transitions; + private Set<BuildConfiguration> allReachableConfigurations; + + private final ImmutableMap<Class<? extends Fragment>, Fragment> fragments; + + // Directories in the output tree + private final Root outputDirectory; // the configuration-specific output directory. + private final Root binDirectory; + private final Root genfilesDirectory; + private final Root coverageMetadataDirectory; // for coverage-related metadata, artifacts, etc. + private final Root testLogsDirectory; + private final Root includeDirectory; + private final Root middlemanDirectory; + + private final PathFragment binFragment; + private final PathFragment genfilesFragment; + + // If false, AnalysisEnviroment doesn't register any actions created by the ConfiguredTarget. + private final boolean actionsEnabled; + + private final ImmutableSet<Label> coverageLabels; + private final ImmutableSet<Label> coverageReportGeneratorLabels; + + // Executables like "perl" or "sh" + private final ImmutableMap<String, PathFragment> executables; + + // All the "defglobals" in //tools:GLOBALS for this platform/configuration: + private final ImmutableMap<String, String> globalMakeEnv; + + private final ImmutableMap<String, String> defaultShellEnvironment; + private final BuildOptions buildOptions; + private final Options options; + + private final String shortName; + private final String mnemonic; + private final String platformName; + + /** + * It is not fingerprinted because it should only be used to access + * variables that do not break the hermetism of build rules. + */ + private final ImmutableMap<String, String> clientEnvironment; + + /** + * Helper container for {@link #transitiveOptionsMap} below. + */ + private static class OptionDetails implements Serializable { + private OptionDetails(Class<? extends OptionsBase> optionsClass, Object value, + boolean allowsMultiple) { + this.optionsClass = optionsClass; + this.value = value; + this.allowsMultiple = allowsMultiple; + } + + /** The {@link FragmentOptions} class that defines this option. */ + private final Class<? extends OptionsBase> optionsClass; + + /** + * The value of the given option (either explicitly defined or default). May be null. + */ + private final Object value; + + /** Whether or not this option supports multiple values. */ + private final boolean allowsMultiple; + } + + /** + * Maps option names to the {@link OptionDetails} the option takes for this configuration. + * + * <p>This can be used to: + * <ol> + * <li>Find an option's (parsed) value given its command-line name</li> + * <li>Parse alternative values for the option.</li> + * </ol> + * + * <p>This map is "transitive" in that it includes *all* options recognizable by this + * configuration, including those defined in child fragments. + */ + private final Map<String, OptionDetails> transitiveOptionsMap; + + + /** + * Validates the options for this BuildConfiguration. Issues warnings for the + * use of deprecated options, and warnings or errors for any option settings + * that conflict. + */ + public void reportInvalidOptions(EventHandler reporter) { + for (Fragment fragment : fragments.values()) { + fragment.reportInvalidOptions(reporter, this.buildOptions); + } + + Set<String> plugins = new HashSet<>(); + for (Label plugin : options.pluginList) { + String name = plugin.getName(); + if (plugins.contains(name)) { + reporter.handle(Event.error("A build cannot have two plugins with the same name")); + } + plugins.add(name); + } + for (Map.Entry<String, String> opt : options.pluginCoptList) { + if (!plugins.contains(opt.getKey())) { + reporter.handle(Event.error("A plugin_copt must refer to an existing plugin")); + } + } + + if (options.shortName != null) { + reporter.handle(Event.error( + "The internal '--configuration short name' option cannot be used on the command line")); + } + + if (options.testShardingStrategy + == TestActionBuilder.TestShardingStrategy.EXPERIMENTAL_HEURISTIC) { + reporter.handle(Event.warn( + "Heuristic sharding is intended as a one-off experimentation tool for determing the " + + "benefit from sharding certain tests. Please don't keep this option in your " + + ".blazerc or continuous build")); + } + } + + private ImmutableMap<String, String> setupShellEnvironment() { + ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>(); + for (Fragment fragment : fragments.values()) { + fragment.setupShellEnvironment(builder); + } + return builder.build(); + } + + BuildConfiguration(BlazeDirectories directories, + Map<Class<? extends Fragment>, Fragment> fragmentsMap, + BuildOptions buildOptions, + Map<String, String> clientEnv, + boolean actionsDisabled) { + this.actionsEnabled = !actionsDisabled; + fragments = ImmutableMap.copyOf(fragmentsMap); + + // This is a view that will be updated upon each client command. + this.clientEnvironment = ImmutableMap.copyOf(clientEnv); + + this.buildOptions = buildOptions; + this.options = buildOptions.get(Options.class); + + this.mnemonic = buildMnemonic(); + String outputDirName = (options.shortName != null) ? options.shortName : mnemonic; + this.shortName = buildShortName(outputDirName); + + this.executables = collectExecutables(); + + Path execRoot = directories.getExecRoot(); + // configuration-specific output tree + Path outputDir = directories.getOutputPath().getRelative(outputDirName); + this.outputDirectory = Root.asDerivedRoot(execRoot, outputDir); + + // specific subdirs under outputDirectory + this.binDirectory = Root.asDerivedRoot(execRoot, outputDir.getRelative("bin")); + this.genfilesDirectory = Root.asDerivedRoot(execRoot, outputDir.getRelative("genfiles")); + this.coverageMetadataDirectory = Root.asDerivedRoot(execRoot, + outputDir.getRelative("coverage-metadata")); + this.testLogsDirectory = Root.asDerivedRoot(execRoot, outputDir.getRelative("testlogs")); + this.includeDirectory = Root.asDerivedRoot(execRoot, + outputDir.getRelative(BlazeDirectories.RELATIVE_INCLUDE_DIR)); + this.middlemanDirectory = Root.middlemanRoot(execRoot, outputDir); + + // precompute some frequently-used relative paths + this.binFragment = getBinDirectory().getExecPath(); + this.genfilesFragment = getGenfilesDirectory().getExecPath(); + + ImmutableSet.Builder<Label> coverageLabelsBuilder = ImmutableSet.builder(); + ImmutableSet.Builder<Label> coverageReportGeneratorLabelsBuilder = ImmutableSet.builder(); + for (Fragment fragment : fragments.values()) { + coverageLabelsBuilder.addAll(fragment.getCoverageLabels()); + coverageReportGeneratorLabelsBuilder.addAll(fragment.getCoverageReportGeneratorLabels()); + } + this.coverageLabels = coverageLabelsBuilder.build(); + this.coverageReportGeneratorLabels = coverageReportGeneratorLabelsBuilder.build(); + + // Platform name + StringBuilder platformNameBuilder = new StringBuilder(); + for (Fragment fragment : fragments.values()) { + platformNameBuilder.append(fragment.getPlatformName()); + } + this.platformName = platformNameBuilder.toString(); + + this.defaultShellEnvironment = setupShellEnvironment(); + + this.transitiveOptionsMap = computeOptionsMap(buildOptions, fragments.values()); + + ImmutableMap.Builder<String, String> globalMakeEnvBuilder = ImmutableMap.builder(); + for (Fragment fragment : fragments.values()) { + fragment.addGlobalMakeVariables(globalMakeEnvBuilder); + } + + // Lots of packages in third_party assume that BINMODE expands to either "-dbg", or "-opt". So + // for backwards compatibility we preserve that invariant, setting BINMODE to "-dbg" rather than + // "-fastbuild" if the compilation mode is "fastbuild". + // We put the real compilation mode in a new variable COMPILATION_MODE. + globalMakeEnvBuilder.put("COMPILATION_MODE", options.compilationMode.toString()); + globalMakeEnvBuilder.put("BINMODE", "-" + + ((options.compilationMode == CompilationMode.FASTBUILD) + ? "dbg" + : options.compilationMode.toString())); + /* + * Attention! Document these in the build-encyclopedia + */ + // the bin directory and the genfiles directory + // These variables will be used on Windows as well, so we need to make sure + // that paths use the correct system file-separator. + globalMakeEnvBuilder.put("BINDIR", binFragment.getPathString()); + globalMakeEnvBuilder.put("INCDIR", + getIncludeDirectory().getExecPath().getPathString()); + globalMakeEnvBuilder.put("GENDIR", genfilesFragment.getPathString()); + globalMakeEnv = globalMakeEnvBuilder.build(); + + cacheKey = computeCacheKey( + directories, fragmentsMap, this.buildOptions, this.clientEnvironment); + shortCacheKey = shortName + "-" + Fingerprint.md5Digest(cacheKey); + } + + + /** + * Computes and returns the transitive optionName -> "option info" map for + * this configuration. + */ + private static Map<String, OptionDetails> computeOptionsMap(BuildOptions buildOptions, + Iterable<Fragment> fragments) { + // Collect from our fragments "alternative defaults" for options where the default + // should be something other than what's specified in Option.defaultValue. + Map<String, Object> lateBoundDefaults = Maps.newHashMap(); + for (Fragment fragment : fragments) { + lateBoundDefaults.putAll(fragment.lateBoundOptionDefaults()); + } + + ImmutableMap.Builder<String, OptionDetails> map = ImmutableMap.builder(); + try { + for (FragmentOptions options : buildOptions.getOptions()) { + for (Field field : options.getClass().getFields()) { + if (field.isAnnotationPresent(Option.class)) { + Option option = field.getAnnotation(Option.class); + Object value = field.get(options); + if (value == null) { + if (lateBoundDefaults.containsKey(option.name())) { + value = lateBoundDefaults.get(option.name()); + } else if (!option.defaultValue().equals("null")) { + // See {@link Option#defaultValue} for an explanation of default "null" strings. + value = option.defaultValue(); + } + } + map.put(option.name(), + new OptionDetails(options.getClass(), value, option.allowMultiple())); + } + } + } + } catch (IllegalAccessException e) { + throw new IllegalStateException( + "Unexpected illegal access trying to create this configuration's options map: ", e); + } + return map.build(); + } + + private String buildShortName(String outputDirName) { + ArrayList<String> nameParts = new ArrayList<>(ImmutableList.of(outputDirName)); + for (Fragment fragment : fragments.values()) { + nameParts.add(fragment.getConfigurationNameSuffix()); + } + return Joiner.on('-').skipNulls().join(nameParts); + } + + private String buildMnemonic() { + // See explanation at getShortName(). + String platformSuffix = (options.platformSuffix != null) ? options.platformSuffix : ""; + ArrayList<String> nameParts = new ArrayList<String>(); + for (Fragment fragment : fragments.values()) { + nameParts.add(fragment.getOutputDirectoryName()); + } + nameParts.add(getCompilationMode() + platformSuffix); + return Joiner.on('-').join(Iterables.filter(nameParts, Predicates.notNull())); + } + + /** + * Set the outgoing configuration transitions. During the lifetime of a given build configuration, + * this must happen exactly once, shortly after the configuration is created. + * TODO(bazel-team): this makes the object mutable, get rid of it. + */ + public void setConfigurationTransitions(Transitions transitions) { + Preconditions.checkNotNull(transitions); + Preconditions.checkState(this.transitions == null); + this.transitions = transitions; + } + + public Transitions getTransitions() { + Preconditions.checkState(this.transitions != null || isHostConfiguration()); + return transitions; + } + + /** + * Returns all configurations that can be reached from this configuration through any kind of + * configuration transition. + */ + public synchronized Collection<BuildConfiguration> getAllReachableConfigurations() { + if (allReachableConfigurations == null) { + // This is needed for every configured target in skyframe m2, so we cache it. + // We could alternatively make the corresponding dependencies into a skyframe node. + this.allReachableConfigurations = computeAllReachableConfigurations(); + } + return allReachableConfigurations; + } + + /** + * Returns all configurations that can be reached from this configuration through any kind of + * configuration transition. + */ + private Set<BuildConfiguration> computeAllReachableConfigurations() { + Set<BuildConfiguration> result = new LinkedHashSet<>(); + Queue<BuildConfiguration> queue = new LinkedList<>(); + queue.add(this); + while (!queue.isEmpty()) { + BuildConfiguration config = queue.remove(); + if (!result.add(config)) { + continue; + } + config.getTransitions().addDirectlyReachableConfigurations(queue); + } + return result; + } + + /** + * Returns the new configuration after traversing a dependency edge with a given configuration + * transition. + * + * @param transition the configuration transition + * @return the new configuration + * @throws IllegalArgumentException if the transition is a {@link SplitTransition} + */ + public BuildConfiguration getConfiguration(Transition transition) { + Preconditions.checkArgument(!(transition instanceof SplitTransition)); + return transitions.getConfiguration(transition); + } + + /** + * Returns the new configurations after traversing a dependency edge with a given split + * transition. + * + * @param transition the split configuration transition + * @return the new configurations + */ + public List<BuildConfiguration> getSplitConfigurations(SplitTransition<?> transition) { + return transitions.getSplitConfigurations(transition); + } + + /** + * Calculates the configurations of a direct dependency. If a rule in some BUILD file refers + * to a target (like another rule or a source file) using a label attribute, that target needs + * to have a configuration, too. This method figures out the proper configuration for the + * dependency. + * + * @param fromRule the rule that's depending on some target + * @param attribute the attribute using which the rule depends on that target (eg. "srcs") + * @param toTarget the target that's dependeded on + * @return the configuration that should be associated to {@code toTarget} + */ + public Iterable<BuildConfiguration> evaluateTransition(final Rule fromRule, + final Attribute attribute, final Target toTarget) { + // Fantastic configurations and where to find them: + + // I. Input files and package groups have no configurations. We don't want to duplicate them. + if (toTarget instanceof InputFile || toTarget instanceof PackageGroup) { + return NULL_LIST; + } + + // II. Host configurations never switch to another. All prerequisites of host targets have the + // same host configuration. + if (isHostConfiguration()) { + return ImmutableList.of(this); + } + + // Make sure config_setting dependencies are resolved in the referencing rule's configuration, + // unconditionally. For example, given: + // + // genrule( + // name = 'myrule', + // tools = select({ '//a:condition': [':sometool'] }) + // + // all labels in "tools" get resolved in the host configuration (since the "tools" attribute + // declares a host configuration transition). We want to explicitly exclude configuration labels + // from these transitions, since their *purpose* is to do computation on the owning + // rule's configuration. + // TODO(bazel-team): implement this more elegantly. This is far too hackish. Specifically: + // don't reference the rule name explicitly and don't require special-casing here. + if (toTarget instanceof Rule && ((Rule) toTarget).getRuleClass().equals("config_setting")) { + return ImmutableList.of(this); + } + + List<BuildConfiguration> toConfigurations; + if (attribute.getConfigurationTransition() instanceof SplitTransition) { + Preconditions.checkState(attribute.getConfigurator() == null); + toConfigurations = getSplitConfigurations( + (SplitTransition<?>) attribute.getConfigurationTransition()); + } else { + // III. Attributes determine configurations. The configuration of a prerequisite is determined + // by the attribute. + @SuppressWarnings("unchecked") + Configurator<BuildConfiguration, Rule> configurator = + (Configurator<BuildConfiguration, Rule>) attribute.getConfigurator(); + toConfigurations = ImmutableList.of((configurator != null) + ? configurator.apply(fromRule, this, attribute, toTarget) + : getConfiguration(attribute.getConfigurationTransition())); + } + + return Iterables.transform(toConfigurations, + new Function<BuildConfiguration, BuildConfiguration>() { + @Override + public BuildConfiguration apply(BuildConfiguration input) { + // IV. Allow the transition object to perform an arbitrary switch. Blaze modules can inject + // configuration transition logic by extending the Transitions class. + BuildConfiguration actual = getTransitions().configurationHook( + fromRule, attribute, toTarget, input); + + // V. Allow rule classes to override their own configurations. + Rule associatedRule = toTarget.getAssociatedRule(); + if (associatedRule != null) { + @SuppressWarnings("unchecked") + RuleClass.Configurator<BuildConfiguration, Rule> func = + associatedRule.getRuleClassObject().<BuildConfiguration, Rule>getConfigurator(); + actual = func.apply(associatedRule, actual); + } + + return actual; + } + }); + } + + /** + * Returns a multimap of all labels that should be implicitly loaded from labels that were + * specified as options, keyed by the name to be displayed to the user if something goes wrong. + * The returned set only contains labels that were derived from command-line options; the + * intention is that it can be used to sanity-check that the command-line options actually contain + * these in their transitive closure. + */ + public ListMultimap<String, Label> getImplicitLabels() { + ListMultimap<String, Label> implicitLabels = ArrayListMultimap.create(); + for (Fragment fragment : fragments.values()) { + fragment.addImplicitLabels(implicitLabels); + } + return implicitLabels; + } + + /** + * For an given environment, returns a subset containing all + * variables in the given list if they are defined in the given + * environment. + */ + @VisibleForTesting + static Map<String, String> getMapping(List<String> variables, + Map<String, String> environment) { + Map<String, String> result = new HashMap<>(); + for (String var : variables) { + if (environment.containsKey(var)) { + result.put(var, environment.get(var)); + } + } + return result; + } + + /** + * Avoid this method. The client environment is not part of the configuration's signature, so + * calls to this method introduce a non-hermetic access to data that is not visible to Skyframe. + * + * @return an unmodifiable view of the bazel client's environment + * upon its most recent request. + */ + // TODO(bazel-team): Remove this. + public Map<String, String> getClientEnv() { + return clientEnvironment; + } + + /** + * Returns the {@link Option} class the defines the given option, null if the + * option isn't recognized. + * + * <p>optionName is the name of the option as it appears on the command line + * e.g. {@link Option#name}). + */ + Class<? extends OptionsBase> getOptionClass(String optionName) { + OptionDetails optionData = transitiveOptionsMap.get(optionName); + return optionData == null ? null : optionData.optionsClass; + } + + /** + * Returns the value of the specified option for this configuration or null if the + * option isn't recognized. Since an option's legitimate value could be null, use + * {@link #getOptionClass} to distinguish between that and an unknown option. + * + * <p>optionName is the name of the option as it appears on the command line + * e.g. {@link Option#name}). + */ + Object getOptionValue(String optionName) { + OptionDetails optionData = transitiveOptionsMap.get(optionName); + return (optionData == null) ? null : optionData.value; + } + + /** + * Returns whether or not the given option supports multiple values at the command line (e.g. + * "--myoption value1 --myOption value2 ..."). Returns false for unrecognized options. Use + * {@link #getOptionClass} to distinguish between those and legitimate single-value options. + * + * <p>As declared in {@link Option#allowMultiple}, multi-value options are expected to be + * of type {@code List<T>}. + */ + boolean allowsMultipleValues(String optionName) { + OptionDetails optionData = transitiveOptionsMap.get(optionName); + return (optionData == null) ? false : optionData.allowsMultiple; + } + + /** + * The platform string, suitable for use as a key into a MakeEnvironment. + */ + public String getPlatformName() { + return platformName; + } + + /** + * Returns the output directory for this build configuration. + */ + public Root getOutputDirectory() { + return outputDirectory; + } + + /** + * Returns the bin directory for this build configuration. + */ + @SkylarkCallable(name = "bin_dir", structField = true, + doc = "The root corresponding to bin directory.") + public Root getBinDirectory() { + return binDirectory; + } + + /** + * Returns a relative path to the bin directory at execution time. + */ + public PathFragment getBinFragment() { + return binFragment; + } + + /** + * Returns the include directory for this build configuration. + */ + public Root getIncludeDirectory() { + return includeDirectory; + } + + /** + * Returns the genfiles directory for this build configuration. + */ + @SkylarkCallable(name = "genfiles_dir", structField = true, + doc = "The root corresponding to genfiles directory.") + public Root getGenfilesDirectory() { + return genfilesDirectory; + } + + /** + * Returns the directory where coverage-related artifacts and metadata files + * should be stored. This includes for example uninstrumented class files + * needed for Jacoco's coverage reporting tools. + */ + public Root getCoverageMetadataDirectory() { + return coverageMetadataDirectory; + } + + /** + * Returns the testlogs directory for this build configuration. + */ + public Root getTestLogsDirectory() { + return testLogsDirectory; + } + + /** + * Returns a relative path to the genfiles directory at execution time. + */ + public PathFragment getGenfilesFragment() { + return genfilesFragment; + } + + /** + * Returns the path separator for the host platform. This is basically the same as {@link + * java.io.File#pathSeparator}, except that that returns the value for this JVM, which may or may + * not match the host platform. You should only use this when invoking tools that are known to use + * the native path separator, i.e., the path separator for the machine that they run on. + */ + @SkylarkCallable(name = "host_path_separator", structField = true, + doc = "Returns the separator for PATH variable, which is ':' on Unix.") + public String getHostPathSeparator() { + // TODO(bazel-team): This needs to change when we support Windows. + return ":"; + } + + /** + * Returns the internal directory (used for middlemen) for this build configuration. + */ + public Root getMiddlemanDirectory() { + return middlemanDirectory; + } + + public boolean getAllowRuntimeDepsOnNeverLink() { + return options.allowRuntimeDepsOnNeverLink; + } + + public boolean isStrictFilesets() { + return options.strictFilesets; + } + + public List<Label> getPlugins() { + return options.pluginList; + } + + public List<Map.Entry<String, String>> getPluginCopts() { + return options.pluginCoptList; + } + + /** + * Implements a non-injective mapping from BuildConfiguration instances to + * strings. The result should identify the aspects of the configuration + * that should be reflected in the output file names. Furthermore the + * returned string must not contain shell metacharacters. + * + * <p>The intention here is that we use this string as the directory name + * for artifacts of this build. + * + * <p>For configuration settings which are NOT part of the short name, + * rebuilding with a different value of such a setting will build in + * the same output directory. This means that any actions whose + * keys (see Action.getKey()) have changed will be rerun. That + * may result in a lot of recompilation. + * + * <p>For configuration settings which ARE part of the short name, + * rebuilding with a different value of such a setting will rebuild + * in a different output directory; this will result in higher disk + * usage and more work the _first_ time you rebuild with a different + * setting, but will result in less work if you regularly switch + * back and forth between different settings. + * + * <p>With one important exception, it's sound to choose any subset of the + * config's components for this string, it just alters the dimensionality + * of the cache. In other words, it's a trade-off on the "injectiveness" + * scale: at one extreme (shortName is in fact a complete fingerprint, and + * thus injective) you get extremely precise caching (no competition for the + * same output-file locations) but you have to rebuild for even the + * slightest change in configuration. At the other extreme + * (PartialFingerprint is a constant) you have very high competition for + * output-file locations, but if a slight change in configuration doesn't + * affect a particular build step, you're guaranteed not to have to + * rebuild it. The important exception has to do with cross-compilation: + * the host and target configurations must not map to the same output + * directory, because then files would need to get built for the host + * and then rebuilt for the target even within a single build, and that + * wouldn't work. + * + * <p>Just to re-iterate: cross-compilation builds (i.e. hostConfig != + * targetConfig) will not work if the two configurations' short names are + * equal. This is an important practical case: the mere addition of + * a compile flag to the target configuration would cause the build to + * fail. In other words, it would break if the host and target + * configurations are not identical but are "too close". The current + * solution is to set the host configuration equal to the target + * configuration if they are "too close"; this may cause the tools to get + * rebuild for the new host configuration though. + */ + public String getShortName() { + return shortName; + } + + /** + * Like getShortName(), but always returns a configuration-dependent string even for + * the host configuration. + */ + public String getMnemonic() { + return mnemonic; + } + + @Override + public String toString() { + return getShortName(); + } + + /** + * Returns the default shell environment + */ + @SkylarkCallable(name = "default_shell_env", structField = true, + doc = "A dictionary representing the default environment. It maps variables " + + "to their values (strings).") + public ImmutableMap<String, String> getDefaultShellEnvironment() { + return defaultShellEnvironment; + } + + /** + * Returns the path to sh. + */ + public PathFragment getShExecutable() { + return executables.get("sh"); + } + + /** + * Returns a regex-based instrumentation filter instance that used to match label + * names to identify targets to be instrumented in the coverage mode. + */ + public RegexFilter getInstrumentationFilter() { + return options.instrumentationFilter; + } + + /** + * Returns the set of labels for coverage. + */ + public Set<Label> getCoverageLabels() { + return coverageLabels; + } + + /** + * Returns the set of labels for the coverage report generator. + */ + public Set<Label> getCoverageReportGeneratorLabels() { + return coverageReportGeneratorLabels; + } + + /** + * Returns true if bazel should show analyses results, even if it did not + * re-run the analysis. + */ + public boolean showCachedAnalysisResults() { + return options.showCachedAnalysisResults; + } + + /** + * Returns a new, unordered mapping of names to values of "Make" variables defined by this + * configuration. + * + * <p>This does *not* include package-defined overrides (e.g. vardef) + * and so should not be used by the build logic. This is used only for + * the 'info' command. + * + * <p>Command-line definitions of make enviroments override variables defined by + * {@code Fragment.addGlobalMakeVariables()}. + */ + public Map<String, String> getMakeEnvironment() { + Map<String, String> makeEnvironment = new HashMap<>(); + makeEnvironment.putAll(globalMakeEnv); + for (Fragment fragment : fragments.values()) { + makeEnvironment.putAll(fragment.getCommandLineDefines()); + } + return ImmutableMap.copyOf(makeEnvironment); + } + + /** + * Returns a new, unordered mapping of names that are set through the command lines. + * (Fragments, in particular the Google C++ support, can set variables through the + * command line.) + */ + public Map<String, String> getCommandLineDefines() { + ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); + for (Fragment fragment : fragments.values()) { + builder.putAll(fragment.getCommandLineDefines()); + } + return builder.build(); + } + + /** + * Returns the global defaults for this configuration for the Make environment. + */ + public Map<String, String> getGlobalMakeEnvironment() { + return globalMakeEnv; + } + + /** + * Returns a (key, value) mapping to insert into the subcommand environment for coverage + * actions. + */ + public Map<String, String> getCoverageEnvironment() { + Map<String, String> env = new HashMap<>(); + for (Fragment fragment : fragments.values()) { + env.putAll(fragment.getCoverageEnvironment()); + } + return env; + } + + /** + * Returns the default value for the specified "Make" variable for this + * configuration. Returns null if no value was found. + */ + public String getMakeVariableDefault(String var) { + return globalMakeEnv.get(var); + } + + /** + * Returns a configuration fragment instances of the given class. + */ + @SkylarkCallable(name = "fragment", doc = "Returns a configuration fragment using the key.") + public <T extends Fragment> T getFragment(Class<T> clazz) { + return clazz.cast(fragments.get(clazz)); + } + + /** + * Returns true if the requested configuration fragment is present. + */ + public <T extends Fragment> boolean hasFragment(Class<T> clazz) { + return getFragment(clazz) != null; + } + + /** + * Returns true if all requested configuration fragment are present (this may be slow). + */ + public boolean hasAllFragments(Set<Class<?>> fragmentClasses) { + for (Class<?> fragmentClass : fragmentClasses) { + if (!hasFragment(fragmentClass.asSubclass(Fragment.class))) { + return false; + } + } + return true; + } + + /** + * Returns true if non-functional build stamps are enabled. + */ + public boolean stampBinaries() { + return options.stampBinaries; + } + + /** + * Returns true if extended sanity checks should be enabled. + */ + public boolean extendedSanityChecks() { + return options.extendedSanityChecks; + } + + /** + * Returns true if we are building runfiles symlinks for this configuration. + */ + public boolean buildRunfiles() { + return options.buildRunfiles; + } + + public boolean getCheckFilesetDependenciesRecursively() { + return options.checkFilesetDependenciesRecursively; + } + + public List<String> getTestArguments() { + return options.testArguments; + } + + public String getTestFilter() { + return options.testFilter; + } + + /** + * Returns user-specified test environment variables and their values, as + * set by the --test_env options. + */ + public Map<String, String> getTestEnv() { + return getTestEnv(options.testEnvironment, clientEnvironment); + } + + /** + * Returns user-specified test environment variables and their values, as + * set by the --test_env options. + * + * @param envOverrides The --test_env flag values. + * @param clientEnvironment The full client environment. + */ + public static Map<String, String> getTestEnv(List<Map.Entry<String, String>> envOverrides, + Map<String, String> clientEnvironment) { + Map<String, String> testEnv = new HashMap<>(); + for (Map.Entry<String, String> var : envOverrides) { + if (var.getValue() != null) { + testEnv.put(var.getKey(), var.getValue()); + } else { + String value = clientEnvironment.get(var.getKey()); + if (value != null) { + testEnv.put(var.getKey(), value); + } + } + } + return testEnv; + } + + public TriState cacheTestResults() { + return options.cacheTestResults; + } + + public int getMinParamFileSize() { + return options.minParamFileSize; + } + + @SkylarkCallable(name = "coverage_enabled", structField = true, + doc = "A boolean that tells whether code coverage is enabled.") + public boolean isCodeCoverageEnabled() { + return options.collectCodeCoverage; + } + + public boolean isMicroCoverageEnabled() { + return options.collectMicroCoverage; + } + + public boolean isActionsEnabled() { + return actionsEnabled; + } + + public TestActionBuilder.TestShardingStrategy testShardingStrategy() { + return options.testShardingStrategy; + } + + /** + * @return number of times the given test should run. + * If the test doesn't match any of the filters, runs it once. + */ + public int getRunsPerTestForLabel(Label label) { + for (PerLabelOptions perLabelRuns : options.runsPerTest) { + if (perLabelRuns.isIncluded(label)) { + return Integer.parseInt(Iterables.getOnlyElement(perLabelRuns.getOptions())); + } + } + return 1; + } + + public RunUnder getRunUnder() { + return options.runUnder; + } + + /** + * Returns true if this is a host configuration. + */ + public boolean isHostConfiguration() { + return options.isHost; + } + + public boolean checkVisibility() { + return options.checkVisibility; + } + + public boolean checkLicenses() { + return options.checkLicenses; + } + + public boolean enforceConstraints() { + return options.enforceConstraints; + } + + public List<Label> getActionListeners() { + return actionsEnabled ? options.actionListeners : ImmutableList.<Label>of(); + } + + /** + * Returns compilation mode. + */ + public CompilationMode getCompilationMode() { + return options.compilationMode; + } + + /** + * Helper method to create a key from the BuildConfiguration initialization + * parameters and any additional component suppliers. + */ + static String computeCacheKey(BlazeDirectories directories, + Map<Class<? extends Fragment>, Fragment> fragments, + BuildOptions buildOptions, Map<String, String> clientEnv) { + + // Creates a full fingerprint of all constructor parameters, used for + // canonicalization. + // + // Note the use of each Path's FileSystem field; the test suite creates + // many paths of equal name but belonging to distinct filesystems, so we + // have to detect this. (Note however that we're relying on the + // injectiveness of identityHashCode for FileSystem, which is inelegant, + // but only affects the tests, since the production code uses only one + // instance.) + + ImmutableList.Builder<String> keys = ImmutableList.builder(); + + // NOTE: identityHashCode isn't sound; may cause tests to fail. + keys.add(String.valueOf(System.identityHashCode(directories.getOutputBase().getFileSystem()))); + keys.add(directories.getOutputBase().toString()); + keys.add(buildOptions.computeCacheKey()); + // This is needed so that if we have --test_env=VAR, the configuration key is updated if the + // environment variable VAR is updated. + keys.add(BuildConfiguration.getTestEnv( + buildOptions.get(Options.class).testEnvironment, clientEnv).toString()); + keys.add(directories.getWorkspace().toString()); + + for (Fragment fragment : fragments.values()) { + keys.add(fragment.cacheKey()); + } + + // TODO(bazel-team): add hash of the FDO/LIPO profile file to config cache key + + return StringUtilities.combineKeys(keys.build()); + } + + /** + * Returns a string that identifies the configuration. + * + * <p>The string uniquely identifies the configuration. As a result, it can be rather long and + * include spaces and other non-alphanumeric characters. If you need a shorter key, use + * {@link #shortCacheKey()}. + * + * @see #computeCacheKey + */ + public final String cacheKey() { + return cacheKey; + } + + /** + * Returns a (relatively) short key that identifies the configuration. + * + * <p>The short key is the short name of the configuration concatenated with a hash of the + * {@link #cacheKey()}. + */ + public final String shortCacheKey() { + return shortCacheKey; + } + + /** Returns a copy of the build configuration options for this configuration. */ + public BuildOptions cloneOptions() { + return buildOptions.clone(); + } + + /** + * Prepare the fdo support. It reads data into memory that is used during analysis. The analysis + * phase is generally not allowed to perform disk I/O. This code is here because it is + * conceptually part of the analysis phase, and it needs to happen when the loading phase is + * complete. + */ + public void prepareToBuild(Path execRoot, ArtifactFactory artifactFactory, + PackageRootResolver resolver) throws ViewCreationFailedException { + for (Fragment fragment : fragments.values()) { + fragment.prepareHook(execRoot, artifactFactory, getGenfilesFragment(), resolver); + } + } + + /** + * Declares dependencies on any relevant Skyframe values (for example, relevant FileValues). + */ + public void declareSkyframeDependencies(SkyFunction.Environment env) { + for (Fragment fragment : fragments.values()) { + fragment.declareSkyframeDependencies(env); + } + } + + /** + * Returns all the roots for this configuration. + */ + public List<Root> getRoots() { + List<Root> roots = new ArrayList<>(); + + // Configuration-specific roots. + roots.add(getBinDirectory()); + roots.add(getGenfilesDirectory()); + roots.add(getIncludeDirectory()); + roots.add(getMiddlemanDirectory()); + roots.add(getTestLogsDirectory()); + + // Fragment-defined roots + for (Fragment fragment : fragments.values()) { + fragment.addRoots(roots); + } + + return ImmutableList.copyOf(roots); + } + + public ListMultimap<String, Label> getAllLabels() { + return buildOptions.getAllLabels(); + } + + public String getCpu() { + return options.cpu; + } + + /** + * Returns true is incremental builds are supported with this configuration. + */ + public boolean supportsIncrementalBuild() { + for (Fragment fragment : fragments.values()) { + if (!fragment.supportsIncrementalBuild()) { + return false; + } + } + return true; + } + + /** + * Returns true if the configuration performs static linking. + */ + public boolean performsStaticLink() { + for (Fragment fragment : fragments.values()) { + if (fragment.performsStaticLink()) { + return true; + } + } + return false; + } + + /** + * Deletes temporary directories before execution phase. This is only called for + * target configuration. + */ + public void prepareForExecutionPhase() throws IOException { + for (Fragment fragment : fragments.values()) { + fragment.prepareForExecutionPhase(); + } + } + + /** + * Collects executables defined by fragments. + */ + private ImmutableMap<String, PathFragment> collectExecutables() { + ImmutableMap.Builder<String, PathFragment> builder = new ImmutableMap.Builder<>(); + for (Fragment fragment : fragments.values()) { + fragment.defineExecutables(builder); + } + return builder.build(); + } + + /** + * See {@code BuildConfigurationCollection.Transitions.getArtifactOwnerConfiguration()}. + */ + public BuildConfiguration getArtifactOwnerConfiguration() { + return transitions.getArtifactOwnerConfiguration(); + } + + /** + * @return whether proto header modules should be built. + */ + public boolean getProtoHeaderModules() { + return options.protoHeaderModules; + } + + /** + * @return the list of default features used for all packages. + */ + public List<String> getDefaultFeatures() { + return options.defaultFeatures; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java new file mode 100644 index 0000000..e36a681 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java
@@ -0,0 +1,276 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.config; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ListMultimap; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Attribute.SplitTransition; +import com.google.devtools.build.lib.packages.Attribute.Transition; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.PackageGroup; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; + +import java.io.PrintStream; +import java.io.Serializable; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * The primary container for all main {@link BuildConfiguration} instances, + * currently "target", "data", and "host". + * + * <p>The target configuration is used for all targets specified on the command + * line. Data dependencies of targets in the target configuration use the data + * configuration instead. + * + * <p>The host configuration is used for tools that are executed during the + * build, e. g, compilers. + * + * <p>The "related" configurations are also contained in this class. + */ +@ThreadSafe +public final class BuildConfigurationCollection implements Serializable { + private final ImmutableList<BuildConfiguration> targetConfigurations; + + public BuildConfigurationCollection(List<BuildConfiguration> targetConfigurations) + throws InvalidConfigurationException { + this.targetConfigurations = ImmutableList.copyOf(targetConfigurations); + + // Except for the host configuration (which may be identical across target configs), the other + // configurations must all have different cache keys or we will end up with problems. + HashMap<String, BuildConfiguration> cacheKeyConflictDetector = new HashMap<>(); + for (BuildConfiguration config : getAllConfigurations()) { + if (cacheKeyConflictDetector.containsKey(config.cacheKey())) { + throw new InvalidConfigurationException("Conflicting configurations: " + config + " & " + + cacheKeyConflictDetector.get(config.cacheKey())); + } + cacheKeyConflictDetector.put(config.cacheKey(), config); + } + } + + /** + * Creates an empty configuration collection which will return null for everything. + */ + public BuildConfigurationCollection() { + this.targetConfigurations = ImmutableList.of(); + } + + public static BuildConfiguration configureTopLevelTarget(BuildConfiguration topLevelConfiguration, + Target toTarget) { + if (toTarget instanceof InputFile || toTarget instanceof PackageGroup) { + return null; + } + return topLevelConfiguration.getTransitions().toplevelConfigurationHook(toTarget); + } + + public ImmutableList<BuildConfiguration> getTargetConfigurations() { + return targetConfigurations; + } + + /** + * Returns all configurations that can be reached from the target configuration through any kind + * of configuration transition. + */ + public Collection<BuildConfiguration> getAllConfigurations() { + Set<BuildConfiguration> result = new LinkedHashSet<>(); + for (BuildConfiguration config : targetConfigurations) { + result.addAll(config.getAllReachableConfigurations()); + } + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof BuildConfigurationCollection)) { + return false; + } + BuildConfigurationCollection that = (BuildConfigurationCollection) obj; + return this.targetConfigurations.equals(that.targetConfigurations); + } + + @Override + public int hashCode() { + return targetConfigurations.hashCode(); + } + + /** + * Prints the configuration graph in dot format to the given print stream. This is only intended + * for debugging. + */ + public void dumpAsDotGraph(PrintStream out) { + out.println("digraph g {"); + out.println(" ratio = 0.3;"); + for (BuildConfiguration config : getAllConfigurations()) { + String from = config.shortCacheKey(); + for (Map.Entry<? extends Transition, ConfigurationHolder> entry : + config.getTransitions().getTransitionTable().entrySet()) { + BuildConfiguration toConfig = entry.getValue().getConfiguration(); + if (toConfig == config) { + continue; + } + String to = toConfig == null ? "ERROR" : toConfig.shortCacheKey(); + out.println(" \"" + from + "\" -> \"" + to + "\" [label=\"" + entry.getKey() + "\"]"); + } + } + out.println("}"); + } + + /** + * The outgoing transitions for a build configuration. + */ + public static class Transitions implements Serializable { + protected final BuildConfiguration configuration; + + /** + * Look up table for the configuration transitions, i.e., HOST, DATA, etc. + */ + private final Map<? extends Transition, ConfigurationHolder> transitionTable; + + // TODO(bazel-team): Consider merging transitionTable into this. + private final ListMultimap<? super SplitTransition<?>, BuildConfiguration> splitTransitionTable; + + public Transitions(BuildConfiguration configuration, + Map<? extends Transition, ConfigurationHolder> transitionTable, + ListMultimap<? extends SplitTransition<?>, BuildConfiguration> splitTransitionTable) { + this.configuration = configuration; + this.transitionTable = ImmutableMap.copyOf(transitionTable); + this.splitTransitionTable = ImmutableListMultimap.copyOf(splitTransitionTable); + } + + public Transitions(BuildConfiguration configuration, + Map<? extends Transition, ConfigurationHolder> transitionTable) { + this(configuration, transitionTable, + ImmutableListMultimap.<SplitTransition<?>, BuildConfiguration>of()); + } + + public Map<? extends Transition, ConfigurationHolder> getTransitionTable() { + return transitionTable; + } + + public ListMultimap<? super SplitTransition<?>, BuildConfiguration> getSplitTransitionTable() { + return splitTransitionTable; + } + + public List<BuildConfiguration> getSplitConfigurations(SplitTransition<?> transition) { + if (splitTransitionTable.containsKey(transition)) { + return splitTransitionTable.get(transition); + } else { + Preconditions.checkState(transition.defaultsToSelf()); + return ImmutableList.of(configuration); + } + } + + /** + * Adds all configurations that are directly reachable from this configuration through + * any kind of configuration transition. + */ + public void addDirectlyReachableConfigurations(Collection<BuildConfiguration> queue) { + for (ConfigurationHolder holder : transitionTable.values()) { + if (holder.configuration != null) { + queue.add(holder.configuration); + } + } + queue.addAll(splitTransitionTable.values()); + } + + /** + * Artifacts need an owner in Skyframe. By default it's the same configuration as what + * the configured target has, but it can be overridden if necessary. + * + * @return the artifact owner configuration + */ + public BuildConfiguration getArtifactOwnerConfiguration() { + return configuration; + } + + /** + * Returns the new configuration after traversing a dependency edge with a + * given configuration transition. + * + * @param configurationTransition the configuration transition + * @return the new configuration + */ + public BuildConfiguration getConfiguration(Transition configurationTransition) { + ConfigurationHolder holder = transitionTable.get(configurationTransition); + if (holder == null && configurationTransition.defaultsToSelf()) { + return configuration; + } + return holder.configuration; + } + + /** + * Arbitrary configuration transitions can be implemented by overriding this hook. + */ + @SuppressWarnings("unused") + public BuildConfiguration configurationHook(Rule fromTarget, + Attribute attribute, Target toTarget, BuildConfiguration toConfiguration) { + return toConfiguration; + } + + /** + * Associating configurations to top-level targets can be implemented by overriding this hook. + */ + @SuppressWarnings("unused") + public BuildConfiguration toplevelConfigurationHook(Target toTarget) { + return configuration; + } + } + + /** + * A holder class for {@link BuildConfiguration} instances that allows {@code null} values, + * because none of the Table implementations allow them. + */ + public static final class ConfigurationHolder implements Serializable { + private final BuildConfiguration configuration; + + public ConfigurationHolder(BuildConfiguration configuration) { + this.configuration = configuration; + } + + public BuildConfiguration getConfiguration() { + return configuration; + } + + @Override + public int hashCode() { + return configuration == null ? 0 : configuration.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof ConfigurationHolder)) { + return false; + } + return Objects.equals(configuration, ((ConfigurationHolder) o).configuration); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationKey.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationKey.java new file mode 100644 index 0000000..e8fcf34 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationKey.java
@@ -0,0 +1,92 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.config; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.ListMultimap; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * A key for the creation of {@link BuildConfigurationCollection} instances. + */ +public final class BuildConfigurationKey { + + private final BuildOptions buildOptions; + private final BlazeDirectories directories; + private final Map<String, String> clientEnv; + private final ImmutableSortedSet<String> multiCpu; + + /** + * Creates a key for the creation of {@link BuildConfigurationCollection} instances. + * + * Note that the BuildConfiguration.Options instance must not contain unresolved relative paths. + */ + public BuildConfigurationKey(BuildOptions buildOptions, BlazeDirectories directories, + Map<String, String> clientEnv, Set<String> multiCpu) { + this.buildOptions = Preconditions.checkNotNull(buildOptions); + this.directories = Preconditions.checkNotNull(directories); + this.clientEnv = ImmutableMap.copyOf(clientEnv); + this.multiCpu = ImmutableSortedSet.copyOf(multiCpu); + } + + public BuildConfigurationKey(BuildOptions buildOptions, BlazeDirectories directories, + Map<String, String> clientEnv) { + this(buildOptions, directories, clientEnv, ImmutableSet.<String>of()); + } + + public BuildOptions getBuildOptions() { + return buildOptions; + } + + public BlazeDirectories getDirectories() { + return directories; + } + + public Map<String, String> getClientEnv() { + return clientEnv; + } + + public ImmutableSortedSet<String> getMultiCpu() { + return multiCpu; + } + + public ListMultimap<String, Label> getLabelsToLoadUnconditionally() { + return buildOptions.getAllLabels(); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof BuildConfigurationKey)) { + return false; + } + BuildConfigurationKey k = (BuildConfigurationKey) o; + return buildOptions.equals(k.buildOptions) + && directories.equals(k.directories) + && clientEnv.equals(k.clientEnv) + && multiCpu.equals(k.multiCpu); + } + + @Override + public int hashCode() { + return Objects.hash(buildOptions, directories, clientEnv, multiCpu); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildOptions.java new file mode 100644 index 0000000..afe408f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildOptions.java
@@ -0,0 +1,254 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.config; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ListMultimap; +import com.google.devtools.build.lib.packages.Attribute.SplitTransition; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.common.options.Options; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsClassProvider; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +/** + * This is a collection of command-line options from all configuration fragments. Contains + * a single instance for all FragmentOptions classes provided by Blaze language modules. + */ +public final class BuildOptions implements Cloneable, Serializable { + /** + * Creates a BuildOptions object with all options set to its default value. + */ + public static BuildOptions createDefaults(Iterable<Class<? extends FragmentOptions>> options) { + Builder builder = builder(); + for (Class<? extends FragmentOptions> optionsClass : options) { + builder.add(Options.getDefaults(optionsClass)); + } + return builder.build(); + } + + /** + * This function creates a new BuildOptions instance for host. + * + * @param fallback if true, we have already tried the user specified hostCpu options + * and it didn't work, so now we try the default options instead. + */ + public BuildOptions createHostOptions(boolean fallback) { + Builder builder = builder(); + for (FragmentOptions options : fragmentOptionsMap.values()) { + builder.add(options.getHost(fallback)); + } + return builder.build(); + } + + /** + * Returns a list of potential split configuration transitions by calling {@link + * FragmentOptions#getPotentialSplitTransitions} on all the fragments. + */ + public List<SplitTransition<BuildOptions>> getPotentialSplitTransitions() { + List<SplitTransition<BuildOptions>> result = new ArrayList<>(); + for (FragmentOptions options : fragmentOptionsMap.values()) { + result.addAll(options.getPotentialSplitTransitions()); + } + return result; + } + + /** + * Creates an BuildOptions class by taking the option values from an options provider + * (eg. an OptionsParser). + */ + public static BuildOptions of(List<Class<? extends FragmentOptions>> optionsList, + OptionsClassProvider provider) { + Builder builder = builder(); + for (Class<? extends FragmentOptions> optionsClass : optionsList) { + builder.add(provider.getOptions(optionsClass)); + } + return builder.build(); + } + + /** + * Creates an BuildOptions class by taking the option values from command-line arguments + */ + @VisibleForTesting + public static BuildOptions of(List<Class<? extends FragmentOptions>> optionsList, String... args) + throws OptionsParsingException { + Builder builder = builder(); + OptionsParser parser = OptionsParser.newOptionsParser( + ImmutableList.<Class<? extends OptionsBase>>copyOf(optionsList)); + parser.parse(args); + for (Class<? extends FragmentOptions> optionsClass : optionsList) { + builder.add(parser.getOptions(optionsClass)); + } + return builder.build(); + } + + /** + * Returns the actual instance of a FragmentOptions class. + */ + @SuppressWarnings("unchecked") + public <T extends FragmentOptions> T get(Class<T> optionsClass) { + FragmentOptions options = fragmentOptionsMap.get(optionsClass); + Preconditions.checkNotNull(options); + Preconditions.checkArgument(optionsClass.isAssignableFrom(options.getClass())); + return (T) options; + } + + /** + * Returns a multimap of all labels that were specified as options, keyed by the name to be + * displayed to the user if something goes wrong. This should be the set of all labels + * mentioned in explicit command line options that are not already covered by the + * tools/defaults package (see the DefaultsPackage class), and nothing else. + */ + public ListMultimap<String, Label> getAllLabels() { + ListMultimap<String, Label> labels = ArrayListMultimap.create(); + for (FragmentOptions optionsBase : fragmentOptionsMap.values()) { + optionsBase.addAllLabels(labels); + } + return labels; + } + + // It would be very convenient to use a Multimap here, but we cannot do that because we need to + // support defaults labels that have zero elements. + ImmutableMap<String, ImmutableSet<Label>> getDefaultsLabels() { + BuildConfiguration.Options opts = get(BuildConfiguration.Options.class); + Map<String, Set<Label>> collector = new TreeMap<>(); + for (FragmentOptions fragment : fragmentOptionsMap.values()) { + for (Map.Entry<String, Set<Label>> entry : fragment.getDefaultsLabels(opts).entrySet()) { + if (!collector.containsKey(entry.getKey())) { + collector.put(entry.getKey(), new TreeSet<Label>()); + } + collector.get(entry.getKey()).addAll(entry.getValue()); + } + } + + ImmutableMap.Builder<String, ImmutableSet<Label>> result = new ImmutableMap.Builder<>(); + for (Map.Entry<String, Set<Label>> entry : collector.entrySet()) { + result.put(entry.getKey(), ImmutableSet.copyOf(entry.getValue())); + } + + return result.build(); + } + + /** + * The cache key for the options collection. Recomputes cache key every time it's called. + */ + public String computeCacheKey() { + StringBuilder keyBuilder = new StringBuilder(); + for (FragmentOptions options : fragmentOptionsMap.values()) { + keyBuilder.append(options.cacheKey()); + } + return keyBuilder.toString(); + } + + /** + * String representation of build options. + */ + @Override + public String toString() { + StringBuilder stringBuilder = new StringBuilder(); + for (FragmentOptions options : fragmentOptionsMap.values()) { + stringBuilder.append(options.toString()); + } + return stringBuilder.toString(); + } + + /** + * Returns the options contained in this collection. + */ + public Iterable<FragmentOptions> getOptions() { + return fragmentOptionsMap.values(); + } + + /** + * Creates a copy of the BuildOptions object that contains copies of the FragmentOptions. + */ + @Override + public BuildOptions clone() { + ImmutableMap.Builder<Class<? extends FragmentOptions>, FragmentOptions> builder = + ImmutableMap.builder(); + for (Map.Entry<Class<? extends FragmentOptions>, FragmentOptions> entry : + fragmentOptionsMap.entrySet()) { + builder.put(entry.getKey(), entry.getValue().clone()); + } + return new BuildOptions(builder.build()); + } + + @Override + public boolean equals(Object other) { + return (this == other) || (other instanceof BuildOptions && + fragmentOptionsMap.equals(((BuildOptions) other).fragmentOptionsMap)); + } + + @Override + public int hashCode() { + return fragmentOptionsMap.hashCode(); + } + + /** + * Maps options class definitions to FragmentOptions objects + */ + private final ImmutableMap<Class<? extends FragmentOptions>, FragmentOptions> fragmentOptionsMap; + + private BuildOptions( + ImmutableMap<Class<? extends FragmentOptions>, FragmentOptions> fragmentOptionsMap) { + this.fragmentOptionsMap = fragmentOptionsMap; + } + + /** + * Creates a builder object for BuildOptions + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Builder class for BuildOptions. + */ + public static class Builder { + /** + * Adds a new FragmentOptions instance to the builder. Overrides previous instances of the + * exact same subclass of FragmentOptions. + */ + public <T extends FragmentOptions> Builder add(T options) { + builderMap.put(options.getClass(), options); + return this; + } + + public BuildOptions build() { + return new BuildOptions(ImmutableMap.copyOf(builderMap)); + } + + private Map<Class<? extends FragmentOptions>, FragmentOptions> builderMap; + + private Builder() { + builderMap = new HashMap<>(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/CompilationMode.java b/src/main/java/com/google/devtools/build/lib/analysis/config/CompilationMode.java new file mode 100644 index 0000000..7f24351 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/CompilationMode.java
@@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.config; + +import com.google.devtools.common.options.EnumConverter; + +/** + * This class represents the debug/optimization mode the binaries will be + * built for. + */ +public enum CompilationMode { + + // Fast build mode (-g0). + FASTBUILD("fastbuild"), + // Debug mode (-g). + DBG("dbg"), + // Release mode (-g0 -O2 -DNDEBUG). + OPT("opt"); + + private final String mode; + + private CompilationMode(String mode) { + this.mode = mode; + } + + @Override + public String toString() { + return mode; + } + + /** + * Converts to {@link CompilationMode}. + */ + public static class Converter extends EnumConverter<CompilationMode> { + public Converter() { + super(CompilationMode.class, "compilation mode"); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigMatchingProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigMatchingProvider.java new file mode 100644 index 0000000..c719191 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigMatchingProvider.java
@@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.config; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; + +/** + * A "configuration target" that asserts whether or not it matches the + * configuration it's bound to. + * + * <p>This can be used, e.g., to declare a BUILD target that defines the + * conditions which trigger a configurable attribute branch. In general, + * this can be used to trigger for any user-configurable build behavior. + */ +@Immutable +public final class ConfigMatchingProvider implements TransitiveInfoProvider { + + private final Label label; + private final boolean matches; + + public ConfigMatchingProvider(Label label, boolean matches) { + this.label = label; + this.matches = matches; + } + + /** + * The target's label. + */ + public Label label() { + return label; + } + + /** + * Whether or not the configuration criteria defined by this target match + * its actual configuration. + */ + public boolean matches() { + return matches; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigRuleClasses.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigRuleClasses.java new file mode 100644 index 0000000..6ee4cce --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigRuleClasses.java
@@ -0,0 +1,204 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.config; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.STRING_DICT; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.Type; + +/** + * Definitions for rule classes that specify or manipulate configuration settings. + * + * <p>These are not "traditional" rule classes in that they can't be requested as top-level + * targets and don't translate input artifacts into output artifacts. Instead, they affect + * how *other* rules work. See individual class comments for details. + */ +public class ConfigRuleClasses { + + private static final String NONCONFIGURABLE_ATTRIBUTE_REASON = + "part of a rule class that *triggers* configurable behavior"; + + /** + * Common settings for all configurability rules. + */ + @BlazeRule(name = "$config_base_rule", + type = RuleClass.Builder.RuleClassType.ABSTRACT, + ancestors = { BaseRuleClasses.BaseRule.class }) + public static final class ConfigBaseRule implements RuleDefinition { + @Override + public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) { + return builder + .override(attr("tags", Type.STRING_LIST) + // No need to show up in ":all", etc. target patterns. + .value(ImmutableList.of("manual")) + .nonconfigurable(NONCONFIGURABLE_ATTRIBUTE_REASON)) + .build(); + } + } + + /** + * A named "partial configuration setting" that specifies a set of command-line + * "flag=value" bindings. + * + * <p>For example: + * <pre> + * config_setting( + * name = 'foo', + * values = { + * 'flag1': 'aValue' + * 'flag2': 'bValue' + * }) + * </pre> + * + * <p>declares a setting that binds command-line flag <pre>flag1</pre> to value + * <pre>aValue</pre> and <pre>flag2</pre> to <pre>bValue</pre>. + * + * <p>This is used by configurable attributes to determine which branch to + * follow based on which <pre>config_setting</pre> instance matches all its + * flag values in the configurable attribute owner's configuration. + * + * <p>This rule isn't accessed through the standard {@link RuleContext#getPrerequisites} + * interface. This is because Bazel constructs a rule's configured attribute map *before* + * its {@link RuleContext} is created (in fact, the map is an input to the context's + * constructor). And the config_settings referenced by the rule's configurable attributes are + * themselves inputs to that map. So Bazel has special logic to read and properly apply + * config_setting instances. See {@link ConfiguredTargetFunction#getConfigConditions} for details. + */ + @BlazeRule(name = "config_setting", + type = RuleClass.Builder.RuleClassType.NORMAL, + ancestors = { ConfigBaseRule.class }, + factoryClass = ConfigSetting.class) + public static final class ConfigSettingRule implements RuleDefinition { + /** + * The name of the attribute that declares flag bindings. + */ + public static final String SETTINGS_ATTRIBUTE = "values"; + + @Override + public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE(config_setting).ATTRIBUTE(values) --> + The set of configuration values that match this rule (expressed as Blaze flags) + + <i>(Dictionary mapping flags to expected values, both expressed as strings; + mandatory)</i> + + <p>This rule inherits the configuration of the configured target that + references it in a <code>select</code> statement. It is considered to + "match" a Blaze invocation if, for every entry in the dictionary, its + configuration matches the entry's expected value. For example + <code>values = {"compilation_mode": "opt"}</code> matches the invocations + <code>blaze build --compilation_mode=opt ...</code> and + <code>blaze build -c opt ...</code> on target-configured rules. + </p> + + <p>For convenience's sake, configuration values are specified as Blaze flags (without + the preceding <code>"--"</code>). But keep in mind that the two are not the same. This + is because targets can be built in multiple configurations within the same + build. For example, a host configuration's "cpu" matches the value of + <code>--host_cpu</code>, not <code>--cpu</code>. So different instances of the + same <code>config_setting</code> may match the same invocation differently + depending on the configuration of the rule using them. + </p> + + <p>If a flag is not explicitly set at the command line, its default value is used. + If a key appears multiple times in the dictionary, only the last instance is used. + If a key references a flag that can be set multiple times on the command line (e.g. + <code>blaze build --copt=foo --copt=bar --copt=baz ...</code>), a match occurs if + *any* of those settings match. + <p> + + <p>This attribute cannot be empty. + </p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr(SETTINGS_ATTRIBUTE, STRING_DICT).mandatory() + .nonconfigurable(NONCONFIGURABLE_ATTRIBUTE_REASON)) + .build(); + } + } + +/*<!-- #BLAZE_RULE (NAME = config_setting, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] --> + +${ATTRIBUTE_SIGNATURE} + +<p> + Matches an expected configuration state (expressed as Blaze flags) for the purpose of triggering + configurable attributes. See <a href="#select">select</a> for how to consume this rule and + <a href="#configurable-attributes">Configurable attributes</a> for an overview of + the general feature. + +${ATTRIBUTE_DEFINITION} + +<h4 id="config_setting_examples">Examples</h4> + +<p>The following matches any Blaze invocation that specifies <code>--compilation_mode=opt</code> + or <code>-c opt</code> (either explicitly at the command line or implicitly from .blazerc + files, etc.), when applied to a target configuration rule: +</p> + +<pre class="code"> +config_setting( + name = "simple", + values = {"compilation_mode": "opt"} +) +</pre> + +<p>The following matches any Blaze invocation that builds for ARM and applies a custom define + (e.g. <code>blaze build --cpu=armeabi --define FOO=bar ...</code>), , when applied to a target + configuration rule: +</p> + +<pre class="code"> +config_setting( + name = "two_conditions", + values = { + "cpu": "armeabi", + "define": "FOO=bar" + } +) +</pre> + +<h4 id="config_setting_notes">Notes</h4> + +<p>See <a href="#select">select</a> for policies on what happens depending on how many rules match + an invocation. +</p> + +<p>For flags that support shorthand forms (e.g. <code>--compilation_mode</code> vs. + <code>-c</code>), <code>values</code> definitions must use the full form. These automatically + match invocations using either form. +</p> + +<p>The currently endorsed method for creating custom conditions that can't be expressed through + dedicated build flags is through the --define flag. Use this flag with caution: it's not ideal + and only endorsed for lack of a currently better workaround. See the + <a href="#configurable-attributes">Configurable attributes</a> section for more discussion. +</p> + +<p>Try to consolidate <code>config_setting</code> definitions as much as possible. In other words, + define <code>//common/conditions:foo</code> in one common package instead of repeating separate + instances in <code>//project1:foo</code>, <code>//project2:foo</code>, etc. that all mean the + same thing. +</p> + +<!-- #END_BLAZE_RULE -->*/ +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigSetting.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigSetting.java new file mode 100644 index 0000000..97ad4ff --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigSetting.java
@@ -0,0 +1,170 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.config; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Implementation for the config_setting rule. + * + * <p>This is a "pseudo-rule" in that its purpose isn't to generate output artifacts + * from input artifacts. Rather, it provides configuration context to rules that + * depend on it. + */ +public class ConfigSetting implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + // Get the required flag=value settings for this rule. + Map<String, String> settings = NonconfigurableAttributeMapper.of(ruleContext.getRule()) + .get(ConfigRuleClasses.ConfigSettingRule.SETTINGS_ATTRIBUTE, Type.STRING_DICT); + if (settings.isEmpty()) { + ruleContext.attributeError(ConfigRuleClasses.ConfigSettingRule.SETTINGS_ATTRIBUTE, + "no settings specified"); + return null; + } + + ConfigMatchingProvider configMatcher; + try { + configMatcher = new ConfigMatchingProvider(ruleContext.getLabel(), + matchesConfig(settings, ruleContext.getConfiguration())); + } catch (OptionsParsingException e) { + ruleContext.attributeError(ConfigRuleClasses.ConfigSettingRule.SETTINGS_ATTRIBUTE, + "error while parsing configuration settings: " + e.getMessage()); + return null; + } + + return new RuleConfiguredTargetBuilder(ruleContext) + .add(RunfilesProvider.class, RunfilesProvider.EMPTY) + .add(FileProvider.class, new FileProvider(ruleContext.getLabel(), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER))) + .add(FilesToRunProvider.class, new FilesToRunProvider(ruleContext.getLabel(), + ImmutableList.<Artifact>of(), null, null)) + .add(ConfigMatchingProvider.class, configMatcher) + .build(); + } + + /** + * Given a list of [flagName, flagValue] pairs, returns true if flagName == flagValue for + * every item in the list under this configuration, false otherwise. + */ + private boolean matchesConfig(Map<String, String> expectedSettings, BuildConfiguration config) + throws OptionsParsingException { + // Rather than returning fast when we find a mismatch, continue looking at the other flags + // to check that they're indeed valid flag specifications. + boolean foundMismatch = false; + + // Since OptionsParser instantiation involves reflection, let's try to minimize that happening. + Map<Class<? extends OptionsBase>, OptionsParser> parserCache = new HashMap<>(); + + for (Map.Entry<String, String> setting : expectedSettings.entrySet()) { + String optionName = setting.getKey(); + String expectedRawValue = setting.getValue(); + + Class<? extends OptionsBase> optionClass = config.getOptionClass(optionName); + if (optionClass == null) { + throw new OptionsParsingException("unknown option: '" + optionName + "'"); + } + + OptionsParser parser = parserCache.get(optionClass); + if (parser == null) { + parser = OptionsParser.newOptionsParser(optionClass); + parserCache.put(optionClass, parser); + } + parser.parse("--" + optionName + "=" + expectedRawValue); + Object expectedParsedValue = parser.getOptions(optionClass).asMap().get(optionName); + + if (!optionMatches(config, optionName, expectedParsedValue)) { + foundMismatch = true; + } + } + return !foundMismatch; + } + + /** + * For single-value options, returns true iff the option's value matches the expected value. + * + * <p>For multi-value List options, returns true iff any of the option's values matches + * the expected value. This means, e.g. "--tool_tag=foo --tool_tag=bar" would match the + * expected condition { 'tool_tag': 'bar' }. + * + * <p>For multi-value Map options, returns true iff the last instance with the same key as the + * expected key has the same value. This means, e.g. "--define foo=1 --define bar=2" would + * match { 'define': 'foo=1' }, but "--define foo=1 --define bar=2 --define foo=3" would not + * match. Note that the definition of --define states that the last instance takes precedence. + */ + private static boolean optionMatches(BuildConfiguration config, String optionName, + Object expectedValue) { + Object actualValue = config.getOptionValue(optionName); + if (actualValue == null) { + return expectedValue == null; + + // Single-value case: + } else if (!config.allowsMultipleValues(optionName)) { + return actualValue.equals(expectedValue); + } + + // Multi-value case: + Preconditions.checkState(actualValue instanceof List); + Preconditions.checkState(expectedValue instanceof List); + List<?> actualList = (List<?>) actualValue; + List<?> expectedList = (List<?>) expectedValue; + + if (actualList.isEmpty() || expectedList.isEmpty()) { + return actualList.isEmpty() && expectedList.isEmpty(); + } + + // We're expecting a single value of a multi-value type: the options parser still embeds + // that single value within a List container. Retrieve it here. + Object expectedSingleValue = Iterables.getOnlyElement(expectedList); + + // Multi-value map: + if (actualList.get(0) instanceof Map.Entry) { + Map.Entry<?, ?> expectedEntry = (Map.Entry<?, ?>) expectedSingleValue; + for (Map.Entry<?, ?> actualEntry : Lists.reverse((List<Map.Entry<?, ?>>) actualList)) { + if (actualEntry.getKey().equals(expectedEntry.getKey())) { + // Found a key match! + return actualEntry.getValue().equals(expectedEntry.getValue()); + } + } + return false; // Never found any matching key. + } + + // Multi-value list: + return actualList.contains(expectedSingleValue); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationEnvironment.java new file mode 100644 index 0000000..89722b5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationEnvironment.java
@@ -0,0 +1,96 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.config; + +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider; +import com.google.devtools.build.lib.pkgcache.PackageProvider; +import com.google.devtools.build.lib.pkgcache.TargetProvider; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.Path; + +import javax.annotation.Nullable; + +/** + * An environment to support creating BuildConfiguration instances in a hermetic fashion; all + * accesses to packages or the file system <b>must</b> go through this interface, so that they can + * be recorded for correct caching. + */ +public interface ConfigurationEnvironment { + + /** + * Returns a target for the given label, loading it if necessary, and throwing an exception if it + * does not exist. + * + * @see TargetProvider#getTarget + */ + Target getTarget(Label label) throws NoSuchPackageException, NoSuchTargetException; + + /** Returns a path for the given file within the given package. */ + Path getPath(Package pkg, String fileName); + + /** Returns fragment based on fragment class and build options. */ + <T extends Fragment> T getFragment(BuildOptions buildOptions, Class<T> fragmentType) + throws InvalidConfigurationException; + + /** Returns global value of BlazeDirectories. */ + @Nullable + BlazeDirectories getBlazeDirectories(); + + /** + * An implementation backed by a {@link PackageProvider} instance. + */ + public static final class TargetProviderEnvironment implements ConfigurationEnvironment { + + private final LoadedPackageProvider loadedPackageProvider; + private final BlazeDirectories blazeDirectories; + + public TargetProviderEnvironment(LoadedPackageProvider loadedPackageProvider, + BlazeDirectories blazeDirectories) { + this.loadedPackageProvider = loadedPackageProvider; + this.blazeDirectories = blazeDirectories; + } + + public TargetProviderEnvironment(LoadedPackageProvider loadedPackageProvider) { + this.loadedPackageProvider = loadedPackageProvider; + this.blazeDirectories = null; + } + + @Override + public Target getTarget(Label label) throws NoSuchPackageException, NoSuchTargetException { + return loadedPackageProvider.getLoadedTarget(label); + } + + @Override + public Path getPath(Package pkg, String fileName) { + return pkg.getPackageDirectory().getRelative(fileName); + } + + @Override + public <T extends Fragment> T getFragment(BuildOptions buildOptions, Class<T> fragmentType) { + throw new UnsupportedOperationException(); + } + + @Override + public BlazeDirectories getBlazeDirectories() { + return blazeDirectories; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationFactory.java new file mode 100644 index 0000000..13afb2a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationFactory.java
@@ -0,0 +1,145 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.config; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedMap; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.ConfigurationCollectionFactory; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.events.EventHandler; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * A factory class for {@link BuildConfiguration} instances. This is unfortunately more complex, + * and should be simplified in the future, if + * possible. Right now, creating a {@link BuildConfiguration} instance involves + * creating the instance itself and the related configurations; the main method + * is {@link #createConfiguration}. + * + * <p>Avoid calling into this class, and instead use the skyframe infrastructure to obtain + * configuration instances. + * + * <p>Blaze currently relies on the fact that all {@link BuildConfiguration} + * instances used in a build can be constructed ahead of time by this class. + */ +@ThreadCompatible // safe as long as separate instances are used +public final class ConfigurationFactory { + private final List<ConfigurationFragmentFactory> configurationFragmentFactories; + private final ConfigurationCollectionFactory configurationCollectionFactory; + + // A cache of key to configuration instances. + private final Cache<String, BuildConfiguration> hostConfigCache = + CacheBuilder.newBuilder().softValues().build(); + + private boolean performSanityCheck = true; + + public ConfigurationFactory( + ConfigurationCollectionFactory configurationCollectionFactory, + List<ConfigurationFragmentFactory> fragmentFactories) { + this.configurationCollectionFactory = + Preconditions.checkNotNull(configurationCollectionFactory); + this.configurationFragmentFactories = fragmentFactories; + } + + @VisibleForTesting + public void forbidSanityCheck() { + performSanityCheck = false; + } + + /** Create the build configurations with the given options. */ + @Nullable + public BuildConfiguration createConfiguration( + PackageProviderForConfigurations loadedPackageProvider, BuildOptions buildOptions, + BuildConfigurationKey key, EventHandler errorEventListener) + throws InvalidConfigurationException { + return configurationCollectionFactory.createConfigurations(this, + loadedPackageProvider, buildOptions, key.getClientEnv(), + errorEventListener, performSanityCheck); + } + + /** + * Returns a (possibly new) canonical host BuildConfiguration instance based + * upon a given request configuration + */ + @Nullable + public BuildConfiguration getHostConfiguration( + PackageProviderForConfigurations loadedPackageProvider, Map<String, String> clientEnv, + BuildOptions buildOptions, boolean fallback) throws InvalidConfigurationException { + return getConfiguration(loadedPackageProvider, buildOptions.createHostOptions(fallback), + clientEnv, false, hostConfigCache); + } + + /** + * The core of BuildConfiguration creation. All host and target instances are + * constructed and cached here. + */ + @Nullable + public BuildConfiguration getConfiguration(PackageProviderForConfigurations loadedPackageProvider, + BuildOptions buildOptions, Map<String, String> clientEnv, + boolean actionsDisabled, Cache<String, BuildConfiguration> cache) + throws InvalidConfigurationException { + + Map<Class<? extends Fragment>, Fragment> fragments = new HashMap<>(); + // Create configuration fragments + for (ConfigurationFragmentFactory factory : configurationFragmentFactories) { + Class<? extends Fragment> fragmentType = factory.creates(); + Fragment fragment = loadedPackageProvider.getFragment(buildOptions, fragmentType); + if (fragment != null && fragments.get(fragment) == null) { + fragments.put(fragment.getClass(), fragment); + } + } + BlazeDirectories directories = loadedPackageProvider.getDirectories(); + if (loadedPackageProvider.valuesMissing()) { + return null; + } + + // Sort the fragments by class name to make sure that the order is stable. Afterwards, copy to + // an ImmutableMap, which keeps the order stable, but uses hashing, and drops the reference to + // the Comparator object. + fragments = ImmutableSortedMap.copyOf(fragments, new Comparator<Class<? extends Fragment>>() { + @Override + public int compare(Class<? extends Fragment> o1, Class<? extends Fragment> o2) { + return o1.getName().compareTo(o2.getName()); + } + }); + fragments = ImmutableMap.copyOf(fragments); + + String key = BuildConfiguration.computeCacheKey( + directories, fragments, buildOptions, clientEnv); + BuildConfiguration configuration = cache.getIfPresent(key); + if (configuration == null) { + configuration = new BuildConfiguration(directories, fragments, buildOptions, + clientEnv, actionsDisabled); + cache.put(key, configuration); + } + return configuration; + } + + public List<ConfigurationFragmentFactory> getFactories() { + return configurationFragmentFactories; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationFragmentFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationFragmentFactory.java new file mode 100644 index 0000000..8ca8f1c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationFragmentFactory.java
@@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.config; + +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; + +import javax.annotation.Nullable; + +/** + * A factory that creates configuration fragments. + */ +public interface ConfigurationFragmentFactory { + /** + * Creates a configuration fragment. + * + * @param env the ConfigurationEnvironment for querying targets and paths + * @param buildOptions command-line options (see {@link FragmentOptions}) + * @return the configuration fragment or null if some required dependencies are missing. + */ + @Nullable + BuildConfiguration.Fragment create(ConfigurationEnvironment env, BuildOptions buildOptions) + throws InvalidConfigurationException; + + /** + * @return the exact type of the fragment this factory creates. + */ + Class<? extends Fragment> creates(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/DefaultsPackage.java b/src/main/java/com/google/devtools/build/lib/analysis/config/DefaultsPackage.java new file mode 100644 index 0000000..207d49a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/DefaultsPackage.java
@@ -0,0 +1,164 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.config; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; + +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * A helper class to compute and inject a defaults package into the package cache. + * + * <p>The <code>//tools/defaults</code> package provides a mechanism let tool locations be + * specified over the commandline, without requiring any special support in the rule code. + * As such, it can be used in genrule <code>$(location)</code> substitutions. + * + * <p>It works as follows: + * <ul> + * + * <li> SomeLanguage.createCompileAction will refer to a host-configured target for the + * compiler by looking for + * <code>env.getHostPrerequisiteArtifact("$somelanguage_compiler")</code>. + * + * <li> the attribute <code>$somelanguage_compiler</code> is defined in the + * {@link RuleDefinition} subclass for that language. + * + * <li> if the attribute cannot be set on the command-line, its value may be a normal label. + * + * <li> if the attribute can be set on the command-line, its value will be + * <code>//tools/defaults:somelanguage_compiler</code>. + * + * <li> in the latter case, the {@link BuildConfiguration.Fragment} subclass will define the + * option (with an existing target, eg. <code>//third_party/somelanguage:compiler</code>), and + * return the name in its implementation of {@link FragmentOptions#getDefaultsLabels}. + * + * <li> On startup, the rule is wired up with <code>//tools/defaults:somelanguage_compiler</code>. + * + * <li> On starting a build, the <code>//tools/defaults</code> package is synthesized, using + * the values as specified on the command-line. The contents of + * <code>tools/defaults/BUILD</code> is ignored. + * + * <li> Hence, changes in the command line values for tools are now handled exactly as if they + * were changes in a BUILD file. + * + * <li> The file <code>tools/defaults/BUILD</code> must exist, so we create a package in that + * location. + * + * <li> The code in {@link DefaultsPackage} can dump the synthesized package as a BUILD file, + * so external tooling does not need to understand the intricacies of handling command-line + * options. + * + * </ul> + * + * <p>For built-in rules (as opposed to genrules), late-bound labels provide an alternative + * method of depending on command-line values. These work by declaring attribute default values + * to be {@link LateBoundLabel} instances, whose <code>getDefault(Rule rule, T + * configuration)</code> method will have access to {@link BuildConfiguration}, which in turn + * may depend on command line flag values. + */ +public final class DefaultsPackage { + + // The template contents are broken into lines such that the resulting file has no more than 80 + // characters per line. + private static final String HEADER = "" + + "# DO NOT EDIT THIS FILE!\n" + + "#\n" + + "# Bazel does not read this file. Instead, it internally replaces the targets in\n" + + "# this package with the correct packages as given on the command line.\n" + + "#\n" + + "# If these options are not given on the command line, Bazel will use the exact\n" + + "# same targets as given here." + + "\n" + + "package(default_visibility = ['//visibility:public'])\n"; + + /** + * The map from entries to their values. + */ + private ImmutableMap<String, ImmutableSet<Label>> values; + + private DefaultsPackage(BuildOptions buildOptions) { + values = buildOptions.getDefaultsLabels(); + } + + private String labelsToString(Set<Label> labels) { + StringBuffer result = new StringBuffer(); + for (Label label : labels) { + if (result.length() != 0) { + result.append(", "); + } + result.append("'").append(label).append("'"); + } + return result.toString(); + } + + /** + * Returns a string of the defaults package with the given settings. + */ + private String getContent() { + Preconditions.checkState(!values.isEmpty()); + StringBuilder result = new StringBuilder(HEADER); + for (Map.Entry<String, ImmutableSet<Label>> entry : values.entrySet()) { + result + .append("filegroup(name = '") + .append(entry.getKey().toLowerCase(Locale.US)).append("',\n") + .append(" srcs = [") + .append(labelsToString(entry.getValue())).append("])\n"); + } + return result.toString(); + } + + /** + * Returns the defaults package for the default settings. + */ + public static String getDefaultsPackageContent( + Iterable<Class<? extends FragmentOptions>> options) { + return getDefaultsPackageContent(BuildOptions.createDefaults(options)); + } + + /** + * Returns the defaults package for the given options. + */ + public static String getDefaultsPackageContent(BuildOptions buildOptions) { + return new DefaultsPackage(buildOptions).getContent(); + } + + public static void parseAndAdd(Set<Label> labels, String optionalLabel) { + if (optionalLabel != null) { + Label label = parseOptionalLabel(optionalLabel); + if (label != null) { + labels.add(label); + } + } + } + + public static Label parseOptionalLabel(String value) { + if (value.startsWith("//")) { + try { + return Label.parseAbsolute(value); + } catch (SyntaxException e) { + // We ignore this exception here - it will cause an error message at a later time. + return null; + } + } else { + return null; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/FragmentOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/FragmentOptions.java new file mode 100644 index 0000000..ce4b2d2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/FragmentOptions.java
@@ -0,0 +1,115 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.config; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.packages.Attribute.SplitTransition; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.common.options.Options; +import com.google.devtools.common.options.OptionsBase; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Command-line build options for a Blaze module. + */ +public abstract class FragmentOptions extends OptionsBase implements Cloneable, Serializable { + + /** + * Adds all labels defined by the options to a multimap. See {@code BuildOptions.getAllLabels()}. + * + * <p>There should generally be no code duplication between this code and DefaultsPackage. Either + * the labels are loaded unconditionally using this method, or they are added as magic labels + * using the tools/defaults package, but not both. + * + * @param labelMap a mutable multimap to which the labels of this fragment should be added + */ + public void addAllLabels(Multimap<String, Label> labelMap) { + } + + /** + * Returns the labels contributed to the defaults package by this fragment. + * + * <p>The set of keys returned by this function should be constant, however, the values are + * allowed to change depending on the value of the options. + */ + @SuppressWarnings("unused") + public Map<String, Set<Label>> getDefaultsLabels(BuildConfiguration.Options commonOptions) { + return ImmutableMap.of(); + } + + /** + * Returns a list of potential split configuration transitions for this fragment. Split + * configurations usually need to be explicitly enabled by passing in an option. + */ + public List<SplitTransition<BuildOptions>> getPotentialSplitTransitions() { + return ImmutableList.of(); + } + + @Override + public FragmentOptions clone() { + try { + return (FragmentOptions) super.clone(); + } catch (CloneNotSupportedException e) { + // This can't happen. + throw new IllegalStateException(e); + } + } + + /** + * Creates a new FragmentOptions instance with all flags set to default. + */ + public FragmentOptions getDefault() { + return Options.getDefaults(getClass()); + } + + /** + * Creates a new FragmentOptions instance with flags adjusted to host platform. + * + * @param fallback see {@code BuildOptions.createHostOptions} + */ + @SuppressWarnings("unused") + public FragmentOptions getHost(boolean fallback) { + return getDefault(); + } + + protected void addOptionalLabel(Multimap<String, Label> map, String key, String value) { + Label label = parseOptionalLabel(value); + if (label != null) { + map.put(key, label); + } + } + + private static Label parseOptionalLabel(String value) { + if ((value != null) && value.startsWith("//")) { + try { + return Label.parseAbsolute(value); + } catch (SyntaxException e) { + // We ignore this exception here - it will cause an error message at a later time. + // TODO(bazel-team): We can use a Converter to check the validity of the crosstoolTop + // earlier. + return null; + } + } else { + return null; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/InvalidConfigurationException.java b/src/main/java/com/google/devtools/build/lib/analysis/config/InvalidConfigurationException.java new file mode 100644 index 0000000..c39325d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/InvalidConfigurationException.java
@@ -0,0 +1,33 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.config; + +/** + * Thrown if the configuration options lead to an invalid configuration, or if any of the + * configuration labels cannot be loaded. + */ +public class InvalidConfigurationException extends Exception { + + public InvalidConfigurationException(String message) { + super(message); + } + + public InvalidConfigurationException(String message, Throwable cause) { + super(message, cause); + } + + public InvalidConfigurationException(Throwable cause) { + this(cause.getMessage(), cause); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/PackageProviderForConfigurations.java b/src/main/java/com/google/devtools/build/lib/analysis/config/PackageProviderForConfigurations.java new file mode 100644 index 0000000..83b4715 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/PackageProviderForConfigurations.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.config; + +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; + +import java.io.IOException; + +/** + * Extended LoadedPackageProvider which is used during a creation of BuildConfiguration.Fragments. + */ +public interface PackageProviderForConfigurations extends LoadedPackageProvider { + /** + * Adds dependency to fileName if needed. Used only in skyframe, for creating correct dependencies + * for {@link com.google.devtools.build.lib.skyframe.ConfigurationCollectionValue}. + */ + void addDependency(Package pkg, String fileName) throws SyntaxException, IOException; + + /** + * Returns fragment based on fragment type and build options. + */ + <T extends Fragment> T getFragment(BuildOptions buildOptions, Class<T> fragmentType) + throws InvalidConfigurationException; + + /** + * Returns blaze directories and adds dependency to that value. + */ + BlazeDirectories getDirectories(); + + /** + * Returns true if any dependency is missing (value of some node hasn't been evaluated yet). + */ + boolean valuesMissing(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/PerLabelOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/PerLabelOptions.java new file mode 100644 index 0000000..1e921e5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/PerLabelOptions.java
@@ -0,0 +1,128 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.config; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.RegexFilter; +import com.google.devtools.build.lib.util.RegexFilter.RegexFilterConverter; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.OptionsParsingException; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Models options that can be added to a command line when a label matches a + * given {@link RegexFilter}. + */ +public class PerLabelOptions implements Serializable { + + /** The filter used to match labels */ + private final RegexFilter regexFilter; + + /** The list of options to add when the filter matches a label */ + private final List<String> optionsList; + + /** + * Converts a String to a {@link PerLabelOptions} object. The syntax of the + * string is {@code regex_filter@option_1,option_2,...,option_n}. Where + * regex_filter stands for the String representation of a {@link RegexFilter}, + * and {@code option_1} to {@code option_n} stand for arbitrary command line + * options. If an option contains a comma it has to be quoted with a + * backslash. Options can contain @. Only the first @ is used to split the + * string. + */ + public static class PerLabelOptionsConverter implements Converter<PerLabelOptions> { + + @Override + public PerLabelOptions convert(String input) throws OptionsParsingException { + int atIndex = input.indexOf('@'); + RegexFilterConverter converter = new RegexFilter.RegexFilterConverter(); + if (atIndex < 0) { + return new PerLabelOptions(converter.convert(input), ImmutableList.<String> of()); + } else { + String filterPiece = input.substring(0, atIndex); + String optionsPiece = input.substring(atIndex + 1); + List<String> optionsList = new ArrayList<>(); + for (String option : optionsPiece.split("(?<!\\\\),")) { // Split on ',' but not on '\,' + if (option != null && !option.trim().equals("")) { + optionsList.add(option.replace("\\,", ",")); + } + } + return new PerLabelOptions(converter.convert(filterPiece), optionsList); + } + } + + @Override + public String getTypeDescription() { + return "a comma-separated list of regex expressions with prefix '-' specifying" + + " excluded paths followed by an @ and a comma separated list of options"; + } + } + + public PerLabelOptions(RegexFilter regexFilter, List<String> optionsList) { + this.regexFilter = regexFilter; + this.optionsList = optionsList; + } + + /** + * @return true if the given label is matched by the {@link RegexFilter}. + */ + public boolean isIncluded(Label label) { + return regexFilter.isIncluded(label.toString()); + } + + /** + * @return true if the execution path (which includes the base name of the file) + * of the given file is matched by the {@link RegexFilter}. + */ + public boolean isIncluded(Artifact artifact) { + return regexFilter.isIncluded(artifact.getExecPathString()); + } + + /** + * Returns the list of options to add to a command line. + */ + public List<String> getOptions() { + return optionsList; + } + + RegexFilter getRegexFilter() { + return regexFilter; + } + + @Override + public String toString() { + return regexFilter + " Options: " + optionsList; + } + + @Override + public boolean equals(Object other) { + PerLabelOptions otherOptions = + other instanceof PerLabelOptions ? (PerLabelOptions) other : null; + return this == other || (otherOptions != null && + this.regexFilter.equals(otherOptions.regexFilter) && + this.optionsList.equals(otherOptions.optionsList)); + } + + @Override + public int hashCode() { + return Objects.hash(regexFilter, optionsList); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/RunUnder.java b/src/main/java/com/google/devtools/build/lib/analysis/config/RunUnder.java new file mode 100644 index 0000000..a51ea25 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/RunUnder.java
@@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.config; + +import com.google.devtools.build.lib.syntax.Label; + +import java.io.Serializable; +import java.util.List; + +/** + * Components of --run_under option. + */ +public interface RunUnder extends Serializable { + /** + * @return the whole value passed to --run_under option. + */ + String getValue(); + + /** + * Returns label corresponding to the first word (according to shell + * tokenization) passed to --run_under. + * + * @return if the first word (according to shell tokenization) passed to + * --run_under starts with {@code "//"} returns the label + * corresponding to that word otherwise {@code null} + */ + Label getLabel(); + + /** + * @return if the first word (according to shell tokenization) passed to + * --run_under starts with {@code "//"} returns {@code null} + * otherwise the first word + */ + String getCommand(); + + /** + * @return everything except the first word (according to shell + * tokenization) passed to --run_under. + */ + List<String> getOptions(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/RunUnderConverter.java b/src/main/java/com/google/devtools/build/lib/analysis/config/RunUnderConverter.java new file mode 100644 index 0000000..1f7b660 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/RunUnderConverter.java
@@ -0,0 +1,133 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.config; + +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.shell.ShellUtils.TokenizationException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * --run_under options converter. + */ +public class RunUnderConverter implements Converter<RunUnder> { + @Override + public RunUnder convert(final String input) throws OptionsParsingException { + final List<String> runUnderList = new ArrayList<>(); + try { + ShellUtils.tokenize(runUnderList, input); + } catch (TokenizationException e) { + throw new OptionsParsingException("Not a valid command prefix " + e.getMessage()); + } + if (runUnderList.isEmpty()) { + throw new OptionsParsingException("Empty command"); + } + final String runUnderCommand = runUnderList.get(0); + if (runUnderCommand.startsWith("//")) { + try { + final Label runUnderLabel = Label.parseAbsolute(runUnderCommand); + return new RunUnderLabel(input, runUnderLabel, runUnderList); + } catch (SyntaxException e) { + throw new OptionsParsingException("Not a valid label " + e.getMessage()); + } + } else { + return new RunUnderCommand(input, runUnderCommand, runUnderList); + } + } + + private static final class RunUnderLabel implements RunUnder { + private final String input; + private final Label runUnderLabel; + private final List<String> runUnderList; + + public RunUnderLabel(String input, Label runUnderLabel, List<String> runUnderList) { + this.input = input; + this.runUnderLabel = runUnderLabel; + this.runUnderList = new ArrayList<String>(runUnderList.subList(1, runUnderList.size())); + } + + @Override public String getValue() { return input; } + @Override public Label getLabel() { return runUnderLabel; } + @Override public String getCommand() { return null; } + @Override public List<String> getOptions() { return runUnderList; } + @Override public String toString() { return input; } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } else if (other instanceof RunUnderLabel) { + RunUnderLabel otherRunUnderLabel = (RunUnderLabel) other; + return Objects.equals(input, otherRunUnderLabel.input) + && Objects.equals(runUnderLabel, otherRunUnderLabel.runUnderLabel) + && Objects.equals(runUnderList, otherRunUnderLabel.runUnderList); + } else { + return false; + } + } + + @Override + public int hashCode() { + return Objects.hash(input, runUnderLabel, runUnderList); + } + } + + private static final class RunUnderCommand implements RunUnder { + private final String input; + private final String runUnderCommand; + private final List<String> runUnderList; + + public RunUnderCommand(String input, String runUnderCommand, List<String> runUnderList) { + this.input = input; + this.runUnderCommand = runUnderCommand; + this.runUnderList = new ArrayList<String>(runUnderList.subList(1, runUnderList.size())); + } + + @Override public String getValue() { return input; } + @Override public Label getLabel() { return null; } + @Override public String getCommand() { return runUnderCommand; } + @Override public List<String> getOptions() { return runUnderList; } + @Override public String toString() { return input; } + + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } else if (other instanceof RunUnderCommand) { + RunUnderCommand otherRunUnderCommand = (RunUnderCommand) other; + return Objects.equals(input, otherRunUnderCommand.input) + && Objects.equals(runUnderCommand, otherRunUnderCommand.runUnderCommand) + && Objects.equals(runUnderList, otherRunUnderCommand.runUnderList); + } else { + return false; + } + } + + @Override + public int hashCode() { + return Objects.hash(input, runUnderCommand, runUnderList); + } + } + @Override + public String getTypeDescription() { + return "a prefix in front of command"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/constraints/ConstraintSemantics.java b/src/main/java/com/google/devtools/build/lib/analysis/constraints/ConstraintSemantics.java new file mode 100644 index 0000000..14ac2bc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/constraints/ConstraintSemantics.java
@@ -0,0 +1,473 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.constraints; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.constraints.EnvironmentCollection.EnvironmentWithGroup; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.EnvironmentGroup; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Implementation of the semantics of Bazel's constraint specification and enforcement system. + * + * <p>This is how the system works: + * + * <p>All build rules can declare which "environments" they can be built for, where an "environment" + * is a label instance of an {@link EnvironmentRule} rule declared in a BUILD file. There are + * various ways to do this: + * + * <ul> + * <li>Through a "restricted to" attribute setting + * ({@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR}). This is the most direct form of + * specification - it declares the exact set of environments the rule supports (for its group - + * see precise details below). + * <li>Through a "compatible with" attribute setting + * ({@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR}. This declares <b>additional</b> + * environments a rule supports in addition to "standard" environments that are supported by + * default (see below). + * <li>Through "default" specifications in {@link EnvironmentGroup} rules. Every environment + * belongs to a group of thematically related peers (e.g. "target architectures", "JDK versions", + * or "mobile devices"). An environment group's definition includes which of these + * environments should be supported "by default" if not otherwise specified by one of the above + * mechanisms. In particular, a rule with no environment-related attributes automatically + * inherits all defaults. + * <li>Through a rule class default ({@link RuleClass.Builder#restrictedTo} and + * {@link RuleClass.Builder#compatibleWith}). This overrides global defaults for all instances + * of the given rule class. This can be used, for example, to make all *_test rules "testable" + * without each instance having to explicitly declare this capability. + * </ul> + * + * <p>Groups exist to model the idea that some environments are related while others have nothing + * to do with each other. Say, for example, we want to say a rule works for PowerPC platforms but + * not x86. We can do so by setting its "restricted to" attribute to + * {@code ['//sample/path:powerpc']}. Because both PowerPC and x86 are in the same + * "target architectures" group, this setting removes x86 from the set of supported environments. + * But since JDK support belongs to its own group ("JDK versions") it says nothing about which JDK + * the rule supports. + * + * <p>More precisely, if a rule has a "restricted to" value of [A, B, C], this removes support + * for all default environments D such that group(D) is in [group(A), group(B), group(C)] AND + * D is not in [A, B, C] (in other words, D isn't explicitly opted back in). The rule's full + * set of supported environments thus becomes [A, B, C] + all defaults that belong to unrelated + * groups. + * + * <p>If the rule has a "compatible with" value of [E, F, G], these are unconditionally + * added to its set of supported environments (in addition to the results from above). + * + * <p>An environment may not appear in both a rule's "restricted to" and "compatible with" values. + * If two environments belong to the same group, they must either both be in "restricted to", + * both be in "compatible with", or not explicitly specified. + * + * <p>Given all the above, constraint enforcement is this: rule A can depend on rule B if, for + * every environment A supports, B also supports that environment. + */ +public class ConstraintSemantics { + private ConstraintSemantics() { + } + + /** + * Provides a set of default environments for a given environment group. + */ + private interface DefaultsProvider { + Collection<Label> getDefaults(EnvironmentGroup group); + } + + /** + * Provides a group's defaults as specified in the environment group's BUILD declaration. + */ + private static class GroupDefaultsProvider implements DefaultsProvider { + @Override + public Collection<Label> getDefaults(EnvironmentGroup group) { + return group.getDefaults(); + } + } + + /** + * Provides a group's defaults, factoring in rule class defaults as specified by + * {@link com.google.devtools.build.lib.packages.RuleClass.Builder#compatibleWith} + * and {@link com.google.devtools.build.lib.packages.RuleClass.Builder#restrictedTo}. + */ + private static class RuleClassDefaultsProvider implements DefaultsProvider { + private final EnvironmentCollection ruleClassDefaults; + private final GroupDefaultsProvider groupDefaults; + + RuleClassDefaultsProvider(EnvironmentCollection ruleClassDefaults) { + this.ruleClassDefaults = ruleClassDefaults; + this.groupDefaults = new GroupDefaultsProvider(); + } + + @Override + public Collection<Label> getDefaults(EnvironmentGroup group) { + if (ruleClassDefaults.getGroups().contains(group)) { + return ruleClassDefaults.getEnvironments(group); + } else { + // If there are no rule class defaults for this group, just inherit global defaults. + return groupDefaults.getDefaults(group); + } + } + } + + /** + * Collects the set of supported environments for a given rule by merging its + * restriction-style and compatibility-style environment declarations as specified by + * the given attributes. Only includes environments from "known" groups, i.e. the groups + * owning the environments explicitly referenced from these attributes. + */ + private static class EnvironmentCollector { + private final RuleContext ruleContext; + private final String restrictionAttr; + private final String compatibilityAttr; + private final DefaultsProvider defaultsProvider; + + private final EnvironmentCollection restrictionEnvironments; + private final EnvironmentCollection compatibilityEnvironments; + private final EnvironmentCollection supportedEnvironments; + + /** + * Constructs a new collector on the given attributes. + * + * @param ruleContext analysis context for the rule + * @param restrictionAttr the name of the attribute that declares "restricted to"-style + * environments. If the rule doesn't have this attribute, this is considered an + * empty declaration. + * @param compatibilityAttr the name of the attribute that declares "compatible with"-style + * environments. If the rule doesn't have this attribute, this is considered an + * empty declaration. + * @param defaultsProvider provider for the default environments within a group if not + * otherwise overriden by the above attributes + */ + EnvironmentCollector(RuleContext ruleContext, String restrictionAttr, String compatibilityAttr, + DefaultsProvider defaultsProvider) { + this.ruleContext = ruleContext; + this.restrictionAttr = restrictionAttr; + this.compatibilityAttr = compatibilityAttr; + this.defaultsProvider = defaultsProvider; + + EnvironmentCollection.Builder environmentsBuilder = new EnvironmentCollection.Builder(); + restrictionEnvironments = collectRestrictionEnvironments(environmentsBuilder); + compatibilityEnvironments = collectCompatibilityEnvironments(environmentsBuilder); + supportedEnvironments = environmentsBuilder.build(); + } + + /** + * Returns the set of environments supported by this rule, as determined by the + * restriction-style attribute, compatibility-style attribute, and group defaults + * provider instantiated with this class. + */ + EnvironmentCollection getEnvironments() { + return supportedEnvironments; + } + + /** + * Validity-checks that no group has its environment referenced in both the "compatible with" + * and restricted to" attributes. Returns true if all is good, returns false and reports + * appropriate errors if there are any problems. + */ + boolean validateEnvironmentSpecifications() { + ImmutableCollection<EnvironmentGroup> restrictionGroups = restrictionEnvironments.getGroups(); + boolean hasErrors = false; + + for (EnvironmentGroup group : compatibilityEnvironments.getGroups()) { + if (restrictionGroups.contains(group)) { + // To avoid error-spamming the user, when we find a conflict we only report one example + // environment from each attribute for that group. + Label compatibilityEnv = + compatibilityEnvironments.getEnvironments(group).iterator().next(); + Label restrictionEnv = restrictionEnvironments.getEnvironments(group).iterator().next(); + + if (compatibilityEnv.equals(restrictionEnv)) { + ruleContext.attributeError(compatibilityAttr, compatibilityEnv + + " cannot appear both here and in " + restrictionAttr); + } else { + ruleContext.attributeError(compatibilityAttr, compatibilityEnv + " and " + + restrictionEnv + " belong to the same environment group. They should be declared " + + "together either here or in " + restrictionAttr); + } + hasErrors = true; + } + } + + return !hasErrors; + } + + /** + * Adds environments specified in the "restricted to" attribute to the set of supported + * environments and returns the environments added. + */ + private EnvironmentCollection collectRestrictionEnvironments( + EnvironmentCollection.Builder supportedEnvironments) { + return collectEnvironments(restrictionAttr, supportedEnvironments); + } + + /** + * Adds environments specified in the "compatible with" attribute to the set of supported + * environments, along with all defaults from the groups they belong to. Returns these + * environments, not including the defaults. + */ + private EnvironmentCollection collectCompatibilityEnvironments( + EnvironmentCollection.Builder supportedEnvironments) { + EnvironmentCollection compatibilityEnvironments = + collectEnvironments(compatibilityAttr, supportedEnvironments); + for (EnvironmentGroup group : compatibilityEnvironments.getGroups()) { + supportedEnvironments.putAll(group, defaultsProvider.getDefaults(group)); + } + return compatibilityEnvironments; + } + + /** + * Adds environments specified by the given attribute to the set of supported environments + * and returns the environments added. + * + * <p>If this rule doesn't have the given attributes, returns an empty set. + */ + private EnvironmentCollection collectEnvironments(String attrName, + EnvironmentCollection.Builder supportedEnvironments) { + if (!ruleContext.getRule().isAttrDefined(attrName, Type.LABEL_LIST)) { + return EnvironmentCollection.EMPTY; + } + EnvironmentCollection.Builder environments = new EnvironmentCollection.Builder(); + for (TransitiveInfoCollection envTarget : + ruleContext.getPrerequisites(attrName, RuleConfiguredTarget.Mode.DONT_CHECK)) { + EnvironmentWithGroup envInfo = resolveEnvironment(envTarget); + environments.put(envInfo.group(), envInfo.environment()); + supportedEnvironments.put(envInfo.group(), envInfo.environment()); + } + return environments.build(); + } + + /** + * Returns the environment and its group. An {@link Environment} rule only "supports" one + * environment: itself. Extract that from its more generic provider interface and sanity + * check that that's in fact what we see. + */ + private static EnvironmentWithGroup resolveEnvironment(TransitiveInfoCollection envRule) { + SupportedEnvironmentsProvider prereq = + Preconditions.checkNotNull(envRule.getProvider(SupportedEnvironmentsProvider.class)); + return Iterables.getOnlyElement(prereq.getEnvironments().getGroupedEnvironments()); + } + } + + /** + * Returns the set of environments this rule supports, applying the logic described in + * {@link ConstraintSemantics}. + * + * <p>Note this set is <b>not complete</b> - it doesn't include environments from groups we don't + * "know about". Environments and groups can be declared in any package. If the rule includes + * no references to that package, then it simply doesn't know anything about them. But the + * constraint semantics say the rule should support the defaults for that group. We encode this + * implicitly: given the returned set, for any group that's not in the set the rule is also + * considered to support that group's defaults. + * + * @param ruleContext analysis context for the rule. A rule error is triggered here if + * invalid constraint settings are discovered. + * @return the environments this rule supports, not counting defaults "unknown" to this rule + * as described above. Returns null if any errors are encountered. + */ + @Nullable + public static EnvironmentCollection getSupportedEnvironments(RuleContext ruleContext) { + if (!validateAttributes(ruleContext)) { + return null; + } + + // This rule's rule class defaults (or null if the rule class has no defaults). + EnvironmentCollector ruleClassCollector = maybeGetRuleClassDefaults(ruleContext); + // Default environments for this rule. If the rule has rule class defaults, this is + // those defaults. Otherwise it's the global defaults specified by environment_group + // declarations. + DefaultsProvider ruleDefaults; + + if (ruleClassCollector != null) { + if (!ruleClassCollector.validateEnvironmentSpecifications()) { + return null; + } + ruleDefaults = new RuleClassDefaultsProvider(ruleClassCollector.getEnvironments()); + } else { + ruleDefaults = new GroupDefaultsProvider(); + } + + EnvironmentCollector ruleCollector = new EnvironmentCollector(ruleContext, + RuleClass.RESTRICTED_ENVIRONMENT_ATTR, RuleClass.COMPATIBLE_ENVIRONMENT_ATTR, ruleDefaults); + if (!ruleCollector.validateEnvironmentSpecifications()) { + return null; + } + + EnvironmentCollection supportedEnvironments = ruleCollector.getEnvironments(); + if (ruleClassCollector != null) { + // If we have rule class defaults from groups that aren't referenced from the rule itself, + // we need to add them in too to override the global defaults. + supportedEnvironments = + addUnknownGroupsToCollection(supportedEnvironments, ruleClassCollector.getEnvironments()); + } + return supportedEnvironments; + } + + /** + * Returns the rule class defaults specified for this rule, or null if there are + * no such defaults. + */ + @Nullable + private static EnvironmentCollector maybeGetRuleClassDefaults(RuleContext ruleContext) { + Rule rule = ruleContext.getRule(); + String restrictionAttr = RuleClass.DEFAULT_RESTRICTED_ENVIRONMENT_ATTR; + String compatibilityAttr = RuleClass.DEFAULT_COMPATIBLE_ENVIRONMENT_ATTR; + + if (rule.isAttrDefined(restrictionAttr, Type.LABEL_LIST) + || rule.isAttrDefined(compatibilityAttr, Type.LABEL_LIST)) { + return new EnvironmentCollector(ruleContext, restrictionAttr, compatibilityAttr, + new GroupDefaultsProvider()); + } else { + return null; + } + } + + /** + * Adds environments to an {@link EnvironmentCollection} from groups that aren't already + * a part of that collection. + * + * @param environments the collection to add to + * @param toAdd the collection to add. All environments in this collection in groups + * that aren't represented in {@code environments} are added to {@code environments}. + * @return the expanded collection. + */ + private static EnvironmentCollection addUnknownGroupsToCollection( + EnvironmentCollection environments, EnvironmentCollection toAdd) { + EnvironmentCollection.Builder builder = new EnvironmentCollection.Builder(); + builder.putAll(environments); + for (EnvironmentGroup candidateGroup : toAdd.getGroups()) { + if (!environments.getGroups().contains(candidateGroup)) { + builder.putAll(candidateGroup, toAdd.getEnvironments(candidateGroup)); + } + } + return builder.build(); + } + + /** + * Validity-checks this rule's constraint-related attributes. Returns true if all is good, + * returns false and reports appropriate errors if there are any problems. + */ + private static boolean validateAttributes(RuleContext ruleContext) { + AttributeMap attributes = ruleContext.attributes(); + + // Report an error if "restricted to" is explicitly set to nothing. Even if this made + // conceptual sense, we don't know which groups we should apply that to. + String restrictionAttr = RuleClass.RESTRICTED_ENVIRONMENT_ATTR; + List<? extends TransitiveInfoCollection> restrictionEnvironments = ruleContext + .getPrerequisites(restrictionAttr, RuleConfiguredTarget.Mode.DONT_CHECK); + if (restrictionEnvironments.isEmpty() + && attributes.isAttributeValueExplicitlySpecified(restrictionAttr)) { + ruleContext.attributeError(restrictionAttr, "attribute cannot be empty"); + return false; + } + + return true; + } + + /** + * Performs constraint checking on the given rule's dependencies and reports any errors. + * + * @param ruleContext the rule to analyze + * @param supportedEnvironments the rule's supported environments, as defined by the return + * value of {@link #getSupportedEnvironments}. In particular, for any environment group that's + * not in this collection, the rule is assumed to support the defaults for that group. + */ + public static void checkConstraints(RuleContext ruleContext, + EnvironmentCollection supportedEnvironments) { + + Set<EnvironmentGroup> knownGroups = supportedEnvironments.getGroups(); + + for (TransitiveInfoCollection dependency : getAllPrerequisites(ruleContext)) { + SupportedEnvironmentsProvider depProvider = + dependency.getProvider(SupportedEnvironmentsProvider.class); + if (depProvider == null) { + // Input files (InputFileConfiguredTarget) don't support environments. We may subsequently + // opt them into constraint checking, but for now just pass them by. + continue; + } + Collection<Label> depEnvironments = depProvider.getEnvironments().getEnvironments(); + Set<EnvironmentGroup> groupsKnownToDep = depProvider.getEnvironments().getGroups(); + + // Environments we support that the dependency does not support. + Set<Label> disallowedEnvironments = new LinkedHashSet<>(); + + // For every environment we support, either the dependency must also support it OR it must be + // a default for a group the dependency doesn't know about. + for (EnvironmentWithGroup supportedEnv : supportedEnvironments.getGroupedEnvironments()) { + EnvironmentGroup group = supportedEnv.group(); + Label environment = supportedEnv.environment(); + if (!depEnvironments.contains(environment) + && (groupsKnownToDep.contains(group) || !group.isDefault(environment))) { + disallowedEnvironments.add(environment); + } + } + + // For any environment group we don't know about, we implicitly support its defaults. Check + // that the dep does, too. + for (EnvironmentGroup depGroup : groupsKnownToDep) { + if (!knownGroups.contains(depGroup)) { + for (Label defaultEnv : depGroup.getDefaults()) { + if (!depEnvironments.contains(defaultEnv)) { + disallowedEnvironments.add(defaultEnv); + } + } + } + } + + // Report errors on bad environments. + if (!disallowedEnvironments.isEmpty()) { + ruleContext.ruleError("dependency " + dependency.getLabel() + + " doesn't support expected environment" + + (disallowedEnvironments.size() == 1 ? "" : "s") + + ": " + Joiner.on(", ").join(disallowedEnvironments)); + } + } + } + + /** + * Returns all dependencies that should be constraint-checked against the current rule. + */ + private static Iterable<TransitiveInfoCollection> getAllPrerequisites(RuleContext ruleContext) { + Set<TransitiveInfoCollection> prerequisites = new LinkedHashSet<>(); + AttributeMap attributes = ruleContext.attributes(); + + for (String attr : attributes.getAttributeNames()) { + Type<?> attrType = attributes.getAttributeType(attr); + // TODO(bazel-team): support specifying which attributes are subject to constraint checking + if ((attrType == Type.LABEL || attrType == Type.LABEL_LIST) + && !RuleClass.isConstraintAttribute(attr) + && !attr.equals("visibility")) { + prerequisites.addAll( + ruleContext.getPrerequisites(attr, RuleConfiguredTarget.Mode.DONT_CHECK)); + } + } + return prerequisites; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/constraints/Environment.java b/src/main/java/com/google/devtools/build/lib/analysis/constraints/Environment.java new file mode 100644 index 0000000..912ed72 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/constraints/Environment.java
@@ -0,0 +1,71 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.constraints; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.EnvironmentGroup; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.syntax.Label; + +/** + * Implementation for the environment rule. + */ +public class Environment implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + + // The main analysis work to do here is to simply fill in SupportedEnvironmentsProvider to + // pass the environment itself to depending rules. + // + // This will likely expand when we add support for environments fulfilling other environments. + Label label = ruleContext.getLabel(); + Package pkg = ruleContext.getRule().getPackage(); + + EnvironmentGroup group = null; + for (EnvironmentGroup pkgGroup : pkg.getTargets(EnvironmentGroup.class)) { + if (pkgGroup.getEnvironments().contains(label)) { + group = pkgGroup; + break; + } + } + + if (group == null) { + ruleContext.ruleError("no matching environment group from the same package"); + return null; + } + + return new RuleConfiguredTargetBuilder(ruleContext) + .addProvider(SupportedEnvironmentsProvider.class, + new SupportedEnvironments( + new EnvironmentCollection.Builder().put(group, label).build())) + .addProvider(RunfilesProvider.class, RunfilesProvider.EMPTY) + .add(FileProvider.class, new FileProvider(ruleContext.getLabel(), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER))) + .add(FilesToRunProvider.class, new FilesToRunProvider(ruleContext.getLabel(), + ImmutableList.<Artifact>of(), null, null)) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/constraints/EnvironmentCollection.java b/src/main/java/com/google/devtools/build/lib/analysis/constraints/EnvironmentCollection.java new file mode 100644 index 0000000..1ce5f1c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/constraints/EnvironmentCollection.java
@@ -0,0 +1,126 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.constraints; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.packages.EnvironmentGroup; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Map; + +/** + * Contains a set of {@link Environment} labels and their associated groups. + */ +@Immutable +public class EnvironmentCollection { + private final ImmutableMultimap<EnvironmentGroup, Label> map; + + private EnvironmentCollection(ImmutableMultimap<EnvironmentGroup, Label> map) { + this.map = map; + } + + /** + * Stores an environment's build label along with the group it belongs to. + */ + static class EnvironmentWithGroup { + private final Label environment; + private final EnvironmentGroup group; + EnvironmentWithGroup(Label environment, EnvironmentGroup group) { + this.environment = environment; + this.group = group; + } + Label environment() { return environment; } + EnvironmentGroup group() { return group; } + } + + /** + * Returns the build labels of each environment in this collection, ordered by + * their insertion order in {@link Builder}. + */ + ImmutableCollection<Label> getEnvironments() { + return map.values(); + } + + /** + * Returns the set of groups the environments in this collection belong to, ordered by + * their insertion order in {@link Builder} + */ + ImmutableSet<EnvironmentGroup> getGroups() { + return map.keySet(); + } + + /** + * Returns the build labels of each environment in this collection paired with the + * group each environment belongs to, ordered by their insertion order in {@link Builder}. + */ + ImmutableCollection<EnvironmentWithGroup> getGroupedEnvironments() { + ImmutableSet.Builder<EnvironmentWithGroup> builder = ImmutableSet.builder(); + for (Map.Entry<EnvironmentGroup, Label> entry : map.entries()) { + builder.add(new EnvironmentWithGroup(entry.getValue(), entry.getKey())); + } + return builder.build(); + } + + /** + * Returns the environments in this collection that belong to the given group, ordered by + * their insertion order in {@link Builder}. If no environments belong to the given group, + * returns an empty collection. + */ + ImmutableCollection<Label> getEnvironments(EnvironmentGroup group) { + return map.get(group); + } + + /** + * An empty collection. + */ + static final EnvironmentCollection EMPTY = + new EnvironmentCollection(ImmutableMultimap.<EnvironmentGroup, Label>of()); + + static class Builder { + private final ImmutableMultimap.Builder<EnvironmentGroup, Label> mapBuilder = + ImmutableMultimap.builder(); + + /** + * Inserts the given environment / owning group pair. + */ + Builder put(EnvironmentGroup group, Label environment) { + mapBuilder.put(group, environment); + return this; + } + + /** + * Inserts the given set of environments, all belonging to the specified group. + */ + Builder putAll(EnvironmentGroup group, Iterable<Label> environments) { + mapBuilder.putAll(group, environments); + return this; + } + + /** + * Inserts the contents of another {@link EnvironmentCollection} into this one. + */ + Builder putAll(EnvironmentCollection other) { + mapBuilder.putAll(other.map); + return this; + } + + EnvironmentCollection build() { + return new EnvironmentCollection(mapBuilder.build()); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/constraints/EnvironmentRule.java b/src/main/java/com/google/devtools/build/lib/analysis/constraints/EnvironmentRule.java new file mode 100644 index 0000000..0553af0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/constraints/EnvironmentRule.java
@@ -0,0 +1,48 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.constraints; + +import static com.google.devtools.build.lib.packages.Attribute.attr; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.Type; + +/** + * Rule definition for environment rules (for Bazel's constraint enforcement system). + */ +@BlazeRule(name = EnvironmentRule.RULE_NAME, + ancestors = { BaseRuleClasses.BaseRule.class }, + factoryClass = Environment.class) +public final class EnvironmentRule implements RuleDefinition { + public static final String RULE_NAME = "environment"; + + @Override + public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) { + return builder + .override(attr("tags", Type.STRING_LIST) + // No need to show up in ":all", etc. target patterns. + .value(ImmutableList.of("manual")) + .nonconfigurable("low-level attribute, used in TargetUtils without configurations")) + .removeAttribute(RuleClass.COMPATIBLE_ENVIRONMENT_ATTR) + .removeAttribute(RuleClass.RESTRICTED_ENVIRONMENT_ATTR) + .setUndocumented() + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/constraints/SupportedEnvironments.java b/src/main/java/com/google/devtools/build/lib/analysis/constraints/SupportedEnvironments.java new file mode 100644 index 0000000..78c835e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/constraints/SupportedEnvironments.java
@@ -0,0 +1,31 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.constraints; + +/** + * Standard {@link SupportedEnvironmentsProvider} implementation. + */ +public class SupportedEnvironments implements SupportedEnvironmentsProvider { + private final EnvironmentCollection supportedEnvironments; + + public SupportedEnvironments(EnvironmentCollection supportedEnvironments) { + this.supportedEnvironments = supportedEnvironments; + } + + @Override + public EnvironmentCollection getEnvironments() { + return supportedEnvironments; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/constraints/SupportedEnvironmentsProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/constraints/SupportedEnvironmentsProvider.java new file mode 100644 index 0000000..8200386 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/constraints/SupportedEnvironmentsProvider.java
@@ -0,0 +1,29 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.analysis.constraints; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; + +/** + * A provider that advertises which environments the associated target is compatible with + * (from the point of view of the constraint enforcement system). + */ +public interface SupportedEnvironmentsProvider extends TransitiveInfoProvider { + + /** + * Returns the environments the associated target is compatible with. + */ + EnvironmentCollection getEnvironments(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelDiffAwarenessModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelDiffAwarenessModule.java new file mode 100644 index 0000000..1dad1f5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelDiffAwarenessModule.java
@@ -0,0 +1,34 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.skyframe.DiffAwareness; +import com.google.devtools.build.lib.skyframe.LocalDiffAwareness; + +/** + * Provides the {@link DiffAwareness} implementation that uses the Java watch service. + */ +public class BazelDiffAwarenessModule extends BlazeModule { + + @Override + public Iterable<DiffAwareness.Factory> getDiffAwarenessFactories(boolean watchFS) { + ImmutableList.Builder<DiffAwareness.Factory> builder = ImmutableList.builder(); + if (watchFS) { + builder.add(new LocalDiffAwareness.Factory()); + } + return builder.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java new file mode 100644 index 0000000..fd3d000 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; + +import java.util.List; + +/** + * The main class. + */ +public final class BazelMain { + private static final List<Class<? extends BlazeModule>> BAZEL_MODULES = ImmutableList.of( + com.google.devtools.build.lib.bazel.BazelShutdownLoggerModule.class, + com.google.devtools.build.lib.bazel.BazelWorkspaceStatusModule.class, + com.google.devtools.build.lib.bazel.BazelDiffAwarenessModule.class, + com.google.devtools.build.lib.bazel.BazelRepositoryModule.class, + com.google.devtools.build.lib.bazel.rules.BazelRulesModule.class, + com.google.devtools.build.lib.standalone.StandaloneModule.class, + com.google.devtools.build.lib.runtime.BuildSummaryStatsModule.class, + com.google.devtools.build.lib.webstatusserver.WebStatusServerModule.class + ); + + public static void main(String[] args) { + BlazeRuntime.main(BAZEL_MODULES, args); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java new file mode 100644 index 0000000..483103f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -0,0 +1,100 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.bazel.repository.HttpArchiveFunction; +import com.google.devtools.build.lib.bazel.repository.HttpJarFunction; +import com.google.devtools.build.lib.bazel.repository.LocalRepositoryFunction; +import com.google.devtools.build.lib.bazel.repository.MavenJarFunction; +import com.google.devtools.build.lib.bazel.repository.NewLocalRepositoryFunction; +import com.google.devtools.build.lib.bazel.repository.RepositoryDelegatorFunction; +import com.google.devtools.build.lib.bazel.repository.RepositoryFunction; +import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule; +import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule; +import com.google.devtools.build.lib.bazel.rules.workspace.LocalRepositoryRule; +import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule; +import com.google.devtools.build.lib.bazel.rules.workspace.NewLocalRepositoryRule; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.Map.Entry; +import java.util.Set; +import java.util.UUID; + +/** + * Adds support for fetching external code. + */ +public class BazelRepositoryModule extends BlazeModule { + + private BlazeDirectories directories; + // A map of repository handlers that can be looked up by rule class name. + private final ImmutableMap<String, RepositoryFunction> repositoryHandlers; + + public BazelRepositoryModule() { + repositoryHandlers = ImmutableMap.of( + LocalRepositoryRule.NAME, new LocalRepositoryFunction(), + HttpArchiveRule.NAME, new HttpArchiveFunction(), + HttpJarRule.NAME, new HttpJarFunction(), + MavenJarRule.NAME, new MavenJarFunction(), + NewLocalRepositoryRule.NAME, new NewLocalRepositoryFunction()); + } + + @Override + public void blazeStartup(OptionsProvider startupOptions, + BlazeVersionInfo versionInfo, UUID instanceId, BlazeDirectories directories, + Clock clock) { + this.directories = directories; + for (RepositoryFunction handler : repositoryHandlers.values()) { + handler.setDirectories(directories); + } + } + + @Override + public Set<Path> getImmutableDirectories() { + return ImmutableSet.of(RepositoryFunction.getExternalRepositoryDirectory(directories)); + } + + @Override + public void initializeRuleClasses(ConfiguredRuleClassProvider.Builder builder) { + for (Entry<String, RepositoryFunction> handler : repositoryHandlers.entrySet()) { + builder.addRuleDefinition(handler.getValue().getRuleDefinition()); + } + } + + @Override + public ImmutableMap<SkyFunctionName, SkyFunction> getSkyFunctions(BlazeDirectories directories) { + ImmutableMap.Builder<SkyFunctionName, SkyFunction> builder = ImmutableMap.builder(); + + // Bazel-specific repository downloaders. + for (RepositoryFunction handler : repositoryHandlers.values()) { + builder.put(handler.getSkyFunctionName(), handler); + } + + // Create the delegator everything flows through. + builder.put(SkyFunctions.REPOSITORY, + new RepositoryDelegatorFunction(repositoryHandlers)); + return builder.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelShutdownLoggerModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelShutdownLoggerModule.java new file mode 100644 index 0000000..3c32611 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelShutdownLoggerModule.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel; + +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.UUID; +import java.util.logging.LogManager; +import java.util.logging.Logger; + +/** + * Shutdown log output when Bazel runs in batch mode + */ +public class BazelShutdownLoggerModule extends BlazeModule { + + private Logger globalLogger; + + @Override + public void blazeStartup(OptionsProvider startupOptions, BlazeVersionInfo versionInfo, + UUID instanceId, BlazeDirectories directories, Clock clock) { + LogManager.getLogManager().reset(); + globalLogger = Logger.getGlobal(); + globalLogger.setLevel(java.util.logging.Level.OFF); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java new file mode 100644 index 0000000..dda983d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelWorkspaceStatusModule.java
@@ -0,0 +1,196 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel; + +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.ActionContextProvider; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ExecutorInitException; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.BuildInfoHelper; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Key; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; + +/** + * Workspace status information for Bazel. + * + * <p>Currently only a stub. + */ +public class BazelWorkspaceStatusModule extends BlazeModule { + private static class BazelWorkspaceStatusAction extends WorkspaceStatusAction { + private final Artifact stableStatus; + private final Artifact volatileStatus; + + private BazelWorkspaceStatusAction( + Artifact stableStatus, Artifact volatileStatus) { + super(BuildInfoHelper.BUILD_INFO_ACTION_OWNER, Artifact.NO_ARTIFACTS, + ImmutableList.of(stableStatus, volatileStatus)); + this.stableStatus = stableStatus; + this.volatileStatus = volatileStatus; + } + + @Override + public String describeStrategy(Executor executor) { + return ""; + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException { + try { + FileSystemUtils.writeContent(stableStatus.getPath(), new byte[] {}); + FileSystemUtils.writeContent(volatileStatus.getPath(), new byte[] {}); + } catch (IOException e) { + throw new ActionExecutionException(e, this, true); + } + } + + // TODO(bazel-team): Add test for equals, add hashCode. + @Override + public boolean equals(Object o) { + if (!(o instanceof BazelWorkspaceStatusAction)) { + return false; + } + + BazelWorkspaceStatusAction that = (BazelWorkspaceStatusAction) o; + return this.stableStatus.equals(that.stableStatus) + && this.volatileStatus.equals(that.volatileStatus); + } + + @Override + public int hashCode() { + return Objects.hash(stableStatus, volatileStatus); + } + + @Override + public String getMnemonic() { + return "BazelWorkspaceStatusAction"; + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + + @Override + protected String computeKey() { + return ""; + } + + @Override + public Artifact getVolatileStatus() { + return volatileStatus; + } + + @Override + public Artifact getStableStatus() { + return stableStatus; + } + } + + private class BazelStatusActionFactory implements WorkspaceStatusAction.Factory { + @Override + public Map<String, String> createDummyWorkspaceStatus() { + return ImmutableMap.of(); + } + + @Override + public WorkspaceStatusAction createWorkspaceStatusAction( + ArtifactFactory factory, ArtifactOwner artifactOwner, Supplier<UUID> buildId) { + Root root = runtime.getDirectories().getBuildDataDirectory(); + + Artifact stableArtifact = factory.getDerivedArtifact( + new PathFragment("stable-status.txt"), root, artifactOwner); + Artifact volatileArtifact = factory.getConstantMetadataArtifact( + new PathFragment("volatile-status.txt"), root, artifactOwner); + + return new BazelWorkspaceStatusAction(stableArtifact, volatileArtifact); + } + } + + @ExecutionStrategy(contextType = WorkspaceStatusAction.Context.class) + private class BazelWorkspaceStatusActionContext implements WorkspaceStatusAction.Context { + @Override + public ImmutableMap<String, Key> getStableKeys() { + return ImmutableMap.of(); + } + + @Override + public ImmutableMap<String, Key> getVolatileKeys() { + return ImmutableMap.of(); + } + } + + + private class WorkspaceActionContextProvider implements ActionContextProvider { + @Override + public Iterable<ActionContext> getActionContexts() { + return ImmutableList.<ActionContext>of(new BazelWorkspaceStatusActionContext()); + } + + @Override + public void executorCreated(Iterable<ActionContext> usedContexts) + throws ExecutorInitException { + } + + @Override + public void executionPhaseEnding() { + } + + @Override + public void executionPhaseStarting(ActionInputFileCache actionInputFileCache, + ActionGraph actionGraph, Iterable<Artifact> topLevelArtifacts) throws ExecutorInitException, + InterruptedException { + } + } + + private BlazeRuntime runtime; + + @Override + public void beforeCommand(BlazeRuntime runtime, Command command) { + this.runtime = runtime; + } + + @Override + public ActionContextProvider getActionContextProvider() { + return new WorkspaceActionContextProvider(); + } + + @Override + public WorkspaceStatusAction.Factory getWorkspaceStatusActionFactory() { + return new BazelStatusActionFactory(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorFactory.java new file mode 100644 index 0000000..1076f24 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorFactory.java
@@ -0,0 +1,218 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule; +import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.ArchiveStreamFactory; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.utils.IOUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.Charset; + +/** + * Creates decompressors to use on archive. Use {@link DecompressorFactory#create} to get the + * correct type of decompressor for the input archive, then call + * {@link Decompressor#decompress} to decompress it. + */ +public abstract class DecompressorFactory { + + + public static Decompressor create(Target target, Path archivePath) + throws DecompressorException { + String baseName = archivePath.getBaseName(); + + if (target.getTargetKind().startsWith(HttpJarRule.NAME + " ")) { + if (baseName.endsWith(".jar")) { + return new JarDecompressor(target, archivePath); + } else { + throw new DecompressorException( + "Expected " + HttpJarRule.NAME + " " + target.getName() + + " to create file with a .jar suffix (got " + archivePath + ")"); + } + } + + if (target.getTargetKind().startsWith(HttpArchiveRule.NAME + " ")) { + if (baseName.endsWith(".zip") || baseName.endsWith(".jar")) { + return new ZipDecompressor(archivePath); + } else { + throw new DecompressorException( + "Expected " + HttpArchiveRule.NAME + " " + target.getName() + + " to create file with a .zip or .jar suffix (got " + archivePath + ")"); + } + } + + throw new DecompressorException( + "No decompressor found for " + target.getTargetKind() + " rule " + target.getName() + + " (got " + archivePath + ")"); + } + + /** + * General decompressor for an archive. Should be overridden for each specific archive type. + */ + public abstract static class Decompressor { + protected final Path archiveFile; + + private Decompressor(Path archiveFile) { + this.archiveFile = archiveFile; + } + + /** + * This is overridden by archive-specific decompression logic. Often this logic will create + * files and directories under the {@link Decompressor#archiveFile}'s parent directory. + * + * @return the path to the repository directory. That is, the returned path will be a directory + * containing a WORKSPACE file. + */ + public abstract Path decompress() throws DecompressorException; + } + + static class JarDecompressor extends Decompressor { + private final Target target; + + public JarDecompressor(Target target, Path archiveFile) { + super(archiveFile); + this.target = target; + } + + /** + * The .jar can be used compressed, so this just exposes it in a way Bazel can use. + * + * <p>It moves the jar from some-name/foo.jar to some-name/repository/jar/foo.jar and creates a + * BUILD file containing one entry: a .jar. + */ + @Override + public Path decompress() throws DecompressorException { + Path destinationDirectory = archiveFile.getParentDirectory().getRelative("repository"); + // Example: archiveFile is .external-repository/some-name/foo.jar. + String baseName = archiveFile.getBaseName(); + + try { + FileSystemUtils.createDirectoryAndParents(destinationDirectory); + // .external-repository/some-name/repository/WORKSPACE. + Path workspaceFile = destinationDirectory.getRelative("WORKSPACE"); + FileSystemUtils.writeContent(workspaceFile, Charset.forName("UTF-8"), + "# DO NOT EDIT: automatically generated WORKSPACE file for " + target.getTargetKind() + + " rule " + target.getName()); + // .external-repository/some-name/repository/jar. + Path jarDirectory = destinationDirectory.getRelative("jar"); + FileSystemUtils.createDirectoryAndParents(jarDirectory); + // .external-repository/some-name/repository/jar/foo.jar is a symbolic link to the jar in + // .external-repository/some-name. + Path jarSymlink = jarDirectory.getRelative(baseName); + if (!jarSymlink.exists()) { + jarSymlink.createSymbolicLink(archiveFile); + } + // .external-repository/some-name/repository/jar/BUILD defines the //jar target. + Path buildFile = jarDirectory.getRelative("BUILD"); + FileSystemUtils.writeLinesAs(buildFile, Charset.forName("UTF-8"), + "# DO NOT EDIT: automatically generated BUILD file for " + target.getTargetKind() + + " rule " + target.getName(), + "java_import(", + " name = 'jar',", + " jars = ['" + baseName + "'],", + " visibility = ['//visibility:public']", + ")"); + } catch (IOException e) { + throw new DecompressorException(e.getMessage()); + } + return destinationDirectory; + } + } + + private static class ZipDecompressor extends Decompressor { + public ZipDecompressor(Path archiveFile) { + super(archiveFile); + } + + /** + * This unzips the zip file to a sibling directory of {@link Decompressor#archiveFile}. The + * zip file is expected to have the WORKSPACE file at the top level, e.g.: + * + * <pre> + * $ unzip -lf some-repo.zip + * Archive: ../repo.zip + * Length Date Time Name + * --------- ---------- ----- ---- + * 0 2014-11-20 15:50 WORKSPACE + * 0 2014-11-20 16:10 foo/ + * 236 2014-11-20 15:52 foo/BUILD + * ... + * </pre> + */ + @Override + public Path decompress() throws DecompressorException { + Path destinationDirectory = archiveFile.getParentDirectory().getRelative("repository"); + try (InputStream is = new FileInputStream(archiveFile.getPathString())) { + ArchiveInputStream in = new ArchiveStreamFactory().createArchiveInputStream( + ArchiveStreamFactory.ZIP, is); + ZipArchiveEntry entry = (ZipArchiveEntry) in.getNextEntry(); + while (entry != null) { + extractZipEntry(in, entry, destinationDirectory); + entry = (ZipArchiveEntry) in.getNextEntry(); + } + } catch (IOException | ArchiveException e) { + throw new DecompressorException( + "Error extracting " + archiveFile + " to " + destinationDirectory + ": " + + e.getMessage()); + } + return destinationDirectory; + } + + private void extractZipEntry( + ArchiveInputStream in, ZipArchiveEntry entry, Path destinationDirectory) + throws IOException, DecompressorException { + PathFragment relativePath = new PathFragment(entry.getName()); + if (relativePath.isAbsolute()) { + throw new DecompressorException("Failed to extract " + relativePath + + ", zipped paths cannot be absolute"); + } + Path outputPath = destinationDirectory.getRelative(relativePath); + FileSystemUtils.createDirectoryAndParents(outputPath.getParentDirectory()); + if (entry.isDirectory()) { + FileSystemUtils.createDirectoryAndParents(outputPath); + } else { + try (OutputStream out = new FileOutputStream(new File(outputPath.getPathString()))) { + IOUtils.copy(in, out); + } catch (IOException e) { + throw new DecompressorException("Error writing " + outputPath + " from " + + archiveFile); + } + } + } + } + + /** + * Exceptions thrown when something goes wrong decompressing an archive. + */ + public static class DecompressorException extends Exception { + public DecompressorException(String message) { + super(message); + } + } +} \ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java new file mode 100644 index 0000000..1cd6db8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java
@@ -0,0 +1,112 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.bazel.repository.DecompressorFactory.DecompressorException; +import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule; +import com.google.devtools.build.lib.packages.AggregatingAttributeMapper; +import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.skyframe.FileValue; +import com.google.devtools.build.lib.skyframe.RepositoryValue; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; + +/** + * Downloads a file over HTTP. + */ +public class HttpArchiveFunction extends RepositoryFunction { + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + RepositoryName repositoryName = (RepositoryName) skyKey.argument(); + Rule rule = RepositoryFunction.getRule(repositoryName, HttpArchiveRule.NAME, env); + if (rule == null) { + return null; + } + + return compute(env, rule); + } + + protected FileValue createOutputDirectory(Environment env, String repositoryName) + throws RepositoryFunctionException { + // The output directory is always under .external-repository (to stay out of the way of + // artifacts from this repository) and uses the rule's name to avoid conflicts with other + // remote repository rules. For example, suppose you had the following WORKSPACE file: + // + // http_archive(name = "png", url = "http://example.com/downloads/png.tar.gz", sha256 = "...") + // + // This would download png.tar.gz to .external-repository/png/png.tar.gz. + Path outputDirectory = getExternalRepositoryDirectory().getRelative(repositoryName); + try { + FileSystemUtils.createDirectoryAndParents(outputDirectory); + } catch (IOException e) { + throw new RepositoryFunctionException(e, Transience.TRANSIENT); + } + return getRepositoryDirectory(outputDirectory, env); + } + + protected SkyValue compute(Environment env, Rule rule) + throws RepositoryFunctionException { + FileValue directoryValue = createOutputDirectory(env, rule.getName()); + if (directoryValue == null) { + return null; + } + Path outputDirectory = directoryValue.realRootedPath().asPath(); + AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule); + URL url = null; + try { + url = new URL(mapper.get("url", Type.STRING)); + } catch (MalformedURLException e) { + throw new RepositoryFunctionException( + new EvalException(rule.getLocation(), "Error parsing URL: " + e.getMessage()), + Transience.PERSISTENT); + } + String sha256 = mapper.get("sha256", Type.STRING); + HttpDownloader downloader = new HttpDownloader(url, sha256, outputDirectory); + try { + Path archiveFile = downloader.download(); + outputDirectory = DecompressorFactory.create(rule, archiveFile).decompress(); + } catch (IOException e) { + // Assumes all IO errors transient. + throw new RepositoryFunctionException(e, Transience.TRANSIENT); + } catch (DecompressorException e) { + throw new RepositoryFunctionException(new IOException(e.getMessage()), Transience.TRANSIENT); + } + return new RepositoryValue(outputDirectory, directoryValue); + } + + @Override + public SkyFunctionName getSkyFunctionName() { + return SkyFunctionName.computed(HttpArchiveRule.NAME.toUpperCase()); + } + + @Override + public Class<? extends RuleDefinition> getRuleDefinition() { + return HttpArchiveRule.class; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloader.java new file mode 100644 index 0000000..0f9ff44 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloader.java
@@ -0,0 +1,107 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; + +/** + * Helper class for downloading a file from a URL. + */ +public class HttpDownloader { + private static final int BUFFER_SIZE = 2048; + + private final URL url; + private final String sha256; + private final Path outputDirectory; + + HttpDownloader(URL url, String sha256, Path outputDirectory) { + this.url = url; + this.sha256 = sha256; + this.outputDirectory = outputDirectory; + } + + /** + * Attempt to download a file from the repository's URL. Returns the path to the file downloaded. + */ + public Path download() throws IOException { + String filename = new PathFragment(url.getPath()).getBaseName(); + if (filename.isEmpty()) { + filename = "temp"; + } + Path destination = outputDirectory.getRelative(filename); + + try (OutputStream outputStream = destination.getOutputStream()) { + ReadableByteChannel rbc = getChannel(url); + ByteBuffer byteBuffer = ByteBuffer.allocate(BUFFER_SIZE); + while (rbc.read(byteBuffer) > 0) { + byteBuffer.flip(); + while (byteBuffer.hasRemaining()) { + outputStream.write(byteBuffer.get()); + } + } + } catch (IOException e) { + throw new IOException( + "Error downloading " + url + " to " + destination + ": " + e.getMessage()); + } + + try { + String downloadedSha256 = getSha256(destination); + if (!downloadedSha256.equals(sha256)) { + throw new IOException( + "Downloaded file at " + destination + " has SHA-256 of " + downloadedSha256 + + ", does not match expected SHA-256 (" + sha256 + ")"); + } + } catch (IOException e) { + throw new IOException( + "Could not hash file " + destination + ": " + e.getMessage() + ", expected SHA-256 of " + + sha256 + ")"); + } + return destination; + } + + @VisibleForTesting + protected ReadableByteChannel getChannel(URL url) throws IOException { + return Channels.newChannel(url.openStream()); + } + + private String getSha256(Path path) throws IOException { + Hasher hasher = Hashing.sha256().newHasher(); + + byte byteBuffer[] = new byte[BUFFER_SIZE]; + try (InputStream stream = path.getInputStream()) { + int numBytesRead = stream.read(byteBuffer); + while (numBytesRead != -1) { + if (numBytesRead != 0) { + // If more than 0 bytes were read, add them to the hash. + hasher.putBytes(byteBuffer, 0, numBytesRead); + } + numBytesRead = stream.read(byteBuffer); + } + } + return hasher.hash().toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpJarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpJarFunction.java new file mode 100644 index 0000000..56e5e55 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpJarFunction.java
@@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule; +import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * Downloads a jar file from a URL. + */ +public class HttpJarFunction extends HttpArchiveFunction { + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + RepositoryName repositoryName = (RepositoryName) skyKey.argument(); + Rule rule = RepositoryFunction.getRule(repositoryName, HttpJarRule.NAME, env); + if (rule == null) { + return null; + } + return compute(env, rule); + } + + @Override + public SkyFunctionName getSkyFunctionName() { + return SkyFunctionName.computed(HttpJarRule.NAME.toUpperCase()); + } + + @Override + public Class<? extends RuleDefinition> getRuleDefinition() { + return HttpJarRule.class; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalRepositoryFunction.java new file mode 100644 index 0000000..1a72dad --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalRepositoryFunction.java
@@ -0,0 +1,82 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.bazel.rules.workspace.LocalRepositoryRule; +import com.google.devtools.build.lib.packages.AggregatingAttributeMapper; +import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.skyframe.FileValue; +import com.google.devtools.build.lib.skyframe.RepositoryValue; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; + +/** + * Access a repository on the local filesystem. + */ +public class LocalRepositoryFunction extends RepositoryFunction { + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + RepositoryName repositoryName = (RepositoryName) skyKey.argument(); + Rule rule = RepositoryFunction.getRule(repositoryName, LocalRepositoryRule.NAME, env); + if (rule == null) { + return null; + } + + AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule); + String path = mapper.get("path", Type.STRING); + PathFragment pathFragment = new PathFragment(path); + if (!pathFragment.isAbsolute()) { + throw new RepositoryFunctionException( + new EvalException( + rule.getLocation(), + "In " + rule + " the 'path' attribute must specify an absolute path"), + Transience.PERSISTENT); + } + Path repositoryPath = getOutputBase().getFileSystem().getPath(pathFragment); + FileValue repositoryValue = getRepositoryDirectory(repositoryPath, env); + if (repositoryValue == null) { + return null; + } + + if (!repositoryValue.isDirectory()) { + throw new RepositoryFunctionException( + new IOException(rule + " must specify an existing directory"), Transience.TRANSIENT); + } + + return new RepositoryValue(repositoryPath, repositoryValue); + } + + @Override + public SkyFunctionName getSkyFunctionName() { + return SkyFunctionName.computed(LocalRepositoryRule.NAME.toUpperCase()); + } + + @Override + public Class<? extends RuleDefinition> getRuleDefinition() { + return LocalRepositoryRule.class; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java new file mode 100644 index 0000000..4f83de6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java
@@ -0,0 +1,189 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.common.base.Ascii; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.bazel.repository.DecompressorFactory.DecompressorException; +import com.google.devtools.build.lib.bazel.repository.DecompressorFactory.JarDecompressor; +import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule; +import com.google.devtools.build.lib.packages.AggregatingAttributeMapper; +import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.skyframe.FileValue; +import com.google.devtools.build.lib.skyframe.RepositoryValue; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import org.apache.maven.repository.internal.MavenRepositorySystemUtils; +import org.eclipse.aether.AbstractRepositoryListener; +import org.eclipse.aether.DefaultRepositorySystemSession; +import org.eclipse.aether.RepositorySystem; +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.artifact.DefaultArtifact; +import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory; +import org.eclipse.aether.impl.DefaultServiceLocator; +import org.eclipse.aether.repository.LocalRepository; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.resolution.ArtifactRequest; +import org.eclipse.aether.resolution.ArtifactResolutionException; +import org.eclipse.aether.resolution.ArtifactResult; +import org.eclipse.aether.spi.connector.RepositoryConnectorFactory; +import org.eclipse.aether.spi.connector.transport.TransporterFactory; +import org.eclipse.aether.transfer.AbstractTransferListener; +import org.eclipse.aether.transport.file.FileTransporterFactory; +import org.eclipse.aether.transport.http.HttpTransporterFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Implementation of maven_jar. + */ +public class MavenJarFunction extends HttpJarFunction { + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws RepositoryFunctionException { + RepositoryName repositoryName = (RepositoryName) skyKey.argument(); + Rule rule = RepositoryFunction.getRule(repositoryName, MavenJarRule.NAME, env); + if (rule == null) { + return null; + } + + AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule); + FileValue outputDirectoryValue = createOutputDirectory(env, rule.getName()); + if (outputDirectoryValue == null) { + return null; + } + Path outputDirectory = outputDirectoryValue.realRootedPath().asPath(); + MavenDownloader downloader = new MavenDownloader( + mapper.get("group_id", Type.STRING), + mapper.get("artifact_id", Type.STRING), + mapper.get("version", Type.STRING), + outputDirectory); + + List<String> repositories = mapper.get("repositories", Type.STRING_LIST); + if (repositories != null && !repositories.isEmpty()) { + downloader.setRepositories(repositories); + } + + Path repositoryJar = null; + try { + repositoryJar = downloader.download(); + } catch (IOException e) { + throw new RepositoryFunctionException(e, Transience.TRANSIENT); + } + + // Add a WORKSPACE file & BUILD file to the Maven jar. + JarDecompressor decompressor = new JarDecompressor(rule, repositoryJar); + Path repositoryDirectory = null; + try { + repositoryDirectory = decompressor.decompress(); + } catch (DecompressorException e) { + throw new RepositoryFunctionException(new IOException(e.getMessage()), Transience.TRANSIENT); + } + FileValue repositoryFileValue = getRepositoryDirectory(repositoryDirectory, env); + if (repositoryFileValue == null) { + return null; + } + return new RepositoryValue(repositoryDirectory, repositoryFileValue); + } + + @Override + public SkyFunctionName getSkyFunctionName() { + return SkyFunctionName.computed(Ascii.toUpperCase(MavenJarRule.NAME)); + } + + @Override + public Class<? extends RuleDefinition> getRuleDefinition() { + return MavenJarRule.class; + } + + private static class MavenDownloader { + private static final String MAVEN_CENTRAL_URL = "http://central.maven.org/maven2/"; + + private final String groupId; + private final String artifactId; + private final String version; + private final Path outputDirectory; + private List<RemoteRepository> repositories; + + MavenDownloader(String groupId, String artifactId, String version, Path outputDirectory) { + this.groupId = groupId; + this.artifactId = artifactId; + this.version = version; + this.outputDirectory = outputDirectory; + + this.repositories = new ArrayList<>(Arrays.asList( + new RemoteRepository.Builder("central", "default", MAVEN_CENTRAL_URL) + .build())); + } + + /** + * Customizes the set of Maven repositories to check. Takes a list of repository addresses. + */ + public void setRepositories(List<String> repositoryUrls) { + repositories = Lists.newArrayList(); + for (String repositoryUrl : repositoryUrls) { + repositories.add(new RemoteRepository.Builder( + "user-defined repository " + repositories.size(), "default", repositoryUrl).build()); + } + } + + public Path download() throws IOException { + RepositorySystem system = newRepositorySystem(); + RepositorySystemSession session = newRepositorySystemSession(system); + + ArtifactRequest artifactRequest = new ArtifactRequest(); + Artifact artifact = new DefaultArtifact(groupId + ":" + artifactId + ":" + version); + artifactRequest.setArtifact(artifact); + artifactRequest.setRepositories(repositories); + + try { + ArtifactResult artifactResult = system.resolveArtifact(session, artifactRequest); + artifact = artifactResult.getArtifact(); + } catch (ArtifactResolutionException e) { + throw new IOException("Failed to fetch Maven dependency: " + e.getMessage()); + } + return outputDirectory.getRelative(artifact.getFile().getAbsolutePath()); + } + + private RepositorySystemSession newRepositorySystemSession(RepositorySystem system) { + DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession(); + LocalRepository localRepo = new LocalRepository(outputDirectory.getPathString()); + session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo)); + session.setTransferListener(new AbstractTransferListener() {}); + session.setRepositoryListener(new AbstractRepositoryListener() {}); + return session; + } + + private RepositorySystem newRepositorySystem() { + DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator(); + locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class); + locator.addService(TransporterFactory.class, FileTransporterFactory.class); + locator.addService(TransporterFactory.class, HttpTransporterFactory.class); + return locator.getService(RepositorySystem.class); + } + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewLocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewLocalRepositoryFunction.java new file mode 100644 index 0000000..b2d9f74 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewLocalRepositoryFunction.java
@@ -0,0 +1,145 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.bazel.rules.workspace.NewLocalRepositoryRule; +import com.google.devtools.build.lib.packages.AggregatingAttributeMapper; +import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.skyframe.FileValue; +import com.google.devtools.build.lib.skyframe.RepositoryValue; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.nio.charset.Charset; + +/** + * Create a repository from a directory on the local filesystem. + */ +public class NewLocalRepositoryFunction extends RepositoryFunction { + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + RepositoryName repositoryName = (RepositoryName) skyKey.argument(); + Rule rule = RepositoryFunction.getRule(repositoryName, NewLocalRepositoryRule.NAME, env); + if (rule == null) { + return null; + } + + // Given a rule that looks like this: + // new_local_repository( + // name = 'x', + // path = '/some/path/to/y', + // build_file = 'x.BUILD' + // ) + // This creates the following directory structure: + // .external-repository/ + // x/ + // WORKSPACE + // x/ + // BUILD -> <build_root>/x.BUILD + // y -> /some/path/to/y + // + // In the structure above, .external-repository/x is the repository directory and + // .external-repository/x/x is the package directory. + Path repositoryDirectory = getExternalRepositoryDirectory().getRelative(rule.getName()); + Path outputDirectory = repositoryDirectory.getRelative(rule.getName()); + try { + FileSystemUtils.createDirectoryAndParents(outputDirectory); + } catch (IOException e) { + throw new RepositoryFunctionException(e, Transience.TRANSIENT); + } + FileValue directoryValue = getRepositoryDirectory(outputDirectory, env); + if (directoryValue == null) { + return null; + } + + // Add x/WORKSPACE. + try { + Path workspaceFile = repositoryDirectory.getRelative("WORKSPACE"); + FileSystemUtils.writeContent(workspaceFile, Charset.forName("UTF-8"), + "# DO NOT EDIT: automatically generated WORKSPACE file for " + rule + "\n"); + } catch (IOException e) { + throw new RepositoryFunctionException(e, Transience.TRANSIENT); + } + + AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule); + // Link x/x/y to /some/path/to/y. + String path = mapper.get("path", Type.STRING); + PathFragment pathFragment = new PathFragment(path); + if (!pathFragment.isAbsolute()) { + throw new RepositoryFunctionException( + new EvalException( + rule.getLocation(), + "In " + rule + " the 'path' attribute must specify an absolute path"), + Transience.PERSISTENT); + } + Path pathTarget = getOutputBase().getFileSystem().getPath(pathFragment); + Path symlinkPath = outputDirectory.getRelative(pathTarget.getBaseName()); + if (createSymbolicLink(symlinkPath, pathTarget, env) == null) { + return null; + } + + // Link x/x/BUILD to <build_root>/x.BUILD. + PathFragment buildFile = new PathFragment(mapper.get("build_file", Type.STRING)); + Path buildFileTarget = getWorkspace().getRelative(buildFile); + if (buildFile.equals(PathFragment.EMPTY_FRAGMENT) || buildFile.isAbsolute() + || !buildFileTarget.exists()) { + throw new RepositoryFunctionException( + new EvalException(rule.getLocation(), "In " + rule + + " the 'build_file' attribute must specify a relative path to an existing file"), + Transience.PERSISTENT); + } + Path buildFilePath = outputDirectory.getRelative("BUILD"); + if (createSymbolicLink(buildFilePath, buildFileTarget, env) == null) { + return null; + } + + return new RepositoryValue(repositoryDirectory, directoryValue); + } + + private FileValue createSymbolicLink(Path from, Path to, Environment env) + throws RepositoryFunctionException { + try { + if (!from.exists()) { + from.createSymbolicLink(to); + } + } catch (IOException e) { + throw new RepositoryFunctionException(e, Transience.TRANSIENT); + } + FileValue fromValue = getRepositoryDirectory(from, env); + return fromValue; + } + + @Override + public SkyFunctionName getSkyFunctionName() { + return SkyFunctionName.computed(NewLocalRepositoryRule.NAME.toUpperCase()); + } + + @Override + public Class<? extends RuleDefinition> getRuleDefinition() { + return NewLocalRepositoryRule.class; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryDelegatorFunction.java new file mode 100644 index 0000000..f0af0c6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryDelegatorFunction.java
@@ -0,0 +1,72 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; + +/** + * Implements delegation to the correct repository fetcher. + */ +public class RepositoryDelegatorFunction implements SkyFunction { + + // Mapping of rule class name to SkyFunction. + private final ImmutableMap<String, RepositoryFunction> handlers; + + public RepositoryDelegatorFunction( + ImmutableMap<String, RepositoryFunction> handlers) { + this.handlers = handlers; + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + RepositoryName repositoryName = (RepositoryName) skyKey.argument(); + Rule rule = RepositoryFunction.getRule(repositoryName, null, env); + if (rule == null) { + return null; + } + RepositoryFunction handler = handlers.get(rule.getRuleClass()); + if (handler == null) { + throw new IllegalStateException("Could not find handler for " + rule); + } + SkyKey key = new SkyKey(handler.getSkyFunctionName(), repositoryName); + + try { + return env.getValueOrThrow( + key, NoSuchPackageException.class, IOException.class, EvalException.class); + } catch (NoSuchPackageException e) { + throw new RepositoryFunction.RepositoryFunctionException(e, Transience.PERSISTENT); + } catch (IOException e) { + throw new RepositoryFunction.RepositoryFunctionException(e, Transience.PERSISTENT); + } catch (EvalException e) { + throw new RepositoryFunction.RepositoryFunctionException(e, Transience.PERSISTENT); + } + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryFunction.java new file mode 100644 index 0000000..906c38b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryFunction.java
@@ -0,0 +1,181 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException; +import com.google.devtools.build.lib.packages.BuildFileNotFoundException; +import com.google.devtools.build.lib.packages.ExternalPackage; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.skyframe.FileSymlinkCycleException; +import com.google.devtools.build.lib.skyframe.FileValue; +import com.google.devtools.build.lib.skyframe.InconsistentFilesystemException; +import com.google.devtools.build.lib.skyframe.PackageFunction; +import com.google.devtools.build.lib.skyframe.PackageValue; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; + +import java.io.IOException; + +import javax.annotation.Nullable; + +/** + * Parent class for repository-related Skyframe functions. + */ +public abstract class RepositoryFunction implements SkyFunction { + private static final String EXTERNAL_REPOSITORY_DIRECTORY = ".external-repository"; + private BlazeDirectories directories; + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + /** + * Gets Skyframe's name for this. + */ + public abstract SkyFunctionName getSkyFunctionName(); + + /** + * Sets up output path information. + */ + public void setDirectories(BlazeDirectories directories) { + this.directories = directories; + } + + protected Path getExternalRepositoryDirectory() { + return RepositoryFunction.getExternalRepositoryDirectory(directories); + } + + public static Path getExternalRepositoryDirectory(BlazeDirectories directories) { + return directories.getOutputBase().getRelative(EXTERNAL_REPOSITORY_DIRECTORY); + } + + /** + * Gets the base directory repositories should be stored in locally. + */ + protected Path getOutputBase() { + return directories.getOutputBase(); + } + + /** + * Gets the directory the WORKSPACE file for the build is in. + */ + protected Path getWorkspace() { + return directories.getWorkspace(); + } + + + /** + * Returns the RuleDefinition class for this type of repository. + */ + public abstract Class<? extends RuleDefinition> getRuleDefinition(); + + /** + * Uses a remote repository name to fetch the corresponding Rule describing how to get it. + * This should be called from {@link SkyFunction#compute} functions, which should return null if + * this returns null. If {@code ruleClassName} is set, the rule found must have a matching rule + * class name. + */ + @Nullable + public static Rule getRule( + RepositoryName repositoryName, @Nullable String ruleClassName, Environment env) + throws RepositoryFunctionException { + SkyKey packageKey = PackageValue.key( + PackageIdentifier.createInDefaultRepo(PackageFunction.EXTERNAL_PACKAGE_NAME)); + PackageValue packageValue; + try { + packageValue = (PackageValue) env.getValueOrThrow(packageKey, + NoSuchPackageException.class); + } catch (NoSuchPackageException e) { + throw new RepositoryFunctionException( + new BuildFileNotFoundException( + PackageFunction.EXTERNAL_PACKAGE_NAME, "Could not load //external package"), + Transience.PERSISTENT); + } + if (packageValue == null) { + return null; + } + ExternalPackage externalPackage = (ExternalPackage) packageValue.getPackage(); + Rule rule = externalPackage.getRepositoryInfo(repositoryName); + if (rule == null) { + throw new RepositoryFunctionException( + new BuildFileContainsErrorsException( + PackageFunction.EXTERNAL_PACKAGE_NAME, + "The repository named '" + repositoryName + "' could not be resolved"), + Transience.PERSISTENT); + } + Preconditions.checkState(ruleClassName == null || rule.getRuleClass().equals(ruleClassName), + "Got " + rule + ", was expecting a " + ruleClassName); + return rule; + } + + /** + * Adds the repository's directory to the graph and, if it's a symlink, resolves it to an + * actual directory. + */ + @Nullable + protected static FileValue getRepositoryDirectory(Path repositoryDirectory, Environment env) + throws RepositoryFunctionException { + SkyKey outputDirectoryKey = FileValue.key(RootedPath.toRootedPath( + repositoryDirectory, PathFragment.EMPTY_FRAGMENT)); + try { + return (FileValue) env.getValueOrThrow(outputDirectoryKey, IOException.class, + FileSymlinkCycleException.class, InconsistentFilesystemException.class); + } catch (IOException | FileSymlinkCycleException | InconsistentFilesystemException e) { + throw new RepositoryFunctionException( + new IOException("Could not access " + repositoryDirectory + ": " + e.getMessage()), + Transience.PERSISTENT); + } + } + + /** + * Exception thrown when something goes wrong accessing a remote repository. + * + * This exception should be used by child classes to limit the types of exceptions + * {@link RepositoryDelegatorFunction} has to know how to catch. + */ + static final class RepositoryFunctionException extends SkyFunctionException { + public RepositoryFunctionException(NoSuchPackageException cause, Transience transience) { + super(cause, transience); + } + + /** + * Error reading or writing to the filesystem. + */ + public RepositoryFunctionException(IOException cause, Transience transience) { + super(cause, transience); + } + + /** + * For errors in WORKSPACE file rules (e.g., malformed paths or URLs). + */ + public RepositoryFunctionException(EvalException cause, Transience transience) { + super(cause, transience); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelBaseRuleClasses.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelBaseRuleClasses.java new file mode 100644 index 0000000..a1b27fe --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelBaseRuleClasses.java
@@ -0,0 +1,73 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LICENSE; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; + +/** + * The foundational rule templates to help in real rule construction. Only attributes truly common + * to all rules go in here. Attributes such as "out", "outs", "src" and "srcs" exhibit enough + * variation that we declare them explicitly for each rule. This leads to stricter error checking + * and prevents users from inadvertently using an attribute that doesn't actually do anything. + */ +public class BazelBaseRuleClasses { + public static final ImmutableSet<String> ALLOWED_RULE_CLASSES = + ImmutableSet.of("filegroup", "genrule", "Fileset"); + + /** + * A base rule for all binary rules. + */ + @BlazeRule(name = "$binary_base_rule", + type = RuleClassType.ABSTRACT) + public static final class BinaryBaseRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .add(attr("args", STRING_LIST) + .nonconfigurable("policy decision: should be consistent across configurations")) + .add(attr("output_licenses", LICENSE)) + .add(attr("$is_executable", BOOLEAN).value(true) + .nonconfigurable("Called from RunCommand.isExecutable, which takes a Target")) + .build(); + } + } + + /** + * Rule class for rules in error. + */ + @BlazeRule(name = "$error_rule", + type = RuleClassType.ABSTRACT, + ancestors = { BaseRuleClasses.BaseRule.class }) + public static final class ErrorRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .publicByDefault() + .build(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfiguration.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfiguration.java new file mode 100644 index 0000000..63aa4e3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfiguration.java
@@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Bazel-specific configuration fragment. + */ +public class BazelConfiguration extends Fragment { + /** + * Loader for Google-specific settings. + */ + public static class Loader implements ConfigurationFragmentFactory { + @Override + public Fragment create(ConfigurationEnvironment env, BuildOptions buildOptions) + throws InvalidConfigurationException { + return new BazelConfiguration(); + } + + @Override + public Class<? extends Fragment> creates() { + return BazelConfiguration.class; + } + } + + public BazelConfiguration() { + } + + @Override + public String getName() { + return "Bazel"; + } + + @Override + public String cacheKey() { + return ""; + } + + @Override + public void defineExecutables(ImmutableMap.Builder<String, PathFragment> builder) { + if (OS.getCurrent() == OS.WINDOWS) { + String path = System.getenv("BAZEL_SH"); + if (path != null) { + builder.put("sh", new PathFragment(path)); + return; + } + } + builder.put("sh", new PathFragment("/bin/bash")); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfigurationCollection.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfigurationCollection.java new file mode 100644 index 0000000..1472b43 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfigurationCollection.java
@@ -0,0 +1,235 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Table; +import com.google.devtools.build.lib.analysis.ConfigurationCollectionFactory; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection.ConfigurationHolder; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection.Transitions; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.analysis.config.PackageProviderForConfigurations; +import com.google.devtools.build.lib.bazel.rules.cpp.BazelCppRuleClasses.CppTransition; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.AggregatingAttributeMapper; +import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition; +import com.google.devtools.build.lib.packages.Attribute.Transition; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Configuration collection used by the rules Bazel knows. + */ +public class BazelConfigurationCollection implements ConfigurationCollectionFactory { + @Override + @Nullable + public BuildConfiguration createConfigurations( + ConfigurationFactory configurationFactory, + PackageProviderForConfigurations loadedPackageProvider, + BuildOptions buildOptions, + Map<String, String> clientEnv, + EventHandler errorEventListener, + boolean performSanityCheck) throws InvalidConfigurationException { + + // We cache all the related configurations for this target configuration in a cache that is + // dropped at the end of this method call. We instead rely on the cache for entire collections + // for caching the target and related configurations, and on a dedicated host configuration + // cache for the host configuration. + Cache<String, BuildConfiguration> cache = + CacheBuilder.newBuilder().<String, BuildConfiguration>build(); + + // Target configuration + BuildConfiguration targetConfiguration = configurationFactory.getConfiguration( + loadedPackageProvider, buildOptions, clientEnv, false, cache); + if (targetConfiguration == null) { + return null; + } + + BuildConfiguration dataConfiguration = targetConfiguration; + + // Host configuration + // Note that this passes in the dataConfiguration, not the target + // configuration. This is intentional. + BuildConfiguration hostConfiguration = getHostConfigurationFromRequest(configurationFactory, + loadedPackageProvider, clientEnv, dataConfiguration, buildOptions); + if (hostConfiguration == null) { + return null; + } + + // Sanity check that the implicit labels are all in the transitive closure of explicit ones. + // This also registers all targets in the cache entry and validates them on subsequent requests. + Set<Label> reachableLabels = new HashSet<>(); + if (performSanityCheck) { + // We allow the package provider to be null for testing. + for (Label label : buildOptions.getAllLabels().values()) { + try { + collectTransitiveClosure(loadedPackageProvider, reachableLabels, label); + } catch (NoSuchThingException e) { + // We've loaded the transitive closure of the labels-to-load above, and made sure that + // there are no errors loading it, so this can't happen. + throw new IllegalStateException(e); + } + } + sanityCheckImplicitLabels(reachableLabels, targetConfiguration); + sanityCheckImplicitLabels(reachableLabels, hostConfiguration); + } + + BuildConfiguration result = setupTransitions( + targetConfiguration, dataConfiguration, hostConfiguration); + result.reportInvalidOptions(errorEventListener); + return result; + } + + /** + * Gets the correct host configuration for this build. The behavior + * depends on the value of the --distinct_host_configuration flag. + * + * <p>With --distinct_host_configuration=false, we use identical configurations + * for the host and target, and you can ignore everything below. But please + * note: if you're cross-compiling for k8 on a piii machine, your build will + * fail. This is a stopgap measure. + * + * <p>Currently, every build is (in effect) a cross-compile, in the strict + * sense that host and target configurations are unequal, thus we do not + * issue a "cross-compiling" warning. (Perhaps we should?) + * * + * @param requestConfig the requested target (not host!) configuration for + * this build. + * @param buildOptions the configuration options used for the target configuration + */ + @Nullable + private BuildConfiguration getHostConfigurationFromRequest( + ConfigurationFactory configurationFactory, + PackageProviderForConfigurations loadedPackageProvider, Map<String, String> clientEnv, + BuildConfiguration requestConfig, BuildOptions buildOptions) + throws InvalidConfigurationException { + BuildConfiguration.Options commonOptions = buildOptions.get(BuildConfiguration.Options.class); + if (!commonOptions.useDistinctHostConfiguration) { + return requestConfig; + } else { + BuildConfiguration hostConfig = configurationFactory.getHostConfiguration( + loadedPackageProvider, clientEnv, buildOptions, /*fallback=*/false); + if (hostConfig == null) { + return null; + } + return hostConfig; + } + } + + static BuildConfiguration setupTransitions(BuildConfiguration targetConfiguration, + BuildConfiguration dataConfiguration, BuildConfiguration hostConfiguration) { + Set<BuildConfiguration> allConfigurations = ImmutableSet.of(targetConfiguration, + dataConfiguration, hostConfiguration); + + Table<BuildConfiguration, Transition, ConfigurationHolder> transitionBuilder = + HashBasedTable.create(); + for (BuildConfiguration from : allConfigurations) { + for (ConfigurationTransition transition : ConfigurationTransition.values()) { + BuildConfiguration to; + if (transition == ConfigurationTransition.HOST) { + to = hostConfiguration; + } else if (transition == ConfigurationTransition.DATA && from == targetConfiguration) { + to = dataConfiguration; + } else { + to = from; + } + transitionBuilder.put(from, transition, new ConfigurationHolder(to)); + } + } + + // TODO(bazel-team): This makes LIPO totally not work. Just a band-aid until we get around to + // implementing a way for the C++ rules to contribute this transition to the configuration + // collection. + for (BuildConfiguration config : allConfigurations) { + transitionBuilder.put(config, CppTransition.LIPO_COLLECTOR, new ConfigurationHolder(config)); + transitionBuilder.put(config, CppTransition.TARGET_CONFIG_FOR_LIPO, + new ConfigurationHolder(config.isHostConfiguration() ? null : config)); + } + + for (BuildConfiguration config : allConfigurations) { + Transitions outgoingTransitions = + new BuildConfigurationCollection.Transitions(config, transitionBuilder.row(config)); + // We allow host configurations to be shared between target configurations. In that case, the + // transitions may already be set. + // TODO(bazel-team): Check that the transitions are identical, or even better, change the + // code to set the host configuration transitions before we even create the target + // configuration. + if (config.isHostConfiguration() && config.getTransitions() != null) { + continue; + } + config.setConfigurationTransitions(outgoingTransitions); + } + + return targetConfiguration; + } + + /** + * Checks that the implicit labels are reachable from the loaded labels. The loaded labels are + * those returned from {@link BuildConfigurationKey#getLabelsToLoadUnconditionally()}, and the + * implicit ones are those that need to be available for late-bound attributes. + */ + private void sanityCheckImplicitLabels(Collection<Label> reachableLabels, + BuildConfiguration config) throws InvalidConfigurationException { + for (Map.Entry<String, Label> entry : config.getImplicitLabels().entries()) { + if (!reachableLabels.contains(entry.getValue())) { + throw new InvalidConfigurationException("The required " + entry.getKey() + + " target is not transitively reachable from a command-line option: '" + + entry.getValue() + "'"); + } + } + } + + private void collectTransitiveClosure(PackageProviderForConfigurations loadedPackageProvider, + Set<Label> reachableLabels, Label from) throws NoSuchThingException { + if (!reachableLabels.add(from)) { + return; + } + Target fromTarget = loadedPackageProvider.getLoadedTarget(from); + if (fromTarget instanceof Rule) { + Rule rule = (Rule) fromTarget; + if (rule.getRuleClassObject().hasAttr("srcs", Type.LABEL_LIST)) { + // TODO(bazel-team): refine this. This visits "srcs" reachable under *any* configuration, + // not necessarily the configuration actually applied to the rule. We should correlate the + // two. However, doing so requires faithfully reflecting the configuration transitions that + // might happen as we traverse the dependency chain. + for (List<Label> labelsForConfiguration : + AggregatingAttributeMapper.of(rule).visitAttribute("srcs", Type.LABEL_LIST)) { + for (Label label : labelsForConfiguration) { + collectTransitiveClosure(loadedPackageProvider, reachableLabels, label); + } + } + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java new file mode 100644 index 0000000..1280bdb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
@@ -0,0 +1,272 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Functions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider.PrerequisiteValidator; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigRuleClasses; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.FragmentOptions; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.analysis.constraints.EnvironmentRule; +import com.google.devtools.build.lib.bazel.rules.common.BazelActionListenerRule; +import com.google.devtools.build.lib.bazel.rules.common.BazelExtraActionRule; +import com.google.devtools.build.lib.bazel.rules.common.BazelFilegroupRule; +import com.google.devtools.build.lib.bazel.rules.common.BazelTestSuiteRule; +import com.google.devtools.build.lib.bazel.rules.cpp.BazelCppRuleClasses; +import com.google.devtools.build.lib.bazel.rules.genrule.BazelGenRuleRule; +import com.google.devtools.build.lib.bazel.rules.java.BazelJavaBinaryRule; +import com.google.devtools.build.lib.bazel.rules.java.BazelJavaBuildInfoFactory; +import com.google.devtools.build.lib.bazel.rules.java.BazelJavaImportRule; +import com.google.devtools.build.lib.bazel.rules.java.BazelJavaLibraryRule; +import com.google.devtools.build.lib.bazel.rules.java.BazelJavaPluginRule; +import com.google.devtools.build.lib.bazel.rules.java.BazelJavaRuleClasses; +import com.google.devtools.build.lib.bazel.rules.java.BazelJavaTestRule; +import com.google.devtools.build.lib.bazel.rules.objc.BazelIosTestRule; +import com.google.devtools.build.lib.bazel.rules.sh.BazelShBinaryRule; +import com.google.devtools.build.lib.bazel.rules.sh.BazelShLibraryRule; +import com.google.devtools.build.lib.bazel.rules.sh.BazelShRuleClasses; +import com.google.devtools.build.lib.bazel.rules.sh.BazelShTestRule; +import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule; +import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule; +import com.google.devtools.build.lib.bazel.rules.workspace.LocalRepositoryRule; +import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule; +import com.google.devtools.build.lib.bazel.rules.workspace.NewLocalRepositoryRule; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.PackageGroup; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.rules.cpp.CcToolchainRule; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppConfigurationLoader; +import com.google.devtools.build.lib.rules.cpp.CppOptions; +import com.google.devtools.build.lib.rules.java.JavaConfiguration; +import com.google.devtools.build.lib.rules.java.JavaConfigurationLoader; +import com.google.devtools.build.lib.rules.java.JavaCpuSupplier; +import com.google.devtools.build.lib.rules.java.JavaImportBaseRule; +import com.google.devtools.build.lib.rules.java.JavaOptions; +import com.google.devtools.build.lib.rules.java.JavaToolchainRule; +import com.google.devtools.build.lib.rules.java.Jvm; +import com.google.devtools.build.lib.rules.java.JvmConfigurationLoader; +import com.google.devtools.build.lib.rules.objc.IosApplicationRule; +import com.google.devtools.build.lib.rules.objc.IosDeviceRule; +import com.google.devtools.build.lib.rules.objc.ObjcBinaryRule; +import com.google.devtools.build.lib.rules.objc.ObjcBundleLibraryRule; +import com.google.devtools.build.lib.rules.objc.ObjcBundleRule; +import com.google.devtools.build.lib.rules.objc.ObjcCommandLineOptions; +import com.google.devtools.build.lib.rules.objc.ObjcConfigurationLoader; +import com.google.devtools.build.lib.rules.objc.ObjcFrameworkRule; +import com.google.devtools.build.lib.rules.objc.ObjcImportRule; +import com.google.devtools.build.lib.rules.objc.ObjcLibraryRule; +import com.google.devtools.build.lib.rules.objc.ObjcOptionsRule; +import com.google.devtools.build.lib.rules.objc.ObjcProtoLibraryRule; +import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses; +import com.google.devtools.build.lib.rules.objc.ObjcXcodeprojRule; +import com.google.devtools.build.lib.rules.workspace.BindRule; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkType; + +/** + * A rule class provider implementing the rules Bazel knows. + */ +public class BazelRuleClassProvider { + + /** + * Used by the build encyclopedia generator. + */ + public static ConfiguredRuleClassProvider create() { + ConfiguredRuleClassProvider.Builder builder = + new ConfiguredRuleClassProvider.Builder(); + setup(builder); + return builder.build(); + } + + public static final JavaCpuSupplier JAVA_CPU_SUPPLIER = new JavaCpuSupplier() { + @Override + public String getJavaCpu(BuildOptions buildOptions, ConfigurationEnvironment env) + throws InvalidConfigurationException { + JavaOptions javaOptions = buildOptions.get(JavaOptions.class); + return javaOptions.javaCpu == null ? "default" : javaOptions.javaCpu; + } + }; + + private static class BazelPrerequisiteValidator implements PrerequisiteValidator { + @Override + public void validate(RuleContext.Builder context, + ConfiguredTarget prerequisite, Attribute attribute) { + validateDirectPrerequisiteVisibility(context, prerequisite, attribute.getName()); + } + + private void validateDirectPrerequisiteVisibility( + RuleContext.Builder context, ConfiguredTarget prerequisite, String attrName) { + Rule rule = context.getRule(); + Target prerequisiteTarget = prerequisite.getTarget(); + Label prerequisiteLabel = prerequisiteTarget.getLabel(); + // We don't check the visibility of late-bound attributes, because it would break some + // features. + if (!context.getRule().getLabel().getPackageName().equals( + prerequisite.getTarget().getLabel().getPackageName()) + && !context.isVisible(prerequisite)) { + if (!context.getConfiguration().checkVisibility()) { + context.ruleWarning(String.format("Target '%s' violates visibility of target " + + "'%s'. Continuing because --nocheck_visibility is active", + rule.getLabel(), prerequisiteLabel)); + } else { + // Oddly enough, we use reportError rather than ruleError here. + context.reportError(rule.getLocation(), + String.format("Target '%s' is not visible from target '%s'. Check " + + "the visibility declaration of the former target if you think " + + "the dependency is legitimate", + prerequisiteLabel, rule.getLabel())); + } + } + + if (prerequisiteTarget instanceof PackageGroup) { + if (!attrName.equals("visibility")) { + context.reportError(rule.getAttributeLocation(attrName), + "in " + attrName + " attribute of " + rule.getRuleClass() + + " rule " + rule.getLabel() + ": package group '" + + prerequisiteLabel + "' is misplaced here " + + "(they are only allowed in the visibility attribute)"); + } + } + } + } + + /** + * List of all build option classes in Blaze. + */ + // TODO(bazel-team): make this private, remove from tests, then BuildOptions.of can be merged + // into RuleClassProvider. + @VisibleForTesting + @SuppressWarnings("unchecked") + public static final ImmutableList<Class<? extends FragmentOptions>> BUILD_OPTIONS = + ImmutableList.of( + BuildConfiguration.Options.class, + CppOptions.class, + JavaOptions.class, + ObjcCommandLineOptions.class + ); + + /** + * Java objects accessible from Skylark rule implementations using this module. + */ + private static final ImmutableMap<String, SkylarkType> skylarkBuiltinJavaObects = + ImmutableMap.of( + "jvm", SkylarkType.of(Jvm.class), + "java_configuration", SkylarkType.of(JavaConfiguration.class), + "cpp", SkylarkType.of(CppConfiguration.class)); + + public static void setup(ConfiguredRuleClassProvider.Builder builder) { + builder + .addBuildInfoFactory(new BazelJavaBuildInfoFactory()) + .setConfigurationCollectionFactory(new BazelConfigurationCollection()) + .setPrerequisiteValidator(new BazelPrerequisiteValidator()) + .setSkylarkAccessibleJavaClasses(skylarkBuiltinJavaObects); + + for (Class<? extends FragmentOptions> fragmentOptions : BUILD_OPTIONS) { + builder.addConfigurationOptions(fragmentOptions); + } + + builder.addRuleDefinition(BaseRuleClasses.BaseRule.class); + builder.addRuleDefinition(BaseRuleClasses.RuleBase.class); + builder.addRuleDefinition(BazelBaseRuleClasses.BinaryBaseRule.class); + builder.addRuleDefinition(BaseRuleClasses.TestBaseRule.class); + builder.addRuleDefinition(BazelBaseRuleClasses.ErrorRule.class); + + builder.addRuleDefinition(EnvironmentRule.class); + + builder.addRuleDefinition(ConfigRuleClasses.ConfigBaseRule.class); + builder.addRuleDefinition(ConfigRuleClasses.ConfigSettingRule.class); + + builder.addRuleDefinition(BazelFilegroupRule.class); + builder.addRuleDefinition(BazelTestSuiteRule.class); + builder.addRuleDefinition(BazelGenRuleRule.class); + + builder.addRuleDefinition(BazelShRuleClasses.ShRule.class); + builder.addRuleDefinition(BazelShLibraryRule.class); + builder.addRuleDefinition(BazelShBinaryRule.class); + builder.addRuleDefinition(BazelShTestRule.class); + + builder.addRuleDefinition(CcToolchainRule.class); + builder.addRuleDefinition(BazelCppRuleClasses.CcLinkingRule.class); + builder.addRuleDefinition(BazelCppRuleClasses.CcDeclRule.class); + builder.addRuleDefinition(BazelCppRuleClasses.CcBaseRule.class); + builder.addRuleDefinition(BazelCppRuleClasses.CcRule.class); + builder.addRuleDefinition(BazelCppRuleClasses.CcBinaryBaseRule.class); + builder.addRuleDefinition(BazelCppRuleClasses.CcBinaryRule.class); + builder.addRuleDefinition(BazelCppRuleClasses.CcTestRule.class); + + builder.addRuleDefinition(BazelCppRuleClasses.CcLibraryBaseRule.class); + builder.addRuleDefinition(BazelCppRuleClasses.CcLibraryRule.class); + + + builder.addRuleDefinition(BazelJavaRuleClasses.BaseJavaBinaryRule.class); + builder.addRuleDefinition(BazelJavaRuleClasses.IjarBaseRule.class); + builder.addRuleDefinition(BazelJavaRuleClasses.JavaBaseRule.class); + builder.addRuleDefinition(JavaImportBaseRule.class); + builder.addRuleDefinition(BazelJavaRuleClasses.JavaRule.class); + builder.addRuleDefinition(BazelJavaBinaryRule.class); + builder.addRuleDefinition(BazelJavaLibraryRule.class); + builder.addRuleDefinition(BazelJavaImportRule.class); + builder.addRuleDefinition(BazelJavaTestRule.class); + builder.addRuleDefinition(BazelJavaPluginRule.class); + builder.addRuleDefinition(JavaToolchainRule.class); + + builder.addRuleDefinition(BazelIosTestRule.class); + builder.addRuleDefinition(IosDeviceRule.class); + builder.addRuleDefinition(ObjcBinaryRule.class); + builder.addRuleDefinition(ObjcBundleRule.class); + builder.addRuleDefinition(ObjcBundleLibraryRule.class); + builder.addRuleDefinition(ObjcFrameworkRule.class); + builder.addRuleDefinition(ObjcImportRule.class); + builder.addRuleDefinition(ObjcLibraryRule.class); + builder.addRuleDefinition(ObjcOptionsRule.class); + builder.addRuleDefinition(ObjcProtoLibraryRule.class); + builder.addRuleDefinition(ObjcXcodeprojRule.class); + builder.addRuleDefinition(ObjcRuleClasses.IosTestBaseRule.class); + builder.addRuleDefinition(ObjcRuleClasses.ObjcHasInfoplistRule.class); + builder.addRuleDefinition(ObjcRuleClasses.ObjcHasEntitlementsRule.class); + builder.addRuleDefinition(ObjcRuleClasses.ObjcCompilationRule.class); + builder.addRuleDefinition(ObjcRuleClasses.ObjcBaseResourcesRule.class); + builder.addRuleDefinition(IosApplicationRule.class); + + builder.addRuleDefinition(BazelExtraActionRule.class); + builder.addRuleDefinition(BazelActionListenerRule.class); + + builder.addRuleDefinition(BindRule.class); + builder.addRuleDefinition(HttpArchiveRule.class); + builder.addRuleDefinition(HttpJarRule.class); + builder.addRuleDefinition(LocalRepositoryRule.class); + builder.addRuleDefinition(MavenJarRule.class); + builder.addRuleDefinition(NewLocalRepositoryRule.class); + + builder.addConfigurationFragment(new BazelConfiguration.Loader()); + builder.addConfigurationFragment(new CppConfigurationLoader( + Functions.<String>identity())); + builder.addConfigurationFragment(new JvmConfigurationLoader(JAVA_CPU_SUPPLIER)); + builder.addConfigurationFragment(new JavaConfigurationLoader(JAVA_CPU_SUPPLIER)); + builder.addConfigurationFragment(new ObjcConfigurationLoader()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java new file mode 100644 index 0000000..214b367 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java
@@ -0,0 +1,159 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.actions.ActionContextConsumer; +import com.google.devtools.build.lib.actions.ActionContextProvider; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ExecutorInitException; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.rules.cpp.CppCompileActionContext; +import com.google.devtools.build.lib.rules.cpp.CppLinkActionContext; +import com.google.devtools.build.lib.rules.cpp.LocalGccStrategy; +import com.google.devtools.build.lib.rules.cpp.LocalLinkStrategy; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.runtime.GotOptionsEvent; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.Map; + +/** + * Module implementing the rule set of Bazel. + */ +public class BazelRulesModule extends BlazeModule { + /** + * Execution options affecting how we execute the build actions (but not their semantics). + */ + public static class BazelExecutionOptions extends OptionsBase { + @Option( + name = "spawn_strategy", + defaultValue = "standalone", + category = "strategy", + help = "Specify how spawn actions are executed by default." + + "'standalone' means run all of them locally." + + "'sandboxed' means run them in namespaces based sandbox (available only on Linux)") + public String spawnStrategy; + + @Option( + name = "genrule_strategy", + defaultValue = "standalone", + category = "strategy", + help = "Specify how to execute genrules." + + "'standalone' means run all of them locally." + + "'sandboxed' means run them in namespaces based sandbox (available only on Linux)") + + public String genruleStrategy; + } + + private static class BazelActionContextConsumer implements ActionContextConsumer { + BazelExecutionOptions options; + + private BazelActionContextConsumer(BazelExecutionOptions options) { + this.options = options; + + } + @Override + public Map<String, String> getSpawnActionContexts() { + ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); + + builder.put("Genrule", options.genruleStrategy); + + // TODO(bazel-team): put this in getActionContexts (key=SpawnActionContext.class) instead + builder.put("", options.spawnStrategy); + + return builder.build(); + } + + @Override + public Map<Class<? extends ActionContext>, String> getActionContexts() { + ImmutableMap.Builder<Class<? extends ActionContext>, String> builder = + ImmutableMap.builder(); + builder.put(CppCompileActionContext.class, ""); + builder.put(CppLinkActionContext.class, ""); + return builder.build(); + } + } + + private class BazelActionContextProvider implements ActionContextProvider { + @Override + public Iterable<ActionContext> getActionContexts() { + return ImmutableList.of( + new LocalGccStrategy(optionsProvider), + new LocalLinkStrategy()); + } + + @Override + public void executorCreated(Iterable<ActionContext> usedContexts) + throws ExecutorInitException { + } + + @Override + public void executionPhaseStarting(ActionInputFileCache actionInputFileCache, + ActionGraph actionGraph, Iterable<Artifact> topLevelArtifacts) + throws ExecutorInitException, InterruptedException { + } + + @Override + public void executionPhaseEnding() { + } + } + + private BlazeRuntime runtime; + private OptionsProvider optionsProvider; + + @Override + public void beforeCommand(BlazeRuntime blazeRuntime, Command command) { + this.runtime = blazeRuntime; + runtime.getEventBus().register(this); + } + + @Override + public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) { + return command.builds() + ? ImmutableList.<Class<? extends OptionsBase>>of(BazelExecutionOptions.class) + : ImmutableList.<Class<? extends OptionsBase>>of(); + } + + @Override + public ActionContextConsumer getActionContextConsumer() { + return new BazelActionContextConsumer( + optionsProvider.getOptions(BazelExecutionOptions.class)); + } + + @Override + public ActionContextProvider getActionContextProvider() { + return new BazelActionContextProvider(); + } + + @Subscribe + public void gotOptions(GotOptionsEvent event) { + optionsProvider = event.getOptions(); + } + + @Override + public void initializeRuleClasses(ConfiguredRuleClassProvider.Builder builder) { + BazelRuleClassProvider.setup(builder); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelActionListenerRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelActionListenerRule.java new file mode 100644 index 0000000..eba1553 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelActionListenerRule.java
@@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.rules.common; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.rules.extra.ActionListener; + +/** + * Rule definition for action_listener rule. + */ +@BlazeRule(name = "action_listener", + ancestors = { BaseRuleClasses.RuleBase.class }, + factoryClass = ActionListener.class) +public final class BazelActionListenerRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + .add(attr("mnemonics", STRING_LIST).mandatory()) + .add(attr("extra_actions", LABEL_LIST).mandatory() + .allowedRuleClasses("extra_action") + .allowedFileTypes()) + .removeAttribute("deps") + .removeAttribute("data") + .removeAttribute(":action_listener") + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelExtraActionRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelExtraActionRule.java new file mode 100644 index 0000000..fa73f93 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelExtraActionRule.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.rules.common; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.rules.extra.ExtraActionFactory; + +/** + * Rule definition for extra_action rule. + */ +@BlazeRule(name = "extra_action", + ancestors = { BaseRuleClasses.RuleBase.class }, + factoryClass = ExtraActionFactory.class) +public final class BazelExtraActionRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + .add(attr("tools", LABEL_LIST).cfg(HOST).allowedFileTypes().exec()) + .add(attr("out_templates", STRING_LIST)) + .add(attr("cmd", STRING).mandatory()) + .add(attr("requires_action_output", BOOLEAN)) + .removeAttribute("deps") + .removeAttribute(":action_listener") + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelFilegroupRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelFilegroupRule.java new file mode 100644 index 0000000..0ff5cd0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelFilegroupRule.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.rules.common; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.DATA; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.LICENSE; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.rules.filegroup.Filegroup; +import com.google.devtools.build.lib.util.FileTypeSet; + +/** + * Rule object implementing "filegroup". + */ +@BlazeRule(name = "filegroup", + ancestors = { BaseRuleClasses.BaseRule.class }, + factoryClass = Filegroup.class) +public final class BazelFilegroupRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + // filegroup ignores any filtering set with setSrcsAllowedFiles. + return builder + .add(attr("srcs", LABEL_LIST).allowedFileTypes(FileTypeSet.ANY_FILE)) + .add(attr("data", LABEL_LIST).cfg(DATA).allowedFileTypes(FileTypeSet.ANY_FILE)) + .add(attr("output_licenses", LICENSE)) + .add(attr("path", STRING)) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelTestSuiteRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelTestSuiteRule.java new file mode 100644 index 0000000..54db469 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/common/BazelTestSuiteRule.java
@@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.rules.common; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.rules.test.TestSuite; + +/** + * Rule object implementing "test_suite". + */ +@BlazeRule(name = "test_suite", + ancestors = { BaseRuleClasses.BaseRule.class }, + factoryClass = TestSuite.class) +public final class BazelTestSuiteRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .override(attr("testonly", BOOLEAN).value(true) + .nonconfigurable("policy decision: should be consistent across configurations")) + .add(attr("tests", LABEL_LIST).orderIndependent().allowedFileTypes() + .nonconfigurable("policy decision: should be consistent across configurations")) + .add(attr("suites", LABEL_LIST).orderIndependent().allowedFileTypes() + .nonconfigurable("policy decision: should be consistent across configurations")) + // This magic attribute contains all *test rules in the package, iff + // tests=[] and suites=[]: + .add(attr("$implicit_tests", LABEL_LIST) + .nonconfigurable("Accessed in TestTargetUtils without config context")) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcBinary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcBinary.java new file mode 100644 index 0000000..e3f62a1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcBinary.java
@@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.cpp; + +import com.google.devtools.build.lib.rules.cpp.CcBinary; + +/** + * Factory class for the {@code cc_binary} rule. + */ +public class BazelCcBinary extends CcBinary { + public BazelCcBinary() { + super(BazelCppSemantics.INSTANCE); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcLibrary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcLibrary.java new file mode 100644 index 0000000..ae38806 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcLibrary.java
@@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.cpp; + +import com.google.devtools.build.lib.rules.cpp.CcLibrary; + +/** + * Factory class for the {@code cc_library} rule. + */ +public class BazelCcLibrary extends CcLibrary { + public BazelCcLibrary() { + super(BazelCppSemantics.INSTANCE); + } +} \ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcTest.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcTest.java new file mode 100644 index 0000000..f42b1dc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcTest.java
@@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.cpp; + +import com.google.devtools.build.lib.rules.cpp.CcTest; + +/** + * Factory class for the {@code cc_test} rule. + */ +public class BazelCcTest extends CcTest { + public BazelCcTest() { + super(BazelCppSemantics.INSTANCE); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppRuleClasses.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppRuleClasses.java new file mode 100644 index 0000000..4a1f3b6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppRuleClasses.java
@@ -0,0 +1,422 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.cpp; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromFunctions; +import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST_DICT; +import static com.google.devtools.build.lib.packages.Type.STRING; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; +import static com.google.devtools.build.lib.packages.Type.TRISTATE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ALWAYS_LINK_LIBRARY; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ALWAYS_LINK_PIC_LIBRARY; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ARCHIVE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.CPP_HEADER; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.CPP_SOURCE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.C_SOURCE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.OBJECT_FILE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.PIC_ARCHIVE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.PIC_OBJECT_FILE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.SHARED_LIBRARY; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.VERSIONED_SHARED_LIBRARY; + +import com.google.common.base.Predicates; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.bazel.rules.BazelBaseRuleClasses; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition; +import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel; +import com.google.devtools.build.lib.packages.Attribute.Transition; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.RawAttributeMapper; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.packages.TriState; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CcLibrary; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppRuleClasses; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode; + +/** + * Rule class definitions for C++ rules. + */ +public class BazelCppRuleClasses { + static final SafeImplicitOutputsFunction CC_LIBRARY_DYNAMIC_LIB = + fromTemplates("%{dirname}lib%{basename}.so"); + + static final SafeImplicitOutputsFunction CC_BINARY_IMPLICIT_OUTPUTS = + fromFunctions(CppRuleClasses.CC_BINARY_STRIPPED, CppRuleClasses.CC_BINARY_DEBUG_PACKAGE); + + static final FileTypeSet ALLOWED_SRC_FILES = FileTypeSet.of( + CPP_SOURCE, + C_SOURCE, + CPP_HEADER, + ASSEMBLER_WITH_C_PREPROCESSOR, + ARCHIVE, + PIC_ARCHIVE, + ALWAYS_LINK_LIBRARY, + ALWAYS_LINK_PIC_LIBRARY, + SHARED_LIBRARY, + VERSIONED_SHARED_LIBRARY, + OBJECT_FILE, + PIC_OBJECT_FILE); + + static final String[] DEPS_ALLOWED_RULES = new String[] { + "cc_library", + }; + + /** + * Miscellaneous configuration transitions. It would be better not to have this - please don't add + * to it. + */ + public static enum CppTransition implements Transition { + /** + * The configuration for LIPO information collection. Requesting this from a configuration that + * does not have lipo optimization enabled may result in an exception. + */ + LIPO_COLLECTOR, + + /** + * The corresponding (target) configuration. + */ + TARGET_CONFIG_FOR_LIPO; + + @Override + public boolean defaultsToSelf() { + return false; + } + } + + private static final RuleClass.Configurator<BuildConfiguration, Rule> LIPO_ON_DEMAND = + new RuleClass.Configurator<BuildConfiguration, Rule>() { + @Override + public BuildConfiguration apply(Rule rule, BuildConfiguration configuration) { + BuildConfiguration toplevelConfig = + configuration.getConfiguration(CppTransition.TARGET_CONFIG_FOR_LIPO); + // If LIPO is enabled, override the default configuration. + if (toplevelConfig != null + && toplevelConfig.getFragment(CppConfiguration.class).isLipoOptimization() + && !configuration.isHostConfiguration() + && !configuration.getFragment(CppConfiguration.class).isLipoContextCollector()) { + // Switch back to data when the cc_binary is not the LIPO context. + return (rule.getLabel().equals( + toplevelConfig.getFragment(CppConfiguration.class).getLipoContextLabel())) + ? toplevelConfig + : configuration.getTransitions().getConfiguration(ConfigurationTransition.DATA); + } + return configuration; + } + }; + + /** + * Label of a pseudo-filegroup that contains all crosstool and libcfiles for + * all configurations, as specified on the command-line. + */ + public static final String CROSSTOOL_LABEL = "//tools/defaults:crosstool"; + + public static final LateBoundLabel<BuildConfiguration> CC_TOOLCHAIN = + new LateBoundLabel<BuildConfiguration>(CROSSTOOL_LABEL) { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.getFragment(CppConfiguration.class).getCcToolchainRuleLabel(); + } + }; + + public static final LateBoundLabel<BuildConfiguration> DEFAULT_MALLOC = + new LateBoundLabel<BuildConfiguration>() { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.getFragment(CppConfiguration.class).customMalloc(); + } + }; + + public static final LateBoundLabel<BuildConfiguration> STL = + new LateBoundLabel<BuildConfiguration>() { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return getStl(rule, configuration); + } + }; + + /** + * Implementation for the :lipo_context_collector attribute. + */ + public static final LateBoundLabel<BuildConfiguration> LIPO_CONTEXT_COLLECTOR = + new LateBoundLabel<BuildConfiguration>() { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + // This attribute connects a target to the LIPO context target configured with the + // lipo input collector configuration. + CppConfiguration cppConfiguration = configuration.getFragment(CppConfiguration.class); + return !cppConfiguration.isLipoContextCollector() + && (cppConfiguration.getLipoMode() == LipoMode.BINARY) + ? cppConfiguration.getLipoContextLabel() + : null; + } + }; + + /** + * Returns the STL prerequisite of the rule. + * + * <p>If rule has an implicit $stl attribute returns STL version set on the + * command line or if not set, the value of the $stl attribute. Returns + * {@code null} otherwise. + */ + private static Label getStl(Rule rule, BuildConfiguration original) { + Label stl = null; + if (rule.getRuleClassObject().hasAttr("$stl", Type.LABEL)) { + Label stlConfigLabel = original.getFragment(CppConfiguration.class).getStl(); + Label stlRuleLabel = RawAttributeMapper.of(rule).get("$stl", Type.LABEL); + if (stlConfigLabel == null) { + stl = stlRuleLabel; + } else if (!stlConfigLabel.equals(rule.getLabel()) && stlRuleLabel != null) { + // prevents self-reference and a cycle through standard STL in the dependency graph + stl = stlConfigLabel; + } + } + return stl; + } + + /** + * Common attributes for all rules that create C++ links. This may + * include non-cc_* rules (e.g. py_binary). + */ + @BlazeRule(name = "$cc_linking_rule", + type = RuleClassType.ABSTRACT) + public static final class CcLinkingRule implements RuleDefinition { + @Override + @SuppressWarnings("unchecked") + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .add(attr(":cc_toolchain", LABEL).value(CC_TOOLCHAIN)) + .setPreferredDependencyPredicate(Predicates.<String>or(CPP_SOURCE, C_SOURCE, CPP_HEADER)) + .build(); + } + } + + /** + * Common attributes for C++ rules. + */ + @BlazeRule(name = "$cc_base_rule", + type = RuleClassType.ABSTRACT, + ancestors = { CcLinkingRule.class }) + public static final class CcBaseRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .add(attr("copts", STRING_LIST)) + .add(attr("$stl", LABEL).value(env.getLabel("//tools/cpp:stl"))) + .add(attr(":stl", LABEL).value(STL)) + .build(); + } + } + + /** + * Helper rule class. + */ + @BlazeRule(name = "$cc_decl_rule", + type = RuleClassType.ABSTRACT, + ancestors = { BaseRuleClasses.RuleBase.class }) + public static final class CcDeclRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .add(attr("abi", STRING).value("$(ABI)")) + .add(attr("abi_deps", LABEL_LIST_DICT)) + .add(attr("defines", STRING_LIST)) + .add(attr("includes", STRING_LIST)) + .add(attr(":lipo_context_collector", LABEL) + .cfg(CppTransition.LIPO_COLLECTOR) + .value(LIPO_CONTEXT_COLLECTOR)) + .build(); + } + } + + /** + * Helper rule class. + */ + @BlazeRule(name = "$cc_rule", + type = RuleClassType.ABSTRACT, + ancestors = { CcDeclRule.class, CcBaseRule.class }) + public static final class CcRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) { + return builder + .add(attr("srcs", LABEL_LIST) + .direct_compile_time_input() + .allowedFileTypes(ALLOWED_SRC_FILES)) + .override(attr("deps", LABEL_LIST) + .allowedRuleClasses(DEPS_ALLOWED_RULES) + .allowedFileTypes() + .skipAnalysisTimeFileTypeCheck()) + .add(attr("linkopts", STRING_LIST)) + .add(attr("nocopts", STRING)) + .add(attr("hdrs_check", STRING).value("strict")) + .add(attr("linkstatic", BOOLEAN).value(true)) + .override(attr("$stl", LABEL).value(new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + // Every cc_rule depends implicitly on STL to make + // sure that the correct headers are used for inclusion. The only exception is + // STL itself to avoid cycles in the dependency graph. + Label stl = env.getLabel("//tools/cpp:stl"); + return rule.getLabel().equals(stl) ? null : stl; + } + })) + .build(); + } + } + + /** + * Helper rule class. + */ + @BlazeRule(name = "$cc_binary_base", + type = RuleClassType.ABSTRACT, + ancestors = CcRule.class) + public static final class CcBinaryBaseRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .add(attr("malloc", LABEL) + .value(env.getLabel("//tools/cpp:malloc")) + .allowedFileTypes() + .allowedRuleClasses("cc_library")) + .add(attr(":default_malloc", LABEL).value(DEFAULT_MALLOC)) + .add(attr("stamp", TRISTATE).value(TriState.AUTO)) + .build(); + } + } + + /** + * Rule definition for cc_binary rules. + */ + @BlazeRule(name = "cc_binary", + ancestors = { CcBinaryBaseRule.class, + BazelBaseRuleClasses.BinaryBaseRule.class }, + factoryClass = BazelCcBinary.class) + public static final class CcBinaryRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .setImplicitOutputsFunction(CC_BINARY_IMPLICIT_OUTPUTS) + .add(attr("linkshared", BOOLEAN).value(false) + .nonconfigurable("used to *determine* the rule's configuration")) + .cfg(LIPO_ON_DEMAND) + .build(); + } + } + + /** + * Implementation for the :lipo_context attribute. + */ + private static final LateBoundLabel<BuildConfiguration> LIPO_CONTEXT = + new LateBoundLabel<BuildConfiguration>() { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + Label result = configuration.getFragment(CppConfiguration.class).getLipoContextLabel(); + return (rule == null || rule.getLabel().equals(result)) ? null : result; + } + }; + + /** + * Rule definition for cc_test rules. + */ + @BlazeRule(name = "cc_test", + type = RuleClassType.TEST, + ancestors = { CcBinaryBaseRule.class, BaseRuleClasses.TestBaseRule.class }, + factoryClass = BazelCcTest.class) + public static final class CcTestRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .setImplicitOutputsFunction(CppRuleClasses.CC_BINARY_DEBUG_PACKAGE) + .override(attr("linkstatic", BOOLEAN).value(false)) + .override(attr("stamp", TRISTATE).value(TriState.NO)) + .add(attr(":lipo_context", LABEL).value(LIPO_CONTEXT)) + .build(); + } + } + + /** + * Helper rule class. + */ + @BlazeRule(name = "$cc_library", + type = RuleClassType.ABSTRACT, + ancestors = { CcRule.class }) + public static final class CcLibraryBaseRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .add(attr("hdrs", LABEL_LIST).orderIndependent().direct_compile_time_input() + .allowedFileTypes(CPP_HEADER)) + .add(attr("linkstamp", LABEL).allowedFileTypes(CPP_SOURCE, C_SOURCE)) + .build(); + } + } + + /** + * Rule definition for the cc_library rule. + */ + @BlazeRule(name = "cc_library", + ancestors = { CcLibraryBaseRule.class}, + factoryClass = BazelCcLibrary.class) + public static final class CcLibraryRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + SafeImplicitOutputsFunction implicitOutputsFunction = new SafeImplicitOutputsFunction() { + @Override + public Iterable<String> getImplicitOutputs(AttributeMap rule) { + boolean alwaysLink = rule.get("alwayslink", Type.BOOLEAN); + boolean linkstatic = rule.get("linkstatic", Type.BOOLEAN); + SafeImplicitOutputsFunction staticLib = fromTemplates( + alwaysLink + ? "%{dirname}lib%{basename}.lo" + : "%{dirname}lib%{basename}.a"); + SafeImplicitOutputsFunction allLibs = + linkstatic || CcLibrary.appearsToHaveNoObjectFiles(rule) + ? staticLib + : fromFunctions(staticLib, CC_LIBRARY_DYNAMIC_LIB); + return allLibs.getImplicitOutputs(rule); + } + }; + + return builder + .setImplicitOutputsFunction(implicitOutputsFunction) + .add(attr("alwayslink", BOOLEAN). + nonconfigurable("value is referenced in an ImplicitOutputsFunction")) + .add(attr("implements", LABEL_LIST) + .allowedFileTypes() + .allowedRuleClasses("cc_public_library$headers")) + .override(attr("linkstatic", BOOLEAN).value(false) + .nonconfigurable("value is referenced in an ImplicitOutputsFunction")) + .build(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppSemantics.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppSemantics.java new file mode 100644 index 0000000..3771e6c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppSemantics.java
@@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.rules.cpp.CppCompilationContext.Builder; +import com.google.devtools.build.lib.rules.cpp.CppCompileActionBuilder; +import com.google.devtools.build.lib.rules.cpp.CppCompileActionContext; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppHelper; +import com.google.devtools.build.lib.rules.cpp.CppSemantics; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * C++ compilation semantics. + */ +public class BazelCppSemantics implements CppSemantics { + public static final CppSemantics INSTANCE = new BazelCppSemantics(); + + private BazelCppSemantics() { + } + + @Override + public PathFragment getEffectiveSourcePath(Artifact source) { + return source.getRootRelativePath(); + } + + @Override + public void finalizeCompileActionBuilder( + RuleContext ruleContext, CppCompileActionBuilder actionBuilder) { + actionBuilder.setCppConfiguration(ruleContext.getFragment(CppConfiguration.class)); + actionBuilder.setActionContext(CppCompileActionContext.class); + actionBuilder.addTransitiveMandatoryInputs(CppHelper.getToolchain(ruleContext).getCompile()); + } + + @Override + public void setupCompilationContext(RuleContext ruleContext, Builder contextBuilder) { + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/BazelGenRuleRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/BazelGenRuleRule.java new file mode 100644 index 0000000..eabb4e9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/BazelGenRuleRule.java
@@ -0,0 +1,77 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.rules.genrule; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.LICENSE; +import static com.google.devtools.build.lib.packages.Type.OUTPUT_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.Type; + +/** + * Rule definition for the genrule rule. + */ +@BlazeRule(name = "genrule", + ancestors = { BaseRuleClasses.RuleBase.class }, + factoryClass = GenRule.class) +public final class BazelGenRuleRule implements RuleDefinition { + public static final String GENRULE_SETUP_LABEL = "//tools/genrule:genrule-setup.sh"; + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .setOutputToGenfiles() + .add(attr("srcs", LABEL_LIST) + .direct_compile_time_input() + .legacyAllowAnyFileType()) + .add(attr("tools", LABEL_LIST).cfg(HOST).legacyAllowAnyFileType()) + .add(attr("$genrule_setup", LABEL).cfg(HOST).value(env.getLabel(GENRULE_SETUP_LABEL))) + .add(attr("outs", OUTPUT_LIST).mandatory()) + .add(attr("cmd", STRING).mandatory()) + .add(attr("output_to_bindir", BOOLEAN).value(false) + .nonconfigurable("policy decision: no reason for this to depend on the configuration")) + .add(attr("local", BOOLEAN).value(false)) + .add(attr("message", STRING)) + .add(attr("output_licenses", LICENSE)) + .add(attr("executable", BOOLEAN).value(false)) + .add(attr("stamp", BOOLEAN).value(false)) + .add(attr("heuristic_label_expansion", BOOLEAN).value(true)) + .add(attr("$is_executable", BOOLEAN) + .nonconfigurable("Called from RunCommand.isExecutable, which takes a Target") + .value( + new Attribute.ComputedDefault("outs", "executable") { + @Override + public Object getDefault(AttributeMap rule) { + return (rule.get("outs", Type.OUTPUT_LIST).size() == 1) + && rule.get("executable", BOOLEAN); + } + })) + .removeAttribute("data") + .removeAttribute("deps") + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/GenRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/GenRule.java new file mode 100644 index 0000000..d70f9a7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/GenRule.java
@@ -0,0 +1,219 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.genrule; + +import static com.google.devtools.build.lib.analysis.RunfilesProvider.withData; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.CommandHelper; +import com.google.devtools.build.lib.analysis.ConfigurationMakeVariableContext; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.MakeVariableExpander.ExpansionException; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.List; +import java.util.Map; + +/** + * An implementation of genrule. + */ +public class GenRule implements RuleConfiguredTargetFactory { + + public static final String GENRULE_SETUP_CMD = + "source tools/genrule/genrule-setup.sh; "; + + private Artifact getExecutable(RuleContext ruleContext, NestedSet<Artifact> filesToBuild) { + if (Iterables.size(filesToBuild) == 1) { + Artifact out = Iterables.getOnlyElement(filesToBuild); + if (ruleContext.attributes().get("executable", Type.BOOLEAN)) { + return out; + } + } + return null; + } + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + final List<Artifact> resolvedSrcs = Lists.newArrayList(); + + final NestedSet<Artifact> filesToBuild = + NestedSetBuilder.wrap(Order.STABLE_ORDER, ruleContext.getOutputArtifacts()); + if (filesToBuild.isEmpty()) { + ruleContext.attributeError("outs", "Genrules without outputs don't make sense"); + } + if (ruleContext.attributes().get("executable", Type.BOOLEAN) + && Iterables.size(filesToBuild) > 1) { + ruleContext.attributeError("executable", + "if genrules produce executables, they are allowed only one output. " + + "If you need the executable=1 argument, then you should split this genrule into " + + "genrules producing single outputs"); + } + + ImmutableMap.Builder<Label, Iterable<Artifact>> labelMap = ImmutableMap.builder(); + for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("srcs", Mode.TARGET)) { + Iterable<Artifact> files = dep.getProvider(FileProvider.class).getFilesToBuild(); + Iterables.addAll(resolvedSrcs, files); + labelMap.put(dep.getLabel(), files); + } + + CommandHelper commandHelper = new CommandHelper(ruleContext, ruleContext + .getPrerequisites("tools", Mode.HOST, FilesToRunProvider.class), labelMap.build()); + + if (ruleContext.hasErrors()) { + return null; + } + + String baseCommand = commandHelper.resolveCommandAndExpandLabels( + ruleContext.attributes().get("heuristic_label_expansion", Type.BOOLEAN), false); + + // Adds the genrule environment setup script before the actual shell command + String command = GENRULE_SETUP_CMD + baseCommand; + + command = resolveCommand(ruleContext, command, resolvedSrcs, filesToBuild); + + String message = ruleContext.attributes().get("message", Type.STRING); + if (message.isEmpty()) { + message = "Executing genrule"; + } + + ImmutableMap<String, String> env = + ruleContext.getConfiguration().getDefaultShellEnvironment(); + + Map<String, String> executionInfo = Maps.newLinkedHashMap(); + executionInfo.putAll(TargetUtils.getExecutionInfo(ruleContext.getRule())); + + if (ruleContext.attributes().get("local", Type.BOOLEAN)) { + executionInfo.put("local", ""); + } + + NestedSetBuilder<Artifact> inputs = NestedSetBuilder.stableOrder(); + inputs.addAll(resolvedSrcs); + inputs.addAll(commandHelper.getResolvedTools()); + FilesToRunProvider genruleSetup = + ruleContext.getPrerequisite("$genrule_setup", Mode.HOST, FilesToRunProvider.class); + inputs.addAll(genruleSetup.getFilesToRun()); + List<String> argv = commandHelper.buildCommandLine(command, inputs, ".genrule_script.sh"); + + if (ruleContext.attributes().get("stamp", Type.BOOLEAN)) { + inputs.add(ruleContext.getAnalysisEnvironment().getStableWorkspaceStatusArtifact()); + inputs.add(ruleContext.getAnalysisEnvironment().getVolatileWorkspaceStatusArtifact()); + } + + ruleContext.registerAction(new GenRuleAction( + ruleContext.getActionOwner(), inputs.build(), filesToBuild, argv, env, + ImmutableMap.copyOf(executionInfo), commandHelper.getRemoteRunfileManifestMap(), + message + ' ' + ruleContext.getLabel())); + + RunfilesProvider runfilesProvider = withData( + // No runfiles provided if not a data dependency. + Runfiles.EMPTY, + // We only need to consider the outputs of a genrule + // No need to visit the dependencies of a genrule. They cross from the target into the host + // configuration, because the dependencies of a genrule are always built for the host + // configuration. + new Runfiles.Builder().addTransitiveArtifacts(filesToBuild).build()); + + return new RuleConfiguredTargetBuilder(ruleContext) + .setFilesToBuild(filesToBuild) + .setRunfilesSupport(null, getExecutable(ruleContext, filesToBuild)) + .addProvider(RunfilesProvider.class, runfilesProvider) + .build(); + } + + private String resolveCommand(final RuleContext ruleContext, final String command, + final List<Artifact> resolvedSrcs, final NestedSet<Artifact> filesToBuild) { + return ruleContext.expandMakeVariables("cmd", command, new ConfigurationMakeVariableContext( + ruleContext.getRule().getPackage(), ruleContext.getConfiguration()) { + @Override + public String lookupMakeVariable(String name) throws ExpansionException { + if (name.equals("SRCS")) { + return Artifact.joinExecPaths(" ", resolvedSrcs); + } else if (name.equals("<")) { + return expandSingletonArtifact(resolvedSrcs, "$<", "input file"); + } else if (name.equals("OUTS")) { + return Artifact.joinExecPaths(" ", filesToBuild); + } else if (name.equals("@")) { + return expandSingletonArtifact(filesToBuild, "$@", "output file"); + } else if (name.equals("@D")) { + // The output directory. If there is only one filename in outs, + // this expands to the directory containing that file. If there are + // multiple filenames, this variable instead expands to the + // package's root directory in the genfiles tree, even if all the + // generated files belong to the same subdirectory! + if (Iterables.size(filesToBuild) == 1) { + Artifact outputFile = Iterables.getOnlyElement(filesToBuild); + PathFragment relativeOutputFile = outputFile.getExecPath(); + if (relativeOutputFile.segmentCount() <= 1) { + // This should never happen, since the path should contain at + // least a package name and a file name. + throw new IllegalStateException("$(@D) for genrule " + ruleContext.getLabel() + + " has less than one segment"); + } + return relativeOutputFile.getParentDirectory().getPathString(); + } else { + PathFragment dir; + if (ruleContext.getRule().hasBinaryOutput()) { + dir = ruleContext.getConfiguration().getBinFragment(); + } else { + dir = ruleContext.getConfiguration().getGenfilesFragment(); + } + PathFragment relPath = ruleContext.getRule().getLabel().getPackageFragment(); + return dir.getRelative(relPath).getPathString(); + } + } else { + return super.lookupMakeVariable(name); + } + } + } + ); + } + + // Returns the path of the sole element "artifacts", generating an exception + // with an informative error message iff the set is not a singleton. + // + // Used to expand "$<", "$@" + private String expandSingletonArtifact(Iterable<Artifact> artifacts, + String variable, + String artifactName) + throws ExpansionException { + if (Iterables.isEmpty(artifacts)) { + throw new ExpansionException("variable '" + variable + + "' : no " + artifactName); + } else if (Iterables.size(artifacts) > 1) { + throw new ExpansionException("variable '" + variable + + "' : more than one " + artifactName); + } + return Iterables.getOnlyElement(artifacts).getExecPathString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/GenRuleAction.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/GenRuleAction.java new file mode 100644 index 0000000..0a9b3e7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/genrule/GenRuleAction.java
@@ -0,0 +1,62 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.genrule; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.List; + +/** + * A spawn action for genrules. Genrules are handled specially in that inputs and outputs are + * checked for directories. + */ +public final class GenRuleAction extends SpawnAction { + + private static final ResourceSet GENRULE_RESOURCES = + // Not chosen scientifically/carefully. 300MB memory, 100% CPU, 20% of total I/O. + new ResourceSet(300, 1.0, 0.0); + + public GenRuleAction(ActionOwner owner, + Iterable<Artifact> inputs, + Iterable<Artifact> outputs, + List<String> argv, + ImmutableMap<String, String> environment, + ImmutableMap<String, String> executionInfo, + ImmutableMap<PathFragment, Artifact> runfilesManifests, + String progressMessage) { + super(owner, inputs, outputs, GENRULE_RESOURCES, + CommandLine.of(argv, false), environment, executionInfo, progressMessage, + runfilesManifests, + "Genrule", null); + } + + @Override + protected void internalExecute( + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException { + EventHandler reporter = actionExecutionContext.getExecutor().getEventHandler(); + checkInputsForDirectories(reporter, actionExecutionContext.getMetadataHandler()); + super.internalExecute(actionExecutionContext); + checkOutputsForDirectories(reporter); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinary.java new file mode 100644 index 0000000..04713c2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinary.java
@@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.java; + +import com.google.devtools.build.lib.rules.java.JavaBinary; + +/** + * Implementation of {@code java_binary} with Bazel semantics. + */ +public class BazelJavaBinary extends JavaBinary { + public BazelJavaBinary() { + super(BazelJavaSemantics.INSTANCE); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java new file mode 100644 index 0000000..279cbdb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.java; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.bazel.rules.BazelBaseRuleClasses; +import com.google.devtools.build.lib.bazel.rules.java.BazelJavaRuleClasses.BaseJavaBinaryRule; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for the java_binary rule. + */ +@BlazeRule(name = "java_binary", + ancestors = { BaseJavaBinaryRule.class, + BazelBaseRuleClasses.BinaryBaseRule.class }, + factoryClass = BazelJavaBinary.class) +public final class BazelJavaBinaryRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .setImplicitOutputsFunction(BazelJavaRuleClasses.JAVA_BINARY_IMPLICIT_OUTPUTS) + .override(attr("$is_executable", BOOLEAN).nonconfigurable("automatic").value( + new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + return rule.get("create_executable", BOOLEAN); + } + })) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBuildInfoFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBuildInfoFactory.java new file mode 100644 index 0000000..db33897 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBuildInfoFactory.java
@@ -0,0 +1,61 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.java; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.rules.java.BuildInfoPropertiesTranslator; +import com.google.devtools.build.lib.rules.java.GenericBuildInfoPropertiesTranslator; +import com.google.devtools.build.lib.rules.java.JavaBuildInfoFactory; + +import java.util.Map; + +/** + * BuildInfoFactory for Java. + */ +public class BazelJavaBuildInfoFactory extends JavaBuildInfoFactory { + private static final Map<String, String> VOLATILE_KEYS = ImmutableMap + .<String, String>builder() + .put("build.time", "%BUILD_TIME%") + .put("build.timestamp.as.int", "%BUILD_TIMESTAMP%") + .put("build.timestamp", "%BUILD_TIMESTAMP%") + .build(); + + private static final Map<String, String> NONVOLATILE_KEYS = ImmutableMap + .<String, String>builder() + .build(); + + private static final Map<String, String> REDACTED_KEYS = ImmutableMap + .<String, String>builder() + .put("build.time", "Thu Jan 01 00:00:00 1970 (0)") + .put("build.timestamp.as.int", "0") + .put("build.timestamp", "Thu Jan 01 00:00:00 1970 (0)") + .build(); + + @Override + protected BuildInfoPropertiesTranslator createVolatileTranslator() { + return new GenericBuildInfoPropertiesTranslator(VOLATILE_KEYS); + } + + @Override + protected BuildInfoPropertiesTranslator createNonVolatileTranslator() { + return new GenericBuildInfoPropertiesTranslator(NONVOLATILE_KEYS); + } + + @Override + protected BuildInfoPropertiesTranslator createRedactedTranslator() { + return new GenericBuildInfoPropertiesTranslator(REDACTED_KEYS); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaImport.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaImport.java new file mode 100644 index 0000000..6c7dcd4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaImport.java
@@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.java; + +import com.google.devtools.build.lib.rules.java.JavaImport; + +/** + * Implementation of {@code java_import} with Bazel semantics. + */ +public class BazelJavaImport extends JavaImport { + public BazelJavaImport() { + super(BazelJavaSemantics.INSTANCE); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaImportRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaImportRule.java new file mode 100644 index 0000000..132df23 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaImportRule.java
@@ -0,0 +1,53 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.java; + +import static com.google.devtools.build.lib.packages.Attribute.ANY_EDGE; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.bazel.rules.java.BazelJavaRuleClasses.IjarBaseRule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.rules.java.JavaImportBaseRule; + +/** + * Rule definition for the java_import rule. + */ +@BlazeRule(name = "java_import", + ancestors = { JavaImportBaseRule.class, IjarBaseRule.class }, + factoryClass = BazelJavaImport.class) +public final class BazelJavaImportRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(exports) --> + Targets to make available to users of this rule. + ${SYNOPSIS} + See <a href="#java_library.exports">java_library.exports</a>. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("exports", LABEL_LIST) + .allowedRuleClasses(ImmutableSet.of( + "java_library", "java_import", "cc_library", "cc_binary")) + .allowedFileTypes() // none allowed + .validityPredicate(ANY_EDGE)) + .build(); + + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaLibrary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaLibrary.java new file mode 100644 index 0000000..6af9450 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaLibrary.java
@@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.java; + +import com.google.devtools.build.lib.rules.java.JavaLibrary; + +/** + * Implementation of {@code java_library} with Bazel semantics. + */ +public class BazelJavaLibrary extends JavaLibrary { + public BazelJavaLibrary() { + super(BazelJavaSemantics.INSTANCE); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaLibraryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaLibraryRule.java new file mode 100644 index 0000000..04c8a0f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaLibraryRule.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.java; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.bazel.rules.java.BazelJavaRuleClasses.JavaRule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Common attributes for Java rules. + */ +@BlazeRule(name = "java_library", + ancestors = { JavaRule.class }, + factoryClass = BazelJavaLibrary.class) +public final class BazelJavaLibraryRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) { + + return builder + .setImplicitOutputsFunction(BazelJavaRuleClasses.JAVA_LIBRARY_IMPLICIT_OUTPUTS) + .add(attr("exports", LABEL_LIST) + .allowedRuleClasses(BazelJavaRuleClasses.ALLOWED_RULES_IN_DEPS) + .allowedFileTypes(/*May not have files in exports!*/)) + .add(attr("neverlink", BOOLEAN).value(false)) + .override(attr("javacopts", STRING_LIST)) + .add(attr("exported_plugins", LABEL_LIST).cfg(HOST).allowedRuleClasses("java_plugin") + .legacyAllowAnyFileType()) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaPlugin.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaPlugin.java new file mode 100644 index 0000000..e6d3478 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaPlugin.java
@@ -0,0 +1,27 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.java; + +import com.google.devtools.build.lib.rules.java.JavaPlugin; + +/** + * Implementation of the {@code java_plugin} rule for bazel. + */ +public class BazelJavaPlugin extends JavaPlugin { + + public BazelJavaPlugin() { + super(BazelJavaSemantics.INSTANCE); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaPluginRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaPluginRule.java new file mode 100644 index 0000000..cbb9411 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaPluginRule.java
@@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.java; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for the java_plugin rule. + */ +@BlazeRule(name = "java_plugin", + ancestors = { BazelJavaLibraryRule.class }, + factoryClass = BazelJavaPlugin.class) +public final class BazelJavaPluginRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .setImplicitOutputsFunction(BazelJavaRuleClasses.JAVA_LIBRARY_IMPLICIT_OUTPUTS) + .override(builder.copy("deps").validityPredicate(Attribute.ANY_EDGE)) + .override(builder.copy("srcs").validityPredicate(Attribute.ANY_EDGE)) + .add(attr("processor_class", STRING)) + .removeAttribute("runtime_deps") + .removeAttribute("exports") + .removeAttribute("exported_plugins") + .build(); + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaRuleClasses.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaRuleClasses.java new file mode 100644 index 0000000..663b82a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaRuleClasses.java
@@ -0,0 +1,173 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.java; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromFunctions; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; +import static com.google.devtools.build.lib.packages.Type.TRISTATE; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.bazel.rules.cpp.BazelCppRuleClasses; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.PredicateWithMessage; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.packages.RuleClass.PackageNameConstraint; +import com.google.devtools.build.lib.packages.TriState; +import com.google.devtools.build.lib.rules.java.JavaSemantics; +import com.google.devtools.build.lib.util.FileTypeSet; + +import java.util.Set; + +/** + * Rule class definitions for Java rules. + */ +public class BazelJavaRuleClasses { + + public static final PredicateWithMessage<Rule> JAVA_PACKAGE_NAMES = new PackageNameConstraint( + PackageNameConstraint.ANY_SEGMENT, "java", "javatests"); + + public static final ImplicitOutputsFunction JAVA_BINARY_IMPLICIT_OUTPUTS = + fromFunctions(JavaSemantics.JAVA_BINARY_CLASS_JAR, JavaSemantics.JAVA_BINARY_SOURCE_JAR, + JavaSemantics.JAVA_BINARY_DEPLOY_JAR, JavaSemantics.JAVA_BINARY_DEPLOY_SOURCE_JAR); + + static final ImplicitOutputsFunction JAVA_LIBRARY_IMPLICIT_OUTPUTS = + fromFunctions(JavaSemantics.JAVA_LIBRARY_CLASS_JAR, JavaSemantics.JAVA_LIBRARY_SOURCE_JAR); + + /** + * Common attributes for rules that depend on ijar. + */ + @BlazeRule(name = "$ijar_base_rule", + type = RuleClassType.ABSTRACT) + public static final class IjarBaseRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .add(attr("$ijar", LABEL).cfg(HOST).exec().value(env.getLabel("//tools/defaults:ijar"))) + .setPreferredDependencyPredicate(JavaSemantics.JAVA_SOURCE) + .build(); + } + } + + + /** + * Common attributes for Java rules. + */ + @BlazeRule(name = "$java_base_rule", + type = RuleClassType.ABSTRACT, + ancestors = { IjarBaseRule.class }) + public static final class JavaBaseRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .add(attr(":jvm", LABEL).cfg(HOST).value(JavaSemantics.JVM)) + .add(attr(":host_jdk", LABEL).cfg(HOST).value(JavaSemantics.HOST_JDK)) + .add(attr(":java_toolchain", LABEL).value(JavaSemantics.JAVA_TOOLCHAIN)) + .add(attr("$java_langtools", LABEL).cfg(HOST) + .value(env.getLabel("//tools/defaults:java_langtools"))) + .add(attr("$javac_bootclasspath", LABEL).cfg(HOST) + .value(env.getLabel(JavaSemantics.JAVAC_BOOTCLASSPATH_LABEL))) + .add(attr("$javabuilder", LABEL).cfg(HOST) + .value(env.getLabel(JavaSemantics.JAVABUILDER_LABEL))) + .add(attr("$singlejar", LABEL).cfg(HOST) + .value(env.getLabel(JavaSemantics.SINGLEJAR_LABEL))) + .build(); + } + } + + static final Set<String> ALLOWED_RULES_IN_DEPS = ImmutableSet.of( + "cc_binary", // NB: linkshared=1 + "cc_library", + "genrule", + "genproto", // TODO(bazel-team): we should filter using providers instead (skylark rule). + "java_import", + "java_library", + "sh_binary", + "sh_library"); + + /** + * Common attributes for Java rules. + */ + @BlazeRule(name = "$java_rule", + type = RuleClassType.ABSTRACT, + ancestors = { BaseRuleClasses.RuleBase.class, JavaBaseRule.class }) + public static final class JavaRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .override(builder.copy("deps") + .allowedFileTypes(JavaSemantics.JAR) + .allowedRuleClasses(ALLOWED_RULES_IN_DEPS) + .skipAnalysisTimeFileTypeCheck()) + .add(attr("runtime_deps", LABEL_LIST) + .allowedFileTypes(JavaSemantics.JAR) + .allowedRuleClasses(ALLOWED_RULES_IN_DEPS) + .skipAnalysisTimeFileTypeCheck()) + .add(attr("srcs", LABEL_LIST) + .orderIndependent() + .direct_compile_time_input() + .allowedFileTypes(JavaSemantics.JAVA_SOURCE, JavaSemantics.JAR, + JavaSemantics.SOURCE_JAR, JavaSemantics.PROPERTIES)) + .add(attr("resources", LABEL_LIST).orderIndependent() + .allowedFileTypes(FileTypeSet.ANY_FILE)) + .add(attr("plugins", LABEL_LIST).cfg(HOST).allowedRuleClasses("java_plugin") + .legacyAllowAnyFileType()) + .add(attr(":java_plugins", LABEL_LIST) + .cfg(HOST) + .allowedRuleClasses("java_plugin") + .silentRuleClassFilter() + .value(JavaSemantics.JAVA_PLUGINS)) + .add(attr("javacopts", STRING_LIST)) + .build(); + } + } + + /** + * Base class for rule definitions producing Java binaries. + */ + @BlazeRule(name = "$base_java_binary", + type = RuleClassType.ABSTRACT, + ancestors = { JavaRule.class, + // java_binary and java_test require the crosstool C++ runtime + // libraries (libstdc++.so, libgcc_s.so). + // TODO(bazel-team): Add tests for Java+dynamic runtime. + BazelCppRuleClasses.CcLinkingRule.class }) + public static final class BaseJavaBinaryRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) { + return builder + .add(attr("classpath_resources", LABEL_LIST).legacyAllowAnyFileType()) + .add(attr("jvm_flags", STRING_LIST)) + .add(attr("main_class", STRING)) + .add(attr("create_executable", BOOLEAN).nonconfigurable("internal").value(true)) + .add(attr("deploy_manifest_lines", STRING_LIST)) + .add(attr("stamp", TRISTATE).value(TriState.AUTO)) + .add(attr(":java_launcher", LABEL).value(JavaSemantics.JAVA_LAUNCHER)) // blaze flag + .build(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java new file mode 100644 index 0000000..b301161 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java
@@ -0,0 +1,341 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.java; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction; +import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.ComputedSubstitution; +import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.Substitution; +import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.Template; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.java.DeployArchiveBuilder; +import com.google.devtools.build.lib.rules.java.DeployArchiveBuilder.Compression; +import com.google.devtools.build.lib.rules.java.DirectDependencyProvider; +import com.google.devtools.build.lib.rules.java.DirectDependencyProvider.Dependency; +import com.google.devtools.build.lib.rules.java.JavaCommon; +import com.google.devtools.build.lib.rules.java.JavaCompilationArtifacts; +import com.google.devtools.build.lib.rules.java.JavaCompilationHelper; +import com.google.devtools.build.lib.rules.java.JavaConfiguration; +import com.google.devtools.build.lib.rules.java.JavaHelper; +import com.google.devtools.build.lib.rules.java.JavaPrimaryClassProvider; +import com.google.devtools.build.lib.rules.java.JavaSemantics; +import com.google.devtools.build.lib.rules.java.JavaTargetAttributes; +import com.google.devtools.build.lib.rules.java.JavaUtil; +import com.google.devtools.build.lib.rules.java.Jvm; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Semantics for Bazel Java rules + */ +public class BazelJavaSemantics implements JavaSemantics { + + public static final BazelJavaSemantics INSTANCE = new BazelJavaSemantics(); + + private static final Template STUB_SCRIPT = + Template.forResource(BazelJavaSemantics.class, "java_stub_template.txt"); + + public static final InstrumentationSpec GREEDY_COLLECTION_SPEC = new InstrumentationSpec( + FileTypeSet.of(FileType.of(".sh"), JavaSemantics.JAVA_SOURCE), + "srcs", "deps", "data"); + + private BazelJavaSemantics() { + } + + private boolean isJavaBinaryOrJavaTest(RuleContext ruleContext) { + String ruleClass = ruleContext.getRule().getRuleClass(); + return ruleClass.equals("java_binary") || ruleClass.equals("java_test"); + } + + @Override + public void checkRule(RuleContext ruleContext, JavaCommon javaCommon) { + if (isJavaBinaryOrJavaTest(ruleContext)) { + checkMainClass(ruleContext, javaCommon); + } + } + + private String getMainClassInternal(RuleContext ruleContext) { + return ruleContext.getRule().isAttrDefined("main_class", Type.STRING) + ? ruleContext.attributes().get("main_class", Type.STRING) : ""; + } + + private void checkMainClass(RuleContext ruleContext, JavaCommon javaCommon) { + boolean createExecutable = ruleContext.attributes().get("create_executable", Type.BOOLEAN); + String mainClass = getMainClassInternal(ruleContext); + + if (!createExecutable && !mainClass.isEmpty()) { + ruleContext.ruleError("main class must not be specified when executable is not created"); + } + + if (createExecutable && mainClass.isEmpty()) { + if (javaCommon.getSrcsArtifacts().isEmpty()) { + ruleContext.ruleError( + "need at least one of 'main_class', 'use_testrunner' or Java source files"); + } + mainClass = javaCommon.determinePrimaryClass(javaCommon.getSrcsArtifacts()); + if (mainClass == null) { + ruleContext.ruleError("cannot determine main class for launching " + + "(found neither a source file '" + ruleContext.getTarget().getName() + + ".java', nor a main_class attribute, and package name " + + "doesn't include 'java' or 'javatests')"); + } + } + } + + @Override + public String getMainClass(RuleContext ruleContext, JavaCommon javaCommon) { + checkMainClass(ruleContext, javaCommon); + return getMainClassInternal(ruleContext); + } + + @Override + public ImmutableList<Artifact> collectResources(RuleContext ruleContext) { + if (!ruleContext.getRule().isAttrDefined("resources", Type.LABEL_LIST)) { + return ImmutableList.of(); + } + + return ruleContext.getPrerequisiteArtifacts("resources", Mode.TARGET).list(); + } + + @Override + public Artifact createInstrumentationMetadataArtifact( + AnalysisEnvironment analysisEnvironment, Artifact outputJar) { + return null; + } + + @Override + public void buildJavaCommandLine(Collection<Artifact> outputs, BuildConfiguration configuration, + CustomCommandLine.Builder result) { + } + + @Override + public void createStubAction(RuleContext ruleContext, final JavaCommon javaCommon, + List<String> jvmFlags, Artifact executable, String javaStartClass, + String javaExecutable) { + + Preconditions.checkNotNull(jvmFlags); + Preconditions.checkNotNull(executable); + Preconditions.checkNotNull(javaStartClass); + Preconditions.checkNotNull(javaExecutable); + BuildConfiguration config = ruleContext.getConfiguration(); + + List<Substitution> arguments = new ArrayList<>(); + arguments.add(Substitution.of("%javabin%", javaExecutable)); + arguments.add(Substitution.of("%needs_runfiles%", + config.getFragment(Jvm.class).getJavaExecutable().isAbsolute() ? "0" : "1")); + arguments.add(new ComputedSubstitution("%classpath%") { + @Override + public String getValue() { + StringBuilder buffer = new StringBuilder(); + Iterable<Artifact> jars = javaCommon.getRuntimeClasspath(); + appendRunfilesRelativeEntries(buffer, jars, ':'); + return buffer.toString(); + } + }); + + arguments.add(Substitution.of("%java_start_class%", + ShellEscaper.escapeString(javaStartClass))); + arguments.add(Substitution.ofSpaceSeparatedList("%jvm_flags%", jvmFlags)); + + ruleContext.registerAction(new TemplateExpansionAction( + ruleContext.getActionOwner(), executable, STUB_SCRIPT, arguments, true)); + } + + /** + * Builds a class path by concatenating the root relative paths of the artifacts separated by the + * delimiter. Each relative path entry is prepended with "${RUNPATH}" which will be expanded by + * the stub script at runtime, to either "${JAVA_RUNFILES}/" or if we are lucky, the empty + * string. + * + * @param buffer the buffer to use for concatenating the entries + * @param artifacts the entries to concatenate in the buffer + * @param delimiter the delimiter character to separate the entries + */ + private static void appendRunfilesRelativeEntries(StringBuilder buffer, + Iterable<Artifact> artifacts, char delimiter) { + for (Artifact artifact : artifacts) { + if (buffer.length() > 0) { + buffer.append(delimiter); + } + buffer.append("${RUNPATH}"); + buffer.append(artifact.getRootRelativePath().getPathString()); + } + } + + @Override + public void addRunfilesForBinary(RuleContext ruleContext, Artifact launcher, + Runfiles.Builder runfilesBuilder) { + } + + @Override + public void addRunfilesForLibrary(RuleContext ruleContext, Runfiles.Builder runfilesBuilder) { + } + + @Override + public void collectTargetsTreatedAsDeps( + RuleContext ruleContext, ImmutableList.Builder<TransitiveInfoCollection> builder) { + } + + @Override + public InstrumentationSpec getCoverageInstrumentationSpec() { + return GREEDY_COLLECTION_SPEC.withAttributes("srcs", "deps", "data", "exports", "runtime_deps"); + } + + @Override + public Iterable<String> getExtraJavacOpts(RuleContext ruleContext) { + return ImmutableList.<String>of(); + } + + @Override + public void addProviders(RuleContext ruleContext, + JavaCommon javaCommon, + List<String> jvmFlags, + Artifact classJar, + Artifact srcJar, + Artifact gensrcJar, + ImmutableMap<Artifact, Artifact> compilationToRuntimeJarMap, + JavaCompilationHelper helper, + NestedSetBuilder<Artifact> filesBuilder, + RuleConfiguredTargetBuilder ruleBuilder) { + if (!isJavaBinaryOrJavaTest(ruleContext)) { + Artifact outputDepsProto = helper.getOutputDepsProtoArtifact(); + if (outputDepsProto != null && helper.getStrictJavaDeps() != StrictDepsMode.OFF) { + ImmutableList<Dependency> strictDependencies = + javaCommon.computeStrictDepsFromJavaAttributes(helper.getAttributes()); + ruleBuilder.add(DirectDependencyProvider.class, + new DirectDependencyProvider(strictDependencies)); + } + } else { + boolean createExec = ruleContext.attributes().get("create_executable", Type.BOOLEAN); + ruleBuilder.add(JavaPrimaryClassProvider.class, + new JavaPrimaryClassProvider(createExec ? getMainClassInternal(ruleContext) : null)); + } + } + + + @Override + public Iterable<String> getJvmFlags(RuleContext ruleContext, JavaCommon javaCommon, + Artifact launcher, List<String> userJvmFlags) { + return userJvmFlags; + } + + @Override + public String addCoverageSupport(JavaCompilationHelper helper, + JavaTargetAttributes.Builder attributes, + Artifact executable, Artifact instrumentationMetadata, + JavaCompilationArtifacts.Builder javaArtifactsBuilder, String mainClass) { + return mainClass; + } + + @Override + public boolean useStrictJavaDeps(BuildConfiguration configuration) { + return true; + } + + @Override + public CustomCommandLine buildSingleJarCommandLine(BuildConfiguration configuration, + Artifact output, String mainClass, ImmutableList<String> manifestLines, + Iterable<Artifact> buildInfoFiles, ImmutableList<Artifact> resources, + Iterable<Artifact> classpath, boolean includeBuildData, + Compression compression, Artifact launcher) { + return DeployArchiveBuilder.defaultSingleJarCommandLine(output, mainClass, manifestLines, + buildInfoFiles, resources, classpath, includeBuildData, compression, launcher).build(); + } + + @Override + public Collection<Artifact> translate(RuleContext ruleContext, JavaConfiguration javaConfig, + List<Artifact> messages) { + return ImmutableList.<Artifact>of(); + } + + @Override + public Artifact getLauncher(RuleContext ruleContext, JavaCommon common, + DeployArchiveBuilder deployArchiveBuilder, Runfiles.Builder runfilesBuilder, + List<String> jvmFlags, JavaTargetAttributes.Builder attributesBuilder) { + return JavaHelper.launcherArtifactForTarget(this, ruleContext); + } + + @Override + public void addDependenciesForRunfiles(RuleContext ruleContext, Runfiles.Builder builder) { + } + + @Override + public boolean forceUseJavaLauncherTarget(RuleContext ruleContext) { + return false; + } + + @Override + public void addArtifactToJavaTargetAttribute(JavaTargetAttributes.Builder builder, + Artifact srcArtifact) { + } + + @Override + public void commonDependencyProcessing(RuleContext ruleContext, + JavaTargetAttributes.Builder attributes, + Collection<? extends TransitiveInfoCollection> deps) { + } + + @Override + public Collection<ActionInput> getExtraJavaCompileOutputs(PathFragment classDirectory) { + return ImmutableList.of(); + } + + @Override + public PathFragment getJavaResourcePath(PathFragment path) { + PathFragment javaPath = JavaUtil.getJavaPath(path); + return javaPath == null ? path : javaPath; + } + + @Override + public List<String> getExtraArguments(RuleContext ruleContext, JavaCommon javaCommon) { + if (ruleContext.getRule().getRuleClass().equals("java_test")) { + if (ruleContext.getConfiguration().getTestArguments().isEmpty() + && !ruleContext.attributes().isAttributeValueExplicitlySpecified("args")) { + ImmutableList.Builder<String> builder = ImmutableList.builder(); + for (Artifact artifact : javaCommon.getSrcsArtifacts()) { + PathFragment path = artifact.getRootRelativePath(); + String className = JavaUtil.getJavaFullClassname(FileSystemUtils.removeExtension(path)); + if (className != null) { + builder.add(className); + } + } + return builder.build(); + } + } + return ImmutableList.<String>of(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTest.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTest.java new file mode 100644 index 0000000..ca94814 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTest.java
@@ -0,0 +1,27 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.java; + +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.java.JavaBinary; + +/** + * An implementation of {@code java_test} rules. + */ +public class BazelJavaTest extends JavaBinary implements RuleConfiguredTargetFactory { + public BazelJavaTest() { + super(BazelJavaSemantics.INSTANCE); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTestRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTestRule.java new file mode 100644 index 0000000..fe881b7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTestRule.java
@@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.java; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.STRING; +import static com.google.devtools.build.lib.packages.Type.TRISTATE; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.bazel.rules.java.BazelJavaRuleClasses.BaseJavaBinaryRule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.packages.TriState; +import com.google.devtools.build.lib.rules.java.JavaSemantics; + +/** + * Rule definition for the java_test rule. + */ +@BlazeRule(name = "java_test", + type = RuleClassType.TEST, + ancestors = { BaseJavaBinaryRule.class, + BaseRuleClasses.TestBaseRule.class }, + factoryClass = BazelJavaTest.class) +public final class BazelJavaTestRule implements RuleDefinition { + + private static final String JUNIT4_RUNNER = "org.junit.runner.JUnitCore"; + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .setImplicitOutputsFunction(BazelJavaRuleClasses.JAVA_BINARY_IMPLICIT_OUTPUTS) + .override(attr("main_class", STRING).value(JUNIT4_RUNNER)) + .override(attr("stamp", TRISTATE).value(TriState.NO)) + .override(attr(":java_launcher", LABEL).value(JavaSemantics.JAVA_LAUNCHER)) + .build(); + } +} \ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/java_stub_template.txt b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/java_stub_template.txt new file mode 100644 index 0000000..a17246f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/java_stub_template.txt
@@ -0,0 +1,195 @@ +#!/bin/bash --posix +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script was generated from java_stub_template.txt. Please +# don't edit it directly. +# +# If present, these flags should either be at the beginning of the command +# line, or they should be wrapped in a --wrapper_script_flag=FLAG argument. +# +# --debug Launch the JVM in remote debugging mode listening +# --debug=<port> to the specified port or the port set in the +# DEFAULT_JVM_DEBUG_PORT environment variable (e.g. +# 'export DEFAULT_JVM_DEBUG_PORT=8000') or else the +# default port of 5005. The JVM starts suspended +# unless the DEFAULT_JVM_DEBUG_SUSPEND environment +# variable is set to 'n'. +# --main_advice=<class> Run an alternate main class with the usual main +# program and arguments appended as arguments. +# --main_advice_classpath=<classpath> +# Prepend additional class path entries. +# --jvm_flag=<flag> Pass <flag> to the "java" command itself. +# <flag> may contain spaces. Can be used multiple times. +# --jvm_flags=<flags> Pass space-separated flags to the "java" command +# itself. Can be used multiple times. +# --singlejar Start the program from the packed-up deployment +# jar rather than from the classpath. +# --print_javabin Print the location of java executable binary and exit. +# +# The remainder of the command line is passed to the program. + +# Make it easy to insert 'set -x' or similar commands when debugging problems with this script. +eval "$JAVA_STUB_DEBUG" + +# Prevent problems where the caller has exported CLASSPATH, causing our +# computed value to be copied into the environment and double-counted +# against the argv limit. +unset CLASSPATH + +JVM_FLAGS_CMDLINE=() + +# Processes an argument for the wrapper. Returns 0 if the given argument +# was recognized as an argument for this wrapper, and 1 if it was not. +function process_wrapper_argument() { + case "$1" in + --debug) JVM_DEBUG_PORT="${DEFAULT_JVM_DEBUG_PORT:-5005}" ;; + --debug=*) JVM_DEBUG_PORT="${1#--debug=}" ;; + --main_advice=*) MAIN_ADVICE="${1#--main_advice=}" ;; + --main_advice_classpath=*) MAIN_ADVICE_CLASSPATH="${1#--main_advice_classpath=}" ;; + --jvm_flag=*) JVM_FLAGS_CMDLINE+=( "${1#--jvm_flag=}" ) ;; + --jvm_flags=*) JVM_FLAGS_CMDLINE+=( ${1#--jvm_flags=} ) ;; + --singlejar) SINGLEJAR=1 ;; + --print_javabin) PRINT_JAVABIN=1 ;; + *) + return 1 ;; + esac + return 0 +} + +die() { + printf "%s: $1\n" "$0" "${@:2}" >&2 + exit 1 +} + +# Parse arguments sequentially until the first unrecognized arg is encountered. +# Scan the remaining args for --wrapper_script_flag=X options and process them. +ARGS=() +for ARG in "$@"; do + if [[ "$ARG" == --wrapper_script_flag=* ]]; then + process_wrapper_argument "${ARG#--wrapper_script_flag=}" \ + || die "invalid wrapper argument '%s'" "$ARG" + elif [[ "${#ARGS}" > 0 ]] || ! process_wrapper_argument "$ARG"; then + ARGS+=( "$ARG" ) + fi +done + +# Find our runfiles tree. We need this to construct the classpath +# (unless --singlejar was passed). +# +# Call this program X. X was generated by a java_binary or java_test rule. +# X may be invoked in many ways: +# 1a) directly by a user, with $0 in the output tree +# 1b) via 'bazel run' (similar to case 1a) +# 2) directly by a user, with $0 in X's runfiles tree +# 3) by another program Y which has a data dependency on X, with $0 in Y's runfiles tree +# 4) via 'bazel test' +# 5) by a genrule cmd, with $0 in the output tree +# 6) case 3 in the context of a genrule +# +# For case 1, $0 will be a regular file, and the runfiles tree will be +# at $0.runfiles. +# For case 2 or 3, $0 will be a symlink to the file seen in case 1. +# For case 4, $JAVA_RUNFILES and $TEST_SRCDIR should already be set. +# Case 5 is handled like case 1. +# Case 6 is handled like case 3. + +case "$0" in + /*) self="$0" ;; + *) self="$PWD/$0" ;; +esac + +if [[ "$SINGLEJAR" != 1 || "%needs_runfiles%" == 1 ]]; then + if [[ -z "$JAVA_RUNFILES" ]]; then + while true; do + if [[ -e "$self.runfiles" ]]; then + JAVA_RUNFILES="$self.runfiles" + break + fi + if [[ $self == *.runfiles/* ]]; then + JAVA_RUNFILES="${self%.runfiles/*}.runfiles" + # don't break; this value is only a last resort for case 6b + fi + if [[ ! -L "$self" ]]; then + break + fi + readlink="$(readlink "$self")" + if [[ "$readlink" = /* ]]; then + self="$readlink" + else + # resolve relative symlink + self="${self%/*}/$readlink" + fi + done + if [[ -n "$JAVA_RUNFILES" ]]; then + export TEST_SRCDIR=${TEST_SRCDIR:-$JAVA_RUNFILES} + elif [[ -f "${self}_deploy.jar" && "%needs_runfiles%" == 0 ]]; then + SINGLEJAR=1; + else + die 'Cannot locate runfiles directory. (Set $JAVA_RUNFILES to inhibit searching.)' + fi + fi +fi + +# Set JAVABIN to the path to the JVM launcher. +%javabin% + +if [[ "$PRINT_JAVABIN" == 1 || "%java_start_class%" == "--print_javabin" ]]; then + echo -n "$JAVABIN" + exit 0 +fi + +if [[ "$SINGLEJAR" == 1 ]]; then + CLASSPATH="${self}_deploy.jar" + # Check for the deploy jar now. If it doesn't exist, we can print a + # more helpful error message than the JVM. + [[ -r "$CLASSPATH" ]] \ + || die "Option --singlejar was passed, but %s does not exist.\n (You may need to build it explicitly.)" "$CLASSPATH" +else + # Create the shortest classpath we can, by making it relative if possible. + RUNPATH="${JAVA_RUNFILES}/" + RUNPATH="${RUNPATH#$PWD/}" + CLASSPATH=%classpath% +fi + +if [[ -n "$JVM_DEBUG_PORT" ]]; then + JVM_DEBUG_SUSPEND=${DEFAULT_JVM_DEBUG_SUSPEND:-"y"} + JVM_DEBUG_FLAGS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=${JVM_DEBUG_SUSPEND},address=${JVM_DEBUG_PORT}" +fi + +if [[ -n "$MAIN_ADVICE_CLASSPATH" ]]; then + CLASSPATH="${MAIN_ADVICE_CLASSPATH}:${CLASSPATH}" +fi + +# Check if TEST_TMPDIR is available to use for scratch. +if [[ -n "$TEST_TMPDIR" && -d "$TEST_TMPDIR" ]]; then + JVM_FLAGS+=" -Djava.io.tmpdir=$TEST_TMPDIR" +fi + +ARGS=( + ${JVM_DEBUG_FLAGS} + ${JVM_FLAGS} + %jvm_flags% + "${JVM_FLAGS_CMDLINE[@]}" + ${MAIN_ADVICE} + %java_start_class% + "${ARGS[@]}") + +# Linux per-arg limit MAX_ARG_STRLEN == 128k! +if (("${#CLASSPATH}" > 120000)); then + set +o posix # Enable process substitution. + exec $JAVABIN -classpath @<(echo $CLASSPATH) "${ARGS[@]}" +else + exec $JAVABIN -classpath $CLASSPATH "${ARGS[@]}" +fi
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelIosTest.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelIosTest.java new file mode 100644 index 0000000..f45ca08 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelIosTest.java
@@ -0,0 +1,57 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.objc; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.objc.IosTest; +import com.google.devtools.build.lib.rules.objc.ObjcCommon; +import com.google.devtools.build.lib.rules.objc.XcodeProvider; + +/** + * Implementation for ios_test rule in Bazel. + */ +public final class BazelIosTest extends IosTest { + static final String IOS_TEST_ON_BAZEL_ATTR = "$ios_test_on_bazel"; + + @Override + public ConfiguredTarget create(RuleContext ruleContext, ObjcCommon common, + XcodeProvider xcodeProvider, NestedSet<Artifact> filesToBuild) throws InterruptedException { + Artifact testRunner = ruleContext.getPrerequisiteArtifact(IOS_TEST_ON_BAZEL_ATTR, Mode.TARGET); + Runfiles runfiles = new Runfiles.Builder() + .addArtifact(testRunner) + .build(); + RunfilesSupport runfilesSupport = + RunfilesSupport.withExecutable(ruleContext, runfiles, testRunner); + + return new RuleConfiguredTargetBuilder(ruleContext) + .setFilesToBuild(NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(filesToBuild) + .add(testRunner) + .build()) + .add(XcodeProvider.class, xcodeProvider) + .add(RunfilesProvider.class, RunfilesProvider.simple(runfiles)) + .setRunfilesSupport(runfilesSupport, testRunner) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelIosTestRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelIosTestRule.java new file mode 100644 index 0000000..114d454 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelIosTestRule.java
@@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.rules.objc.ApplicationSupport; +import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses; +import com.google.devtools.build.lib.rules.objc.XcodeSupport; + +/** + * Rule definition for the ios_test rule. + */ +@BlazeRule(name = "ios_test", + type = RuleClassType.TEST, + ancestors = { ObjcRuleClasses.IosTestBaseRule.class, + BaseRuleClasses.TestBaseRule.class }, + factoryClass = BazelIosTest.class) +public final class BazelIosTestRule implements RuleDefinition { + @Override + public RuleClass build(RuleClass.Builder builder, final RuleDefinitionEnvironment env) { + return builder + /*<!-- #BLAZE_RULE(ios_test).IMPLICIT_OUTPUTS --> + <ul> + <li><code><var>name</var>.ipa</code>: the test bundle as an + <code>.ipa</code> file + <li><code><var>name</var>.xcodeproj/project.pbxproj: An Xcode project file which can be + used to develop or build on a Mac.</li> + </ul> + <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/ + .setImplicitOutputsFunction( + ImplicitOutputsFunction.fromFunctions(ApplicationSupport.IPA, XcodeSupport.PBXPROJ)) + .add(attr(BazelIosTest.IOS_TEST_ON_BAZEL_ATTR, LABEL) + .value(env.getLabel("//tools/objc:ios_test_on_bazel")).exec()) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = ios_test, TYPE = TEST, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule provides a way to build iOS unit tests written in KIF, GTM and XCTest test frameworks +on both iOS simulator and real devices. +</p> + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShBinaryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShBinaryRule.java new file mode 100644 index 0000000..39d4a0c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShBinaryRule.java
@@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.rules.sh; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.bazel.rules.BazelBaseRuleClasses; +import com.google.devtools.build.lib.bazel.rules.sh.BazelShRuleClasses.ShRule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for the sh_binary rule. + */ +@BlazeRule(name = "sh_binary", + ancestors = { ShRule.class, BazelBaseRuleClasses.BinaryBaseRule.class }, + factoryClass = ShBinary.class) +public final class BazelShBinaryRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder.add( + attr("bash_version", STRING) + .value(BazelShRuleClasses.DEFAULT_BASH_VERSION) + .allowedValues(BazelShRuleClasses.BASH_VERSION_ALLOWED_VALUES)).build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShLibraryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShLibraryRule.java new file mode 100644 index 0000000..9d9640b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShLibraryRule.java
@@ -0,0 +1,113 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.rules.sh; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.bazel.rules.sh.BazelShRuleClasses.ShRule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for the sh_library rule. + */ +@BlazeRule(name = "sh_library", + ancestors = { ShRule.class }, + factoryClass = ShLibrary.class) +public final class BazelShLibraryRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE(sh_library).ATTRIBUTE(deps) --> + The list of other targets to be aggregated in to this "library" target. + <i>(List of <a href="build-ref.html#labels">labels</a>; optional)</i><br/> + See general comments about <code>deps</code> + at <a href="#common-attributes">Attributes common to all build rules</a>. + You should use this attribute to list other + <code>sh_library</code> or <code>proto_library</code> rules that provide + interpreted program source code depended on by the code in + <code>srcs</code>. If you depend on a <code>proto_library</code> target, + the proto sources in that target will be included in this library, but + no generated files will be built. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + + /* <!-- #BLAZE_RULE(sh_library).ATTRIBUTE(srcs) --> + The list of input files. + <i>(List of <a href="build-ref.html#labels">labels</a>, + optional)</i><br/> + You should use this attribute to list interpreted program + source files that belong to this package, such as additional + files containing Bourne shell subroutines, loaded via the shell's + <code>source</code> or <code>.</code> command. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .override(attr("srcs", LABEL_LIST).legacyAllowAnyFileType()) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = sh_library, TYPE = LIBRARY, FAMILY = Shell) --> + +${ATTRIBUTE_SIGNATURE} + +<p> + The main use for this rule is to aggregate together a logical + "library" consisting of related scripts—programs in an + interpreted language that does not require compilation or linking, + such as the Bourne shell—and any data those programs need at + run-time. Such "libraries" can then be used from + the <code>data</code> attribute of one or + more <code>sh_binary</code> rules. +</p> + +<p> + Historically, a second use was to aggregate a collection of data files + together, to ensure that they are available at runtime in + the <code>.runfiles</code> area of one or more <code>*_binary</code> + rules (not necessarily <code>sh_binary</code>). + However, the <a href="#filegroup"><code>filegroup()</code></a> rule + should be used now; it is intended to replace this use of + <code>sh_library</code>. +</p> + +<p> + In interpreted programming languages, there's not always a clear + distinction between "code" and "data": after all, the program is + just "data" from the interpreter's point of view. For this reason + (and historical accident) this rule has three attributes which are + all essentially equivalent: <code>srcs</code>, <code>deps</code> + and <code>data</code>. + The recommended usage of each attribute is mentioned below. The + current implementation does not distinguish the elements of these lists. + All three attributes accept rules, source files and derived files. +</p> + +${ATTRIBUTE_DEFINITION} + +<h4 id="sh_library_examples">Examples</h4> + +<pre class="code"> +sh_library( + name = "foo", + data = [ + ":foo_service_script", # a sh_binary with srcs + ":deploy_foo", # another sh_binary with srcs + ], +) +</pre> + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShRuleClasses.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShRuleClasses.java new file mode 100644 index 0000000..6f66465 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShRuleClasses.java
@@ -0,0 +1,101 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.sh; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.Attribute.AllowedValueSet; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.PredicateWithMessage; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; + +import java.util.Collection; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Rule definitions for rule classes implementing shell support. + */ +public final class BazelShRuleClasses { + + static final Collection<String> ALLOWED_RULES_IN_DEPS_WITH_WARNING = ImmutableSet.of( + "filegroup", "Fileset", "genrule", "sh_binary", "sh_test", "test_suite"); + + /** + * Common attributes for shell rules. + */ + @BlazeRule(name = "$sh_target", + type = RuleClassType.ABSTRACT, + ancestors = { BaseRuleClasses.RuleBase.class }) + public static final class ShRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + .add(attr("srcs", LABEL_LIST).mandatory().legacyAllowAnyFileType()) + .override(builder.copy("deps") + .allowedRuleClasses("sh_library", "proto_library") + .allowedRuleClassesWithWarning(ALLOWED_RULES_IN_DEPS_WITH_WARNING) + .allowedFileTypes()) + .build(); + } + } + + /** + * Defines the file name of an sh_binary's implicit .sar (script package) output. + */ + static final ImplicitOutputsFunction SAR_PACKAGE_FILENAME = + fromTemplates("%{name}.sar"); + + /** + * Convenience structure for the bash dependency combinations defined + * by BASH_BINARY_BINDINGS. + */ + static class BashBinaryBinding { + public final String execPath; + public BashBinaryBinding(@Nullable String execPath) { + this.execPath = execPath; + } + } + + /** + * Attribute value specifying the local system's bash version. + */ + static final String SYSTEM_BASH_VERSION = "system"; + + static final Map<String, BashBinaryBinding> BASH_BINARY_BINDINGS = + ImmutableMap.of( + // "system": don't package any bash with the target, but rather use whatever is + // available on the system the script is run on. + SYSTEM_BASH_VERSION, new BashBinaryBinding("/bin/bash") + ); + + static final String DEFAULT_BASH_VERSION = SYSTEM_BASH_VERSION; + + // TODO(bazel-team): refactor sh_binary and sh_base to have a common root + // with srcs and bash_version attributes + static final PredicateWithMessage<Object> BASH_VERSION_ALLOWED_VALUES = + new AllowedValueSet(BASH_BINARY_BINDINGS.keySet()); +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShTestRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShTestRule.java new file mode 100644 index 0000000..4a1e51f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/BazelShTestRule.java
@@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.rules.sh; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.bazel.rules.sh.BazelShRuleClasses.ShRule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; + +/** + * Rule definition for the sh_test rule. + */ +@BlazeRule(name = "sh_test", + type = RuleClassType.TEST, + ancestors = { ShRule.class, BaseRuleClasses.TestBaseRule.class }, + factoryClass = ShTest.class) +public final class BazelShTestRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + .add(attr("bash_version", STRING) + .value(BazelShRuleClasses.DEFAULT_BASH_VERSION) + .allowedValues(BazelShRuleClasses.BASH_VERSION_ALLOWED_VALUES)) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShBinary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShBinary.java new file mode 100644 index 0000000..4e6ba81 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShBinary.java
@@ -0,0 +1,82 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.rules.sh; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.actions.ExecutableSymlinkAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * Implementation for the sh_binary rule. + */ +public class ShBinary implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + ImmutableList<Artifact> srcs = ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list(); + if (srcs.size() != 1) { + ruleContext.attributeError("srcs", "you must specify exactly one file in 'srcs'"); + return null; + } + + Artifact symlink = ruleContext.createOutputArtifact(); + Artifact src = srcs.get(0); + Artifact executableScript = getExecutableScript(ruleContext, src); + // The interpretation of this deceptively simple yet incredibly generic rule is complicated + // by the distinction between targets and (not properly encapsulated) artifacts. It depends + // on the notion of other rule's "files-to-build" sets, which are undocumented, making it + // impossible to give a precise definition of what this rule does in all cases (e.g. what + // happens when srcs = ['x', 'y'] but 'x' is an empty filegroup?). This is a pervasive + // problem in Blaze. + ruleContext.registerAction( + new ExecutableSymlinkAction(ruleContext.getActionOwner(), executableScript, symlink)); + + NestedSet<Artifact> filesToBuild = NestedSetBuilder.<Artifact>stableOrder() + .add(src) + .add(executableScript) // May be the same as src, in which case set semantics apply. + .add(symlink) + .build(); + Runfiles runfiles = new Runfiles.Builder() + .addTransitiveArtifacts(filesToBuild) + .addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES) + .build(); + RunfilesSupport runfilesSupport = RunfilesSupport.withExecutable( + ruleContext, runfiles, symlink); + return new RuleConfiguredTargetBuilder(ruleContext) + .setFilesToBuild(filesToBuild) + .setRunfilesSupport(runfilesSupport, symlink) + .addProvider(RunfilesProvider.class, RunfilesProvider.simple(runfiles)) + .build(); + } + + /** + * Hook for sh_test to provide the executable. + * + * @param ruleContext + * @param src + */ + protected Artifact getExecutableScript(RuleContext ruleContext, Artifact src) { + return src; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShLibrary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShLibrary.java new file mode 100644 index 0000000..e2744ea --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShLibrary.java
@@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.rules.sh; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * Implementation for the sh_library rule. + */ +public class ShLibrary implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + NestedSet<Artifact> filesToBuild = NestedSetBuilder.<Artifact>stableOrder() + .addAll(ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list()) + .addAll(ruleContext.getPrerequisiteArtifacts("deps", Mode.TARGET).list()) + .addAll(ruleContext.getPrerequisiteArtifacts("data", Mode.DATA).list()) + .build(); + Runfiles runfiles = new Runfiles.Builder() + .addTransitiveArtifacts(filesToBuild) + .build(); + return new RuleConfiguredTargetBuilder(ruleContext) + .setFilesToBuild(filesToBuild) + .addProvider(RunfilesProvider.class, RunfilesProvider.simple(runfiles)) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShTest.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShTest.java new file mode 100644 index 0000000..cc965aa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/sh/ShTest.java
@@ -0,0 +1,53 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.rules.sh; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Implementation for sh_test rules. + */ +public class ShTest extends ShBinary implements RuleConfiguredTargetFactory { + + @Override + protected Artifact getExecutableScript(RuleContext ruleContext, Artifact src) { + if (ruleContext.attributes().get("bash_version", Type.STRING) + .equals(BazelShRuleClasses.SYSTEM_BASH_VERSION)) { + return src; + } + + // What *will* this script run with the wrapper? + PathFragment newOutput = src.getRootRelativePath().getParentDirectory().getRelative( + ruleContext.getLabel().getName() + "_runner.sh"); + Artifact testRunner = ruleContext.getAnalysisEnvironment().getDerivedArtifact( + newOutput, ruleContext.getConfiguration().getBinDirectory()); + + String bashPath = BazelShRuleClasses.BASH_BINARY_BINDINGS + .get(BazelShRuleClasses.SYSTEM_BASH_VERSION).execPath; + + // Generate the runner contents. + String runnerContents = + "#!/bin/bash\n" + + bashPath + " \"" + src.getRootRelativePath().getPathString() + "\" \"$@\"\n"; + + ruleContext.registerAction( + new FileWriteAction(ruleContext.getActionOwner(), testRunner, runnerContents, true)); + return testRunner; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java new file mode 100644 index 0000000..c7f3677 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java
@@ -0,0 +1,113 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.workspace; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; + +/** + * Rule definition for the http_archive rule. + */ +@BlazeRule(name = HttpArchiveRule.NAME, + type = RuleClassType.WORKSPACE, + ancestors = { WorkspaceBaseRule.class }, + factoryClass = WorkspaceConfiguredTargetFactory.class) +public class HttpArchiveRule implements RuleDefinition { + + public static final String NAME = "http_archive"; + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE(http_archive).ATTRIBUTE(url) --> + A URL to an archive file containing a Bazel repository + + <p>This must be an http URL that ends with .zip. There is no support for authentication or + redirection.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("url", STRING).mandatory()) + /* <!-- #BLAZE_RULE(http_archive).ATTRIBUTE(sha256) --> + The expected SHA-256 hash of the file downloaded + + <p>This must match the SHA-256 hash of the file downloaded.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("sha256", STRING).mandatory()) + .setWorkspaceOnly() + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = http_archive, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] --> + +${ATTRIBUTE_SIGNATURE} + +<p>Downloads a Bazel repository as a compressed archive file, decompresses it, and makes its + targets available for binding.</p> + +<p>Only Zip-formatted archives with the .zip extension are supported.</p> + +${ATTRIBUTE_DEFINITION} + +<h4 id="http_archive_examples">Examples</h4> + +<p>Suppose the current repository contains the source code for a chat program, rooted at the + directory <i>~/chat-app</i>. It needs to depend on an SSL library which is available from + <i>http://example.com/openssl.zip</i>. This .zip file contains the following directory + structure:</p> + +<pre class="code"> +WORKSPACE +src/ + BUILD + openssl.cc + openssl.h +</pre> + +<p><i>src/BUILD</i> contains the following target definition:</p> + +<pre class="code"> +cc_library( + name = "openssl-lib", + srcs = ["openssl.cc"], + hdrs = ["openssl.h"], +) +</pre> + +<p>Targets in the <i>~/chat-app</i> repository can depend on this target if the following lines are + added to <i>~/chat-app/WORKSPACE</i>:</p> + +<pre class="code"> +http_archive( + name = "my-ssl", + url = "http://example.com/openssl.zip", + sha256 = "03a58ac630e59778f328af4bcc4acb4f80208ed4", +) + +bind( + name = "openssl", + actual = "@my-ssl//src:openssl-lib", +) +</pre> + +<p>See <a href="#bind_examples">Bind</a> for how to use bound targets.</p> + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpJarRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpJarRule.java new file mode 100644 index 0000000..862c6d7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpJarRule.java
@@ -0,0 +1,91 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.workspace; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; + +/** + * Rule definition for the http_jar rule. + */ +@BlazeRule(name = HttpJarRule.NAME, + type = RuleClassType.WORKSPACE, + ancestors = { WorkspaceBaseRule.class }, + factoryClass = WorkspaceConfiguredTargetFactory.class) +public class HttpJarRule implements RuleDefinition { + + public static final String NAME = "http_jar"; + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE(http_jar).ATTRIBUTE(url) --> + A URL to an archive file containing a Bazel repository + + <p>This must be an http or https URL that ends with .jar. Redirects are not followed.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("url", STRING).mandatory()) + /* <!-- #BLAZE_RULE(http_jar).ATTRIBUTE(sha256) --> + The expected SHA-256 of the file downloaded + + <p>This must match the SHA-256 of the file downloaded.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("sha256", STRING).mandatory()) + .setWorkspaceOnly() + .build(); + } +} +/*<!-- #BLAZE_RULE (NAME = http_jar, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] --> + +${ATTRIBUTE_SIGNATURE} + +<p>Downloads a jar from a URL and makes it available to be used as a Java dependency.</p> + +<p>Downloaded files must have a .jar extension.</p> + +${ATTRIBUTE_DEFINITION} + +<h4 id="http_jar_examples">Examples</h4> + +<p>Suppose the current repository contains the source code for a chat program, rooted at the + directory <i>~/chat-app</i>. It needs to depend on an SSL library which is available from + <i>http://example.com/openssl-0.2.jar</i>.</p> + +<p>Targets in the <i>~/chat-app</i> repository can depend on this target if the following lines are + added to <i>~/chat-app/WORKSPACE</i>:</p> + +<pre class="code"> +http_jar( + name = "my-ssl", + url = "http://example.com/openssl-0.2.jar", + sha256 = "03a58ac630e59778f328af4bcc4acb4f80208ed4", +) + +bind( + name = "openssl", + actual = "@my-ssl//jar:openssl-0.2.jar", +) +</pre> + +<p>See <a href="#bind_examples">Bind</a> for how to use bound targets.</p> + +<!-- #END_BLAZE_RULE -->*/ \ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/LocalRepositoryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/LocalRepositoryRule.java new file mode 100644 index 0000000..b904bc5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/LocalRepositoryRule.java
@@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.workspace; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; + +/** + * Rule definition for the local_repository rule. + */ +@BlazeRule(name = LocalRepositoryRule.NAME, + type = RuleClassType.WORKSPACE, + ancestors = { WorkspaceBaseRule.class }, + factoryClass = WorkspaceConfiguredTargetFactory.class) +public class LocalRepositoryRule implements RuleDefinition { + + public static final String NAME = "local_repository"; + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE(local_repository).ATTRIBUTE(path) --> + The path to the local repository's directory. + + <p>This must be an absolute path to the directory containing the repository's + <i>WORKSPACE</i> file.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("path", STRING).mandatory()) + .setWorkspaceOnly() + .build(); + } +} +/*<!-- #BLAZE_RULE (NAME = local_repository, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] --> + +${ATTRIBUTE_SIGNATURE} + +<p>Allows targets from a local directory to be bound. This means that the current repository can + use targets defined in this other directory. See the <a href="#bind_examples">bind section</a> + for more details.</p> + +${ATTRIBUTE_DEFINITION} + +<h4 id="local_repository_examples">Examples</h4> + +<p>Suppose the current repository is a chat client, rooted at the directory <i>~/chat-app</i>. It + would like to use an SSL library which is defined in a different repository: <i>~/ssl</i>. The + SSL library has a target <code>//src:openssl-lib</code>.</p> + +<p>The user can add a dependency on this target by adding the following lines to + <i>~/chat-app/WORKSPACE</i>:</p> + +<pre class="code"> +local_repository( + name = "my-ssl", + path = "/home/user/ssl", +) + +bind( + name = "openssl", + actual = "@my-ssl//src:openssl-lib", +) +</pre> + +<p>See <a href="#bind_examples">Bind</a> for how to use bound targets.</p> + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/MavenJarRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/MavenJarRule.java new file mode 100644 index 0000000..f4496dd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/MavenJarRule.java
@@ -0,0 +1,127 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.workspace; + +import static com.google.devtools.build.lib.packages.Attribute.attr; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.packages.Type; + +/** + * Rule definition for the maven_jar rule. + */ +@BlazeRule(name = MavenJarRule.NAME, + type = RuleClassType.WORKSPACE, + ancestors = { WorkspaceBaseRule.class }, + factoryClass = WorkspaceConfiguredTargetFactory.class) +public class MavenJarRule implements RuleDefinition { + + public static final String NAME = "maven_jar"; + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE(maven_jar).ATTRIBUTE(artifact_id) --> + The artifactId of the Maven dependency. + + <p>Required.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("artifact_id", Type.STRING).mandatory()) + /* <!-- #BLAZE_RULE(maven_jar).ATTRIBUTE(group_id) --> + The groupId of the Maven dependency. + + <p>Required.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("group_id", Type.STRING).mandatory()) + /* <!-- #BLAZE_RULE(maven_jar).ATTRIBUTE(version) --> + The version of the Maven dependency. + + <p>Required.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("version", Type.STRING).mandatory()) + /* <!-- #BLAZE_RULE(maven_jar).ATTRIBUTE(repositories) --> + A list of repositories to use to attempt to fetch the jar. + + <p>Defaults to Maven Central ("repo1.maven.org"). If repositories are specified, they will + be checked in the order listed here (Maven Central will not be checked in this case, + unless it is on the list).</p> + + <p><b>To be implemented: add a maven_repositories rule that allows a list of repositories + to be labeled.</b></p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("repositories", Type.STRING_LIST)) + /* <!-- #BLAZE_RULE(maven_jar).ATTRIBUTE(exclusions) --> + Transitive dependencies of this dependency that should not be downloaded. + + <p>Defaults to None: Bazel will download all of the dependencies requested by the Maven + dependency. If exclusions are specified, they will not be downloaded.</p> + + <p>Exclusions are specified in the format "<group_id>:<artifact_id>", for example, + "com.google.guava:guava".</p> + + <p><b>Not yet implemented.</b></p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("exclusions", Type.STRING_LIST)) + .setWorkspaceOnly() + .build(); + } +} +/*<!-- #BLAZE_RULE (NAME = maven_jar, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] --> + +${ATTRIBUTE_SIGNATURE} + +<p>Downloads a jar from Maven and makes it available to be used as a Java dependency.</p> + +${ATTRIBUTE_DEFINITION} + +<h4 id="http_jar_examples">Examples</h4> + +Suppose that the current repostory contains a java_library target that needs to depend on Guava. +Using Maven, this dependency would be defined in the pom.xml file as: + +<pre> +<dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>18.0</version> +</dependency> +</pre> + +In Bazel, the following lines can be added to the WORKSPACE file: + +<pre> +maven_jar( + name = "guava", + group_id = "com.google.guava", + artifact_id = "guava", + version = "18.0", +) + +bind( + name = "guava-jar", + actual = "@guava//jar" +) +</pre> + +Then the java_library can depend on <code>//external:guava-jar</code>. + +<p>See <a href="#bind_examples">Bind</a> for how to use bound targets.</p> + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewLocalRepositoryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewLocalRepositoryRule.java new file mode 100644 index 0000000..0061d3d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewLocalRepositoryRule.java
@@ -0,0 +1,135 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.workspace; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; + +/** + * Rule definition for the new_repository rule. + */ +@BlazeRule(name = NewLocalRepositoryRule.NAME, + type = RuleClassType.WORKSPACE, + ancestors = { WorkspaceBaseRule.class }, + factoryClass = WorkspaceConfiguredTargetFactory.class) +public class NewLocalRepositoryRule implements RuleDefinition { + public static final String NAME = "new_local_repository"; + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE(new_local_repository).ATTRIBUTE(path) --> + A path on the local filesystem. + + <p>This must be an absolute path to an existing file or a directory.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("path", STRING).mandatory()) + /* <!-- #BLAZE_RULE(new_local_repository).ATTRIBUTE(build_file) --> + A file to use as a BUILD file for this directory. + + <p>This path must be relative to the build's workspace. The file does not need to be named + BUILD, but can be (something like BUILD.new-repo-name may work well for distinguishing it + from the repository's actual BUILD files.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("build_file", STRING).mandatory()) + .setWorkspaceOnly() + .build(); + } +} +/*<!-- #BLAZE_RULE (NAME = new_local_repository, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] --> + +${ATTRIBUTE_SIGNATURE} + +<p>Allows a local directory to be turned into a Bazel repository. This means that the current + repository can define and use targets from anywhere on the filesystem.</p> + +<p>This rule creates a Bazel repository by creating a WORKSPACE file and subdirectory containing +symlinks to the BUILD file and path given. The build file should create targets relative to the +path, which can then be bound and used by the current build. + +${ATTRIBUTE_DEFINITION} + +<h4 id="new_local_repository_examples">Examples</h4> + +<p>Suppose the current repository is a chat client, rooted at the directory <i>~/chat-app</i>. It + would like to use an SSL library which is defined in a different directory: <i>~/ssl</i>.</p> + +<p>The user can add a dependency by creating a BUILD file for the SSL library +(~/chat-app/BUILD.my-ssl) containing: + +<pre class="code"> +java_library( + name = "openssl", + srcs = glob(['ssl/*.java']) +) +</pre> + +<p>Then they can add the following lines to <i>~/chat-app/WORKSPACE</i>:</p> + +<pre class="code"> +new_local_repository( + name = "my-ssl", + path = "/home/user/ssl", + build_file = "BUILD.my-ssl", +) + +bind( + name = "openssl", + actual = "@my-ssl//my-ssl:openssl", +) +</pre> + +<p>This will create a @my-ssl repository containing a my-ssl package that contains a symlink to +/home/user/ssl named ssl (so the BUILD file must refer to paths within /home/user/ssl relative to +ssl).</p> + +<p>See <a href="#bind_examples">Bind</a> for how to use bound targets.</p> + +<p>You can also use <code>new_local_repository</code> to include single files, not just +directories. For example, suppose you had a jar file at /home/username/Downloads/piano.jar. You +could add just that file to your build by adding the following to your WORKSPACE file: + +<pre class="code"> +new_local_repository( + name = "piano", + path = "/home/username/Downloads/piano.jar", + build_file = "BUILD.piano", +) + +bind( + name = "music", + actual = "@piano//piano:play-music", +) +</pre> + +<p>And creating the following BUILD file:</p> + +<pre class="code"> +java_import( + name = "play-music", + jars = ["piano.jar"], +) +</pre> + +Then targets can depend on //external:music to use piano.jar. + +<!-- #END_BLAZE_RULE -->*/ \ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/WorkspaceBaseRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/WorkspaceBaseRule.java new file mode 100644 index 0000000..2719cb8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/WorkspaceBaseRule.java
@@ -0,0 +1,34 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.workspace; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; + +/** + * Base rule for rules in the WORKSPACE file. + */ +@BlazeRule(name = "$workspace_base_rule", + type = RuleClassType.ABSTRACT) +public class WorkspaceBaseRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/WorkspaceConfiguredTargetFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/WorkspaceConfiguredTargetFactory.java new file mode 100644 index 0000000..6162228 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/WorkspaceConfiguredTargetFactory.java
@@ -0,0 +1,37 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.rules.workspace; + +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * Implementation of workspace rules. Generally, these don't have any providers, since they + * "forward" to the SkyFunctions which actually create the repositories and then are accessed via + * "normal" rules. + */ +public class WorkspaceConfiguredTargetFactory implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + return new RuleConfiguredTargetBuilder(ruleContext) + .addProvider(RunfilesProvider.class, RunfilesProvider.EMPTY) + .build(); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java new file mode 100644 index 0000000..95fbde5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java
@@ -0,0 +1,532 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.analysis.BuildView; +import com.google.devtools.build.lib.analysis.TopLevelArtifactContext; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.runtime.BlazeCommandEventHandler; +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.Converters.RangeConverter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsClassProvider; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +/** + * A BuildRequest represents a single invocation of the build tool by a user. + * A request specifies a list of targets to be built for a single + * configuration, a pair of output/error streams, and additional options such + * as --keep_going, --jobs, etc. + */ +public class BuildRequest implements OptionsClassProvider { + private static final String DEFAULT_SYMLINK_PREFIX_MARKER = "...---:::@@@DEFAULT@@@:::--..."; + + /** + * A converter for symlink prefixes that defaults to {@code Constants.PRODUCT_NAME} and a + * minus sign if the option is not given. + * + * <p>Required because you cannot specify a non-constant value in annotation attributes. + */ + public static class SymlinkPrefixConverter implements Converter<String> { + @Override + public String convert(String input) throws OptionsParsingException { + return input.equals(DEFAULT_SYMLINK_PREFIX_MARKER) + ? Constants.PRODUCT_NAME + "-" + : input; + } + + @Override + public String getTypeDescription() { + return "a string"; + } + } + + /** + * Options interface--can be used to parse command-line arguments. + * + * See also ExecutionOptions; from the user's point of view, there's no + * qualitative difference between these two sets of options. + */ + public static class BuildRequestOptions extends OptionsBase { + + /* "Execution": options related to the execution of a build: */ + + @Option(name = "jobs", + abbrev = 'j', + defaultValue = "200", + category = "strategy", + help = "The number of concurrent jobs to run. " + + "0 means build sequentially. Values above " + MAX_JOBS + + " are not allowed.") + public int jobs; + + @Option(name = "progress_report_interval", + defaultValue = "0", + category = "verbosity", + converter = ProgressReportIntervalConverter.class, + help = "The number of seconds to wait between two reports on" + + " still running jobs. The default value 0 means to use" + + " the default 10:30:60 incremental algorithm.") + public int progressReportInterval; + + @Option(name = "show_builder_stats", + defaultValue = "false", + category = "verbosity", + help = "If set, parallel builder will report worker-related statistics.") + public boolean useBuilderStatistics; + + @Option(name = "explain", + defaultValue = "null", + category = "verbosity", + converter = OptionsUtils.PathFragmentConverter.class, + help = "Causes Blaze to explain each executed step of the build. " + + "The explanation is written to the specified log file.") + public PathFragment explanationPath; + + @Option(name = "verbose_explanations", + defaultValue = "false", + category = "verbosity", + help = "Increases the verbosity of the explanations issued if --explain is enabled. " + + "Has no effect if --explain is not enabled.") + public boolean verboseExplanations; + + @Deprecated + @Option(name = "dump_makefile", + defaultValue = "false", + category = "undocumented", + help = "this flag has no effect.") + public boolean dumpMakefile; + + @Deprecated + @Option(name = "dump_action_graph", + defaultValue = "false", + category = "undocumented", + help = "this flag has no effect.") + + public boolean dumpActionGraph; + + @Deprecated + @Option(name = "dump_action_graph_for_package", + allowMultiple = true, + defaultValue = "", + category = "undocumented", + help = "this flag has no effect.") + public List<String> dumpActionGraphForPackage = new ArrayList<>(); + + @Deprecated + @Option(name = "dump_action_graph_with_middlemen", + defaultValue = "true", + category = "undocumented", + help = "this flag has no effect.") + public boolean dumpActionGraphWithMiddlemen; + + @Deprecated + @Option(name = "dump_providers", + defaultValue = "false", + category = "undocumented", + help = "This is a no-op.") + public boolean dumpProviders; + + @Option(name = "incremental_builder", + deprecationWarning = "incremental_builder is now a no-op and will be removed in an" + + " upcoming Blaze release", + defaultValue = "true", + category = "strategy", + help = "Enables an incremental builder aimed at faster " + + "incremental builds. Currently it has the greatest effect on null" + + "builds.") + public boolean useIncrementalDependencyChecker; + + @Deprecated + @Option(name = "dump_targets", + defaultValue = "null", + category = "undocumented", + help = "this flag has no effect.") + public String dumpTargets; + + @Deprecated + @Option(name = "dump_host_deps", + defaultValue = "true", + category = "undocumented", + help = "Deprecated") + public boolean dumpHostDeps; + + @Deprecated + @Option(name = "dump_to_stdout", + defaultValue = "false", + category = "undocumented", + help = "Deprecated") + public boolean dumpToStdout; + + @Option(name = "analyze", + defaultValue = "true", + category = "undocumented", + help = "Execute the analysis phase; this is the usual behaviour. " + + "Specifying --noanalyze causes the build to stop before starting the " + + "analysis phase, returning zero iff the package loading completed " + + "successfully; this mode is useful for testing.") + public boolean performAnalysisPhase; + + @Option(name = "build", + defaultValue = "true", + category = "what", + help = "Execute the build; this is the usual behaviour. " + + "Specifying --nobuild causes the build to stop before executing the " + + "build actions, returning zero iff the package loading and analysis " + + "phases completed successfully; this mode is useful for testing " + + "those phases.") + public boolean performExecutionPhase; + + @Option(name = "compile_only", + defaultValue = "false", + category = "what", + help = "If specified, Blaze will only build files that are generated by lightweight " + + "compilation actions, skipping more expensive build steps (such as linking).") + public boolean compileOnly; + + @Option(name = "compilation_prerequisites_only", + defaultValue = "false", + category = "what", + help = "If specified, Blaze will only build files that are prerequisites to compilation " + + "of the given target (for example, generated source files and headers) without " + + "building the target itself. This flag is ignored if --compile_only is enabled.") + public boolean compilationPrerequisitesOnly; + + @Option(name = "output_groups", + converter = Converters.CommaSeparatedOptionListConverter.class, + allowMultiple = true, + defaultValue = "", + category = "undocumented", + help = "Specifies, which output groups of the top-level target to build.") + public List<String> outputGroups; + + @Option(name = "show_result", + defaultValue = "1", + category = "verbosity", + help = "Show the results of the build. For each " + + "target, state whether or not it was brought up-to-date, and if " + + "so, a list of output files that were built. The printed files " + + "are convenient strings for copy+pasting to the shell, to " + + "execute them.\n" + + "This option requires an integer argument, which " + + "is the threshold number of targets above which result " + + "information is not printed. " + + "Thus zero causes suppression of the message and MAX_INT " + + "causes printing of the result to occur always. The default is one.") + public int maxResultTargets; + + @Option(name = "announce", + defaultValue = "false", + category = "verbosity", + help = "Deprecated. No-op.", + deprecationWarning = "This option is now deprecated and is a no-op") + public boolean announce; + + @Option(name = "symlink_prefix", + defaultValue = DEFAULT_SYMLINK_PREFIX_MARKER, + converter = SymlinkPrefixConverter.class, + category = "misc", + help = "The prefix that is prepended to any of the convenience symlinks that are created " + + "after a build. If '/' is passed, then no symlinks are created and no warning is " + + "emitted." + ) + public String symlinkPrefix; + + @Option(name = "experimental_multi_cpu", + converter = Converters.CommaSeparatedOptionListConverter.class, + allowMultiple = true, + defaultValue = "", + category = "semantics", + help = "This flag allows specifying multiple target CPUs. If this is specified, " + + "the --cpu option is ignored.") + public List<String> multiCpus; + + @Option(name = "experimental_check_output_files", + defaultValue = "true", + category = "undocumented", + help = "Check for modifications made to the output files of a build. Consider setting " + + "this flag to false to see the effect on incremental build times.") + public boolean checkOutputFiles; + } + + /** + * Converter for progress_report_interval: [0, 3600]. + */ + public static class ProgressReportIntervalConverter extends RangeConverter { + public ProgressReportIntervalConverter() { + super(0, 3600); + } + } + + private static final int MAX_JOBS = 2000; + private static final int JOBS_TOO_HIGH_WARNING = 1000; + + private final UUID id; + private final LoadingCache<Class<? extends OptionsBase>, Optional<OptionsBase>> optionsCache; + + /** A human-readable description of all the non-default option settings. */ + private final String optionsDescription; + + /** + * The name of the Blaze command that the user invoked. + * Used for --announce. + */ + private final String commandName; + + private final OutErr outErr; + private final List<String> targets; + + private long startTimeMillis = 0; // milliseconds since UNIX epoch. + + private boolean runningInEmacs = false; + private boolean runTests = false; + + private static final List<Class<? extends OptionsBase>> MANDATORY_OPTIONS = ImmutableList.of( + BuildRequestOptions.class, + PackageCacheOptions.class, + LoadingPhaseRunner.Options.class, + BuildView.Options.class, + ExecutionOptions.class); + + private BuildRequest(String commandName, + final OptionsProvider options, + final OptionsProvider startupOptions, + List<String> targets, + OutErr outErr, + UUID id, + long startTimeMillis) { + this.commandName = commandName; + this.optionsDescription = OptionsUtils.asShellEscapedString(options); + this.outErr = outErr; + this.targets = targets; + this.id = id; + this.startTimeMillis = startTimeMillis; + this.optionsCache = CacheBuilder.newBuilder() + .build(new CacheLoader<Class<? extends OptionsBase>, Optional<OptionsBase>>() { + @Override + public Optional<OptionsBase> load(Class<? extends OptionsBase> key) throws Exception { + OptionsBase result = options.getOptions(key); + if (result == null && startupOptions != null) { + result = startupOptions.getOptions(key); + } + + return Optional.fromNullable(result); + } + }); + + for (Class<? extends OptionsBase> optionsClass : MANDATORY_OPTIONS) { + Preconditions.checkNotNull(getOptions(optionsClass)); + } + } + + /** + * Returns a unique identifier that universally identifies this build. + */ + public UUID getId() { + return id; + } + + /** + * Returns the name of the Blaze command that the user invoked. + */ + public String getCommandName() { + return commandName; + } + + /** + * Set to true if this build request was initiated by Emacs. + * (Certain output formatting may be necessary.) + */ + public void setRunningInEmacs() { + runningInEmacs = true; + } + + boolean isRunningInEmacs() { + return runningInEmacs; + } + + /** + * Enables test execution for this build request. + */ + public void setRunTests() { + runTests = true; + } + + /** + * Returns true if tests should be run by the build tool. + */ + public boolean shouldRunTests() { + return runTests; + } + + /** + * Returns the (immutable) list of targets to build in commandline + * form. + */ + public List<String> getTargets() { + return targets; + } + + /** + * Returns the output/error streams to which errors and progress messages + * should be sent during the fulfillment of this request. + */ + public OutErr getOutErr() { + return outErr; + } + + @Override + @SuppressWarnings("unchecked") + public <T extends OptionsBase> T getOptions(Class<T> clazz) { + try { + return (T) optionsCache.get(clazz).orNull(); + } catch (ExecutionException e) { + throw new IllegalStateException(e); + } + } + + /** + * Returns the set of command-line options specified for this request. + */ + public BuildRequestOptions getBuildOptions() { + return getOptions(BuildRequestOptions.class); + } + + /** + * Returns the set of options related to the loading phase. + */ + public PackageCacheOptions getPackageCacheOptions() { + return getOptions(PackageCacheOptions.class); + } + + /** + * Returns the set of options related to the loading phase. + */ + public LoadingPhaseRunner.Options getLoadingOptions() { + return getOptions(LoadingPhaseRunner.Options.class); + } + + /** + * Returns the set of command-line options related to the view specified for + * this request. + */ + public BuildView.Options getViewOptions() { + return getOptions(BuildView.Options.class); + } + + /** + * Returns the human-readable description of the non-default options + * for this build request. + */ + public String getOptionsDescription() { + return optionsDescription; + } + + /** + * Return the time (according to System.currentTimeMillis()) at which the + * service of this request was started. + */ + public long getStartTime() { + return startTimeMillis; + } + + /** + * Validates the options for this BuildRequest. + * + * <p>Issues warnings or throws {@code InvalidConfigurationException} for option settings that + * conflict. + * + * @return list of warnings + */ + public List<String> validateOptions() throws InvalidConfigurationException { + List<String> warnings = new ArrayList<>(); + // Validate "jobs". + int jobs = getBuildOptions().jobs; + if (jobs < 0 || jobs > MAX_JOBS) { + throw new InvalidConfigurationException(String.format( + "Invalid parameter for --jobs: %d. Only values 0 <= jobs <= %d are allowed.", jobs, + MAX_JOBS)); + } + if (jobs > JOBS_TOO_HIGH_WARNING) { + warnings.add( + String.format("High value for --jobs: %d. You may run into memory issues", jobs)); + } + + // Validate other BuildRequest options. + if (getBuildOptions().verboseExplanations && getBuildOptions().explanationPath == null) { + warnings.add("--verbose_explanations has no effect when --explain=<file> is not enabled"); + } + if (getBuildOptions().compileOnly && getBuildOptions().compilationPrerequisitesOnly) { + throw new InvalidConfigurationException( + "--compile_only and --compilation_prerequisites_only are not compatible"); + } + + return warnings; + } + + /** Creates a new TopLevelArtifactContext from this build request. */ + public TopLevelArtifactContext getTopLevelArtifactContext() { + return new TopLevelArtifactContext(getCommandName(), + getBuildOptions().compileOnly, getBuildOptions().compilationPrerequisitesOnly, + getOptions(ExecutionOptions.class).testStrategy.equals("exclusive"), + ImmutableSet.<String>copyOf(getBuildOptions().outputGroups), shouldRunTests()); + } + + public String getSymlinkPrefix() { + return getBuildOptions().symlinkPrefix; + } + + public ImmutableSortedSet<String> getMultiCpus() { + return ImmutableSortedSet.copyOf(getBuildOptions().multiCpus); + } + + public static BuildRequest create(String commandName, OptionsProvider options, + OptionsProvider startupOptions, + List<String> targets, OutErr outErr, UUID commandId, long commandStartTime) { + + BuildRequest request = new BuildRequest(commandName, options, startupOptions, targets, outErr, + commandId, commandStartTime); + + // All this, just to pass a global boolean from the client to the server. :( + if (options.getOptions(BlazeCommandEventHandler.Options.class).runningInEmacs) { + request.setRunningInEmacs(); + } + + return request; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildResult.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildResult.java new file mode 100644 index 0000000..22c36f8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildResult.java
@@ -0,0 +1,196 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.util.ExitCode; + +import java.util.Collection; +import java.util.Collections; + +import javax.annotation.Nullable; + +/** + * Contains information about the result of a build. While BuildRequest is immutable, this class is + * mutable. + */ +public final class BuildResult { + private long startTimeMillis = 0; // milliseconds since UNIX epoch. + private long stopTimeMillis = 0; + + private Throwable crash = null; + private boolean catastrophe = false; + private ExitCode exitCondition = ExitCode.BLAZE_INTERNAL_ERROR; + private Collection<ConfiguredTarget> actualTargets; + private Collection<ConfiguredTarget> testTargets; + private Collection<ConfiguredTarget> successfulTargets; + + public BuildResult(long startTimeMillis) { + this.startTimeMillis = startTimeMillis; + } + + /** + * Record the time (according to System.currentTimeMillis()) at which the + * service of this request was completed. + */ + public void setStopTime(long stopTimeMillis) { + this.stopTimeMillis = stopTimeMillis; + } + + /** + * Return the time (according to System.currentTimeMillis()) at which the + * service of this request was completed. + */ + public long getStopTime() { + return stopTimeMillis; + } + + /** + * Returns the elapsed time in seconds for the service of this request. Not + * defined for requests that have not been serviced. + */ + public double getElapsedSeconds() { + if (startTimeMillis == 0 || stopTimeMillis == 0) { + throw new IllegalStateException("BuildRequest has not been serviced"); + } + return (stopTimeMillis - startTimeMillis) / 1000.0; + } + + public void setExitCondition(ExitCode exitCondition) { + this.exitCondition = exitCondition; + } + + /** + * True iff the build request has been successfully completed. + */ + public boolean getSuccess() { + return exitCondition == ExitCode.SUCCESS; + } + + /** + * Gets the Blaze exit condition. + */ + public ExitCode getExitCondition() { + return exitCondition; + } + + /** + * Sets the RuntimeException / Error that induced a Blaze crash. + */ + public void setUnhandledThrowable(Throwable crash) { + Preconditions.checkState(crash == null || + ((crash instanceof RuntimeException) || (crash instanceof Error))); + this.crash = crash; + } + + /** + * Sets a "catastrophe": A build failure severe enough to halt a keep_going build. + */ + public void setCatastrophe() { + this.catastrophe = true; + } + + /** + * Was the build a "catastrophe": A build failure severe enough to halt a keep_going build. + */ + public boolean wasCatastrophe() { + return catastrophe; + } + + /** + * Gets the Blaze crash Throwable. Null if Blaze did not crash. + */ + public Throwable getUnhandledThrowable() { + return crash; + } + + /** + * @see #getActualTargets + */ + public void setActualTargets(Collection<ConfiguredTarget> actualTargets) { + this.actualTargets = actualTargets; + } + + /** + * Returns the actual set of targets which we attempted to build. This value + * is set during the build, after the target patterns have been parsed and + * resolved. If --keep_going is specified, this set may exclude targets that + * could not be found or successfully analyzed. It may be examined after the + * build. May be null even after the build, if there were errors in the + * loading or analysis phases. + */ + public Collection<ConfiguredTarget> getActualTargets() { + return actualTargets; + } + + /** + * @see #getTestTargets + */ + public void setTestTargets(@Nullable Collection<ConfiguredTarget> testTargets) { + this.testTargets = testTargets == null ? null : Collections.unmodifiableCollection(testTargets); + } + + /** + * Returns the actual unmodifiable collection of targets which we attempted to + * test. This value is set at the end of the build analysis phase, after the + * test target patterns have been parsed and resolved. If --keep_going is + * specified, this collection may exclude targets that could not be found or + * successfully analyzed. It may be examined after the build. May be null even + * after the build, if there were errors in the loading or analysis phases or + * if testing was not requested. + */ + public Collection<ConfiguredTarget> getTestTargets() { + return testTargets; + } + + /** + * @see #getSuccessfulTargets + */ + void setSuccessfulTargets(Collection<ConfiguredTarget> successfulTargets) { + this.successfulTargets = successfulTargets; + } + + /** + * Returns the set of targets which successfully built. This value + * is set at the end of the build, after the target patterns have been parsed + * and resolved and after attempting to build the targets. If --keep_going + * is specified, this set may exclude targets that could not be found or + * successfully analyzed, or could not be built. It may be examined after + * the build. May be null if the execution phase was not attempted, as + * may happen if there are errors in the loading phase, for example. + */ + public Collection<ConfiguredTarget> getSuccessfulTargets() { + return successfulTargets; + } + + /** For debugging. */ + @Override + @SuppressWarnings("deprecation") + public String toString() { + // We need to be compatible with Guava, so we use this, even though it is deprecated. + return Objects.toStringHelper(this) + .add("startTimeMillis", startTimeMillis) + .add("stopTimeMillis", stopTimeMillis) + .add("crash", crash) + .add("catastrophe", catastrophe) + .add("exitCondition", exitCondition) + .add("actualTargets", actualTargets) + .add("testTargets", testTargets) + .add("successfulTargets", successfulTargets) + .toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java new file mode 100644 index 0000000..a27cc50 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
@@ -0,0 +1,540 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.actions.BuildFailedException; +import com.google.devtools.build.lib.actions.ExecutorInitException; +import com.google.devtools.build.lib.actions.TestExecException; +import com.google.devtools.build.lib.analysis.AnalysisPhaseCompleteEvent; +import com.google.devtools.build.lib.analysis.BuildInfoEvent; +import com.google.devtools.build.lib.analysis.BuildView; +import com.google.devtools.build.lib.analysis.BuildView.AnalysisResult; +import com.google.devtools.build.lib.analysis.ConfigurationsCreatedEvent; +import com.google.devtools.build.lib.analysis.ConfiguredAttributeMapper; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.LicensesProvider; +import com.google.devtools.build.lib.analysis.LicensesProvider.TargetLicense; +import com.google.devtools.build.lib.analysis.MakeEnvironmentEvent; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.ViewCreationFailedException; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.DefaultsPackage; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions; +import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent; +import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent; +import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent; +import com.google.devtools.build.lib.buildtool.buildevent.TestFilteringCompleteEvent; +import com.google.devtools.build.lib.cmdline.TargetParsingException; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.License; +import com.google.devtools.build.lib.packages.License.DistributionType; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.pkgcache.LoadingFailedException; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner.Callback; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner.LoadingResult; +import com.google.devtools.build.lib.profiler.ProfilePhase; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * Provides the bulk of the implementation of the 'blaze build' command. + * + * <p>The various concrete build command classes handle the command options and request + * setup, then delegate the handling of the request (the building of targets) to this class. + * + * <p>The main entry point is {@link #buildTargets}. + * + * <p>This class is always instantiated and managed as a singleton, being constructed and held by + * {@link BlazeRuntime}. This is so multiple kinds of build commands can share this single + * instance. + * + * <p>Most of analysis is handled in {@link BuildView}, and execution in {@link ExecutionTool}. + */ +public class BuildTool { + + private static final Logger LOG = Logger.getLogger(BuildTool.class.getName()); + + protected final BlazeRuntime runtime; + + /** + * Constructs a BuildTool. + * + * @param runtime a reference to the blaze runtime. + */ + public BuildTool(BlazeRuntime runtime) { + this.runtime = runtime; + } + + /** + * The crux of the build system. Builds the targets specified in the request using the specified + * Executor. + * + * <p>Performs loading, analysis and execution for the specified set of targets, honoring the + * configuration options in the BuildRequest. Returns normally iff successful, throws an exception + * otherwise. + * + * <p>Callers must ensure that {@link #stopRequest} is called after this method, even if it + * throws. + * + * <p>The caller is responsible for setting up and syncing the package cache. + * + * <p>During this function's execution, the actualTargets and successfulTargets + * fields of the request object are set. + * + * @param request the build request that this build tool is servicing, which specifies various + * options; during this method's execution, the actualTargets and successfulTargets fields + * of the request object are populated + * @param result the build result that is the mutable result of this build + * @param validator target validator + */ + public void buildTargets(BuildRequest request, BuildResult result, TargetValidator validator) + throws BuildFailedException, LocalEnvironmentException, + InterruptedException, ViewCreationFailedException, + TargetParsingException, LoadingFailedException, ExecutorInitException, + AbruptExitException, InvalidConfigurationException, TestExecException { + validateOptions(request); + BuildOptions buildOptions = runtime.createBuildOptions(request); + // Sync the package manager before sending the BuildStartingEvent in runLoadingPhase() + runtime.setupPackageCache(request.getPackageCacheOptions(), + DefaultsPackage.getDefaultsPackageContent(buildOptions)); + + ExecutionTool executionTool = null; + LoadingResult loadingResult = null; + BuildConfigurationCollection configurations = null; + try { + getEventBus().post(new BuildStartingEvent(runtime.getOutputFileSystem(), request)); + LOG.info("Build identifier: " + request.getId()); + executionTool = new ExecutionTool(runtime, request); + if (needsExecutionPhase(request.getBuildOptions())) { + // Initialize the execution tool early if we need it. This hides the latency of setting up + // the execution backends. + executionTool.init(); + } + + // Loading phase. + loadingResult = runLoadingPhase(request, validator); + + // Create the build configurations. + if (!request.getMultiCpus().isEmpty()) { + getReporter().handle(Event.warn( + "The --experimental_multi_cpu option is _very_ experimental and only intended for " + + "internal testing at this time. If you do not work on the build tool, then you " + + "should stop now!")); + if (!"build".equals(request.getCommandName()) && !"test".equals(request.getCommandName())) { + throw new InvalidConfigurationException( + "The experimental setting to select multiple CPUs is only supported for 'build' and " + + "'test' right now!"); + } + } + configurations = getConfigurations( + runtime.getBuildConfigurationKey(buildOptions, request.getMultiCpus()), + request.getViewOptions().keepGoing); + + getEventBus().post(new ConfigurationsCreatedEvent(configurations)); + runtime.throwPendingException(); + if (configurations.getTargetConfigurations().size() == 1) { + // TODO(bazel-team): This is not optimal - we retain backwards compatibility in the case + // where there's only a single configuration, but we don't send an event in the multi-config + // case. Can we do better? [multi-config] + getEventBus().post(new MakeEnvironmentEvent( + configurations.getTargetConfigurations().get(0).getMakeEnvironment())); + } + LOG.info("Configurations created"); + + // Analysis phase. + AnalysisResult analysisResult = runAnalysisPhase(request, loadingResult, configurations); + result.setActualTargets(analysisResult.getTargetsToBuild()); + result.setTestTargets(analysisResult.getTargetsToTest()); + + reportTargets(analysisResult); + + // Execution phase. + if (needsExecutionPhase(request.getBuildOptions())) { + executionTool.executeBuild(analysisResult, result, runtime.getSkyframeExecutor(), + configurations, mergePackageRoots(loadingResult.getPackageRoots(), + runtime.getSkyframeExecutor().getPackageRoots())); + } + + String delayedErrorMsg = analysisResult.getError(); + if (delayedErrorMsg != null) { + throw new BuildFailedException(delayedErrorMsg); + } + } catch (RuntimeException e) { + // Print an error message for unchecked runtime exceptions. This does not concern Error + // subclasses such as OutOfMemoryError. + request.getOutErr().printErrLn("Unhandled exception thrown during build; message: " + + e.getMessage()); + throw e; + } finally { + // Delete dirty nodes to ensure that they do not accumulate indefinitely. + long versionWindow = request.getViewOptions().versionWindowForDirtyNodeGc; + if (versionWindow != -1) { + runtime.getSkyframeExecutor().deleteOldNodes(versionWindow); + } + + if (executionTool != null) { + executionTool.shutdown(); + } + // The workspace status actions will not run with certain flags, or if an error + // occurs early in the build. Tell a lie so that the event is not missing. + // If multiple build_info events are sent, only the first is kept, so this does not harm + // successful runs (which use the workspace status action). + getEventBus().post(new BuildInfoEvent( + runtime.getworkspaceStatusActionFactory().createDummyWorkspaceStatus())); + } + + if (loadingResult != null && loadingResult.hasTargetPatternError()) { + throw new BuildFailedException("execution phase successful, but there were errors " + + "parsing the target pattern"); + } + } + + private ImmutableMap<PathFragment, Path> mergePackageRoots( + ImmutableMap<PackageIdentifier, Path> first, + ImmutableMap<PackageIdentifier, Path> second) { + Map<PathFragment, Path> builder = Maps.newHashMap(); + for (Map.Entry<PackageIdentifier, Path> entry : first.entrySet()) { + builder.put(entry.getKey().getPackageFragment(), entry.getValue()); + } + for (Map.Entry<PackageIdentifier, Path> entry : second.entrySet()) { + if (first.containsKey(entry.getKey())) { + Preconditions.checkState(first.get(entry.getKey()).equals(entry.getValue())); + } else { + // This could overwrite entries from first in other repositories. + builder.put(entry.getKey().getPackageFragment(), entry.getValue()); + } + } + return ImmutableMap.copyOf(builder); + } + + private void reportExceptionError(Exception e) { + if (e.getMessage() != null) { + getReporter().handle(Event.error(e.getMessage())); + } + } + /** + * The crux of the build system. Builds the targets specified in the request using the specified + * Executor. + * + * <p>Performs loading, analysis and execution for the specified set of targets, honoring the + * configuration options in the BuildRequest. Returns normally iff successful, throws an exception + * otherwise. + * + * <p>The caller is responsible for setting up and syncing the package cache. + * + * <p>During this function's execution, the actualTargets and successfulTargets + * fields of the request object are set. + * + * @param request the build request that this build tool is servicing, which specifies various + * options; during this method's execution, the actualTargets and successfulTargets fields + * of the request object are populated + * @param validator target validator + * @return the result as a {@link BuildResult} object + */ + public BuildResult processRequest(BuildRequest request, TargetValidator validator) { + BuildResult result = new BuildResult(request.getStartTime()); + runtime.getEventBus().register(result); + Throwable catastrophe = null; + ExitCode exitCode = ExitCode.BLAZE_INTERNAL_ERROR; + try { + buildTargets(request, result, validator); + exitCode = ExitCode.SUCCESS; + } catch (BuildFailedException e) { + if (e.isErrorAlreadyShown()) { + // The actual error has already been reported by the Builder. + } else { + reportExceptionError(e); + } + if (e.isCatastrophic()) { + result.setCatastrophe(); + } + exitCode = ExitCode.BUILD_FAILURE; + } catch (InterruptedException e) { + exitCode = ExitCode.INTERRUPTED; + getReporter().handle(Event.error("build interrupted")); + getEventBus().post(new BuildInterruptedEvent()); + } catch (TargetParsingException | LoadingFailedException | ViewCreationFailedException e) { + exitCode = ExitCode.PARSING_FAILURE; + reportExceptionError(e); + } catch (TestExecException e) { + // ExitCode.SUCCESS means that build was successful. Real return code of program + // is going to be calculated in TestCommand.doTest(). + exitCode = ExitCode.SUCCESS; + reportExceptionError(e); + } catch (InvalidConfigurationException e) { + exitCode = ExitCode.COMMAND_LINE_ERROR; + reportExceptionError(e); + } catch (AbruptExitException e) { + exitCode = e.getExitCode(); + reportExceptionError(e); + result.setCatastrophe(); + } catch (Throwable throwable) { + catastrophe = throwable; + Throwables.propagate(throwable); + } finally { + stopRequest(request, result, catastrophe, exitCode); + } + + return result; + } + + protected final BuildConfigurationCollection getConfigurations(BuildConfigurationKey key, + boolean keepGoing) + throws InvalidConfigurationException, InterruptedException { + SkyframeExecutor executor = runtime.getSkyframeExecutor(); + // TODO(bazel-team): consider a possibility of moving ConfigurationFactory construction into + // skyframe. + return executor.createConfigurations(keepGoing, runtime.getConfigurationFactory(), key); + } + + @VisibleForTesting + protected final LoadingResult runLoadingPhase(final BuildRequest request, + final TargetValidator validator) + throws LoadingFailedException, TargetParsingException, InterruptedException, + AbruptExitException { + Profiler.instance().markPhase(ProfilePhase.LOAD); + runtime.throwPendingException(); + + final boolean keepGoing = request.getViewOptions().keepGoing; + + Callback callback = new Callback() { + @Override + public void notifyTargets(Collection<Target> targets) throws LoadingFailedException { + if (validator != null) { + validator.validateTargets(targets, keepGoing); + } + } + + @Override + public void notifyVisitedPackages(Set<PackageIdentifier> visitedPackages) { + runtime.getSkyframeExecutor().updateLoadedPackageSet(visitedPackages); + } + }; + + LoadingResult result = runtime.getLoadingPhaseRunner().execute(getReporter(), + getEventBus(), request.getTargets(), request.getLoadingOptions(), + runtime.createBuildOptions(request).getAllLabels(), keepGoing, + request.shouldRunTests(), callback); + runtime.throwPendingException(); + return result; + } + + /** + * Performs the initial phases 0-2 of the build: Setup, Loading and Analysis. + * <p> + * Postcondition: On success, populates the BuildRequest's set of targets to + * build. + * + * @return null if loading / analysis phases were successful; a useful error + * message if loading or analysis phase errors were encountered and + * request.keepGoing. + * @throws InterruptedException if the current thread was interrupted. + * @throws ViewCreationFailedException if analysis failed for any reason. + */ + private AnalysisResult runAnalysisPhase(BuildRequest request, LoadingResult loadingResult, + BuildConfigurationCollection configurations) + throws InterruptedException, ViewCreationFailedException { + Stopwatch timer = Stopwatch.createStarted(); + if (!request.getBuildOptions().performAnalysisPhase) { + getReporter().handle(Event.progress("Loading complete.")); + LOG.info("No analysis requested, so finished"); + return AnalysisResult.EMPTY; + } + + getReporter().handle(Event.progress("Loading complete. Analyzing...")); + Profiler.instance().markPhase(ProfilePhase.ANALYZE); + + AnalysisResult analysisResult = getView().update(loadingResult, configurations, + request.getViewOptions(), request.getTopLevelArtifactContext(), getReporter(), + getEventBus()); + + // TODO(bazel-team): Merge these into one event. + getEventBus().post(new AnalysisPhaseCompleteEvent(analysisResult.getTargetsToBuild(), + getView().getTargetsVisited(), timer.stop().elapsed(TimeUnit.MILLISECONDS))); + getEventBus().post(new TestFilteringCompleteEvent(analysisResult.getTargetsToBuild(), + analysisResult.getTargetsToTest())); + + // Check licenses. + // We check licenses if the first target configuration has license checking enabled. Right now, + // it is not possible to have multiple target configurations with different settings for this + // flag, which allows us to take this short cut. + boolean checkLicenses = configurations.getTargetConfigurations().get(0).checkLicenses(); + if (checkLicenses) { + Profiler.instance().markPhase(ProfilePhase.LICENSE); + validateLicensingForTargets(analysisResult.getTargetsToBuild(), + request.getViewOptions().keepGoing); + } + + return analysisResult; + } + + private static boolean needsExecutionPhase(BuildRequestOptions options) { + return options.performAnalysisPhase && options.performExecutionPhase; + } + + /** + * Stops processing the specified request. + * + * <p>This logs the build result, cleans up and stops the clock. + * + * @param request the build request that this build tool is servicing + * @param crash Any unexpected RuntimeException or Error. May be null + * @param exitCondition A suggested exit condition from either the build logic or + * a thrown exception somewhere along the way. + */ + public void stopRequest(BuildRequest request, BuildResult result, Throwable crash, + ExitCode exitCondition) { + Preconditions.checkState((crash == null) || (exitCondition != ExitCode.SUCCESS)); + result.setUnhandledThrowable(crash); + result.setExitCondition(exitCondition); + // The stop time has to be captured before we send the BuildCompleteEvent. + result.setStopTime(runtime.getClock().currentTimeMillis()); + getEventBus().post(new BuildCompleteEvent(request, result)); + } + + private void reportTargets(AnalysisResult analysisResult) { + Collection<ConfiguredTarget> targetsToBuild = analysisResult.getTargetsToBuild(); + Collection<ConfiguredTarget> targetsToTest = analysisResult.getTargetsToTest(); + if (targetsToTest != null) { + int testCount = targetsToTest.size(); + int targetCount = targetsToBuild.size() - testCount; + if (targetCount == 0) { + getReporter().handle(Event.info("Found " + + testCount + (testCount == 1 ? " test target..." : " test targets..."))); + } else { + getReporter().handle(Event.info("Found " + + targetCount + (targetCount == 1 ? " target and " : " targets and ") + + testCount + (testCount == 1 ? " test target..." : " test targets..."))); + } + } else { + int targetCount = targetsToBuild.size(); + getReporter().handle(Event.info("Found " + + targetCount + (targetCount == 1 ? " target..." : " targets..."))); + } + } + + /** + * Validates the options for this BuildRequest. + * + * <p>Issues warnings for the use of deprecated options, and warnings or errors for any option + * settings that conflict. + */ + @VisibleForTesting + public void validateOptions(BuildRequest request) throws InvalidConfigurationException { + for (String issue : request.validateOptions()) { + getReporter().handle(Event.warn(issue)); + } + } + + /** + * Takes a set of configured targets, and checks if the distribution methods + * declared for the targets are compatible with the constraints imposed by + * their prerequisites' licenses. + * + * @param configuredTargets the targets to check + * @param keepGoing if false, and a licensing error is encountered, both + * generates an error message on the reporter, <em>and</em> throws an + * exception. If true, then just generates a message on the reporter. + * @throws ViewCreationFailedException if the license checking failed (and not + * --keep_going) + */ + private void validateLicensingForTargets(Iterable<ConfiguredTarget> configuredTargets, + boolean keepGoing) throws ViewCreationFailedException { + for (ConfiguredTarget configuredTarget : configuredTargets) { + final Target target = configuredTarget.getTarget(); + + if (TargetUtils.isTestRule(target)) { + continue; // Tests are exempt from license checking + } + + final Set<DistributionType> distribs = target.getDistributions(); + BuildConfiguration config = configuredTarget.getConfiguration(); + boolean staticallyLinked = (config != null) && config.performsStaticLink(); + staticallyLinked |= (config != null) && (target instanceof Rule) + && ((Rule) target).getRuleClassObject().hasAttr("linkopts", Type.STRING_LIST) + && ConfiguredAttributeMapper.of((RuleConfiguredTarget) configuredTarget) + .get("linkopts", Type.STRING_LIST).contains("-static"); + + LicensesProvider provider = configuredTarget.getProvider(LicensesProvider.class); + if (provider != null) { + NestedSet<TargetLicense> licenses = provider.getTransitiveLicenses(); + for (TargetLicense targetLicense : licenses) { + if (!targetLicense.getLicense().checkCompatibility( + distribs, target, targetLicense.getLabel(), getReporter(), staticallyLinked)) { + if (!keepGoing) { + throw new ViewCreationFailedException("Build aborted due to licensing error"); + } + } + } + } else if (configuredTarget.getTarget() instanceof InputFile) { + // Input file targets do not provide licenses because they do not + // depend on the rule where their license is taken from. This is usually + // not a problem, because the transitive collection of licenses always + // hits the rule they come from, except when the input file is a + // top-level target. Thus, we need to handle that case specially here. + // + // See FileTarget#getLicense for more information about the handling of + // license issues with File targets. + License license = configuredTarget.getTarget().getLicense(); + if (!license.checkCompatibility(distribs, target, configuredTarget.getLabel(), + getReporter(), staticallyLinked)) { + if (!keepGoing) { + throw new ViewCreationFailedException("Build aborted due to licensing error"); + } + } + } + } + } + + public BuildView getView() { + return runtime.getView(); + } + + private Reporter getReporter() { + return runtime.getReporter(); + } + + private EventBus getEventBus() { + return runtime.getEventBus(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/CachesSavedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/CachesSavedEvent.java new file mode 100644 index 0000000..5b3229c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/CachesSavedEvent.java
@@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +/** + * Event that is raised when the action and artifact metadata caches are saved at the end of the + * build. Contains statistics. + */ +public class CachesSavedEvent { + /** Cache serialization statistics. */ + private final long actionCacheSaveTimeInMillis; + private final long actionCacheSizeInBytes; + + public CachesSavedEvent( + long actionCacheSaveTimeInMillis, + long actionCacheSizeInBytes) { + this.actionCacheSaveTimeInMillis = actionCacheSaveTimeInMillis; + this.actionCacheSizeInBytes = actionCacheSizeInBytes; + } + + public long getActionCacheSaveTimeInMillis() { + return actionCacheSaveTimeInMillis; + } + + public long getActionCacheSizeInBytes() { + return actionCacheSizeInBytes; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java new file mode 100644 index 0000000..74143cc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.common.collect.ImmutableMap; + +import java.util.HashMap; +import java.util.Map; + +/** + * Event signaling the end of the execution phase. Contains statistics about the action cache, + * the metadata cache and about last file save times. + */ +public class ExecutionFinishedEvent { + /** The mtime of the most recently saved source file when the build starts. */ + private long lastFileSaveTimeInMillis; + + /** + * The (filename, mtime) pairs of all files saved between the last build's + * start time and the current build's start time. Only applies to builds + * running with existing Blaze servers. Currently disabled. + */ + private Map<String, Long> changedFileSaveTimes = new HashMap<>(); + + public ExecutionFinishedEvent(Map<String, Long> changedFileSaveTimes, + long lastFileSaveTimeInMillis) { + this.changedFileSaveTimes = ImmutableMap.copyOf(changedFileSaveTimes); + this.lastFileSaveTimeInMillis = lastFileSaveTimeInMillis; + } + + public long getLastFileSaveTimeInMillis() { + return lastFileSaveTimeInMillis; + } + + public Map<String, Long> getChangedFileSaveTimes() { + return changedFileSaveTimes; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java new file mode 100644 index 0000000..771cfe6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
@@ -0,0 +1,875 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Stopwatch; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; +import com.google.common.collect.Table; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionCacheChecker; +import com.google.devtools.build.lib.actions.ActionContextConsumer; +import com.google.devtools.build.lib.actions.ActionContextMarker; +import com.google.devtools.build.lib.actions.ActionContextProvider; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BlazeExecutor; +import com.google.devtools.build.lib.actions.BuildFailedException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ExecutorInitException; +import com.google.devtools.build.lib.actions.LocalHostCapacity; +import com.google.devtools.build.lib.actions.ResourceManager; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.actions.TestExecException; +import com.google.devtools.build.lib.actions.cache.ActionCache; +import com.google.devtools.build.lib.analysis.BuildView; +import com.google.devtools.build.lib.analysis.BuildView.AnalysisResult; +import com.google.devtools.build.lib.analysis.CompilationPrerequisitesProvider; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToCompileProvider; +import com.google.devtools.build.lib.analysis.InputFileConfiguredTarget; +import com.google.devtools.build.lib.analysis.OutputFileConfiguredTarget; +import com.google.devtools.build.lib.analysis.TempsProvider; +import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.ViewCreationFailedException; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; +import com.google.devtools.build.lib.buildtool.buildevent.ExecutionPhaseCompleteEvent; +import com.google.devtools.build.lib.buildtool.buildevent.ExecutionStartingEvent; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.exec.CheckUpToDateFilter; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.exec.OutputService; +import com.google.devtools.build.lib.exec.SingleBuildFileCache; +import com.google.devtools.build.lib.exec.SourceManifestActionContextImpl; +import com.google.devtools.build.lib.exec.SymlinkTreeStrategy; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.profiler.ProfilePhase; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.rules.fileset.FilesetActionContext; +import com.google.devtools.build.lib.rules.fileset.FilesetActionContextImpl; +import com.google.devtools.build.lib.rules.test.TestActionContext; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.skyframe.Builder; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +/** + * This class manages the execution phase. The entry point is {@link #executeBuild}. + * + * <p>This is only intended for use by {@link BuildTool}. + * + * <p>This class contains an ActionCache, and refers to the BlazeRuntime's BuildView and + * PackageCache. + * + * @see BuildTool + * @see BuildView + */ +public class ExecutionTool { + private static class StrategyConverter { + private Table<Class<? extends ActionContext>, String, ActionContext> classMap = + HashBasedTable.create(); + private Map<Class<? extends ActionContext>, ActionContext> defaultClassMap = + new HashMap<>(); + + /** + * Aggregates all {@link ActionContext}s that are in {@code contextProviders}. + */ + @SuppressWarnings("unchecked") + private StrategyConverter(Iterable<ActionContextProvider> contextProviders) { + for (ActionContextProvider provider : contextProviders) { + for (ActionContext strategy : provider.getActionContexts()) { + ExecutionStrategy annotation = + strategy.getClass().getAnnotation(ExecutionStrategy.class); + if (annotation != null) { + defaultClassMap.put(annotation.contextType(), strategy); + + for (String name : annotation.name()) { + classMap.put(annotation.contextType(), name, strategy); + } + } + } + } + } + + @SuppressWarnings("unchecked") + private <T extends ActionContext> T getStrategy(Class<T> clazz, String name) { + return (T) (name.isEmpty() ? defaultClassMap.get(clazz) : classMap.get(clazz, name)); + } + + private String getValidValues(Class<? extends ActionContext> context) { + return Joiner.on(", ").join(Ordering.natural().sortedCopy(classMap.row(context).keySet())); + } + + private String getUserFriendlyName(Class<? extends ActionContext> context) { + ActionContextMarker marker = context.getAnnotation(ActionContextMarker.class); + return marker != null + ? marker.name() + : context.getSimpleName(); + } + } + + static final Logger LOG = Logger.getLogger(ExecutionTool.class.getName()); + + private final BlazeRuntime runtime; + private final BuildRequest request; + private BlazeExecutor executor; + private ActionInputFileCache fileCache; + private List<ActionContextProvider> actionContextProviders; + + private Map<String, ActionContext> spawnStrategyMap = new HashMap<>(); + private List<ActionContext> strategies = new ArrayList<>(); + + ExecutionTool(BlazeRuntime runtime, BuildRequest request) throws ExecutorInitException { + this.runtime = runtime; + this.request = request; + + List<ActionContextConsumer> actionContextConsumers = new ArrayList<>(); + actionContextProviders = new ArrayList<>(); + for (BlazeModule module : runtime.getBlazeModules()) { + ActionContextProvider provider = module.getActionContextProvider(); + if (provider != null) { + actionContextProviders.add(provider); + } + + ActionContextConsumer consumer = module.getActionContextConsumer(); + if (consumer != null) { + actionContextConsumers.add(consumer); + } + } + + actionContextProviders.add(new FilesetActionContextImpl.Provider( + runtime.getReporter(), runtime.getWorkspaceName())); + + strategies.add(new SourceManifestActionContextImpl(runtime.getRunfilesPrefix())); + strategies.add(new SymlinkTreeStrategy(runtime.getOutputService(), runtime.getBinTools())); + + StrategyConverter strategyConverter = new StrategyConverter(actionContextProviders); + strategies.add(strategyConverter.getStrategy(FilesetActionContext.class, "")); + strategies.add(strategyConverter.getStrategy(WorkspaceStatusAction.Context.class, "")); + + for (ActionContextConsumer consumer : actionContextConsumers) { + // There are many different SpawnActions, and we want to control the action context they use + // independently from each other, for example, to run genrules locally and Java compile action + // in prod. Thus, for SpawnActions, we decide the action context to use not only based on the + // context class, but also the mnemonic of the action. + for (Map.Entry<String, String> entry : consumer.getSpawnActionContexts().entrySet()) { + SpawnActionContext context = + strategyConverter.getStrategy(SpawnActionContext.class, entry.getValue()); + if (context == null) { + throw makeExceptionForInvalidStrategyValue(entry.getValue(), "spawn", + strategyConverter.getValidValues(SpawnActionContext.class)); + } + + spawnStrategyMap.put(entry.getKey(), context); + } + + for (Map.Entry<Class<? extends ActionContext>, String> entry : + consumer.getActionContexts().entrySet()) { + ActionContext context = strategyConverter.getStrategy(entry.getKey(), entry.getValue()); + if (context != null) { + strategies.add(context); + } else if (!entry.getValue().isEmpty()) { + // If the action context consumer requested the default value (by passing in the empty + // string), we do not throw the exception, because we assume that whoever put together + // the modules in this Blaze binary knew what they were doing. + throw makeExceptionForInvalidStrategyValue(entry.getValue(), + strategyConverter.getUserFriendlyName(entry.getKey()), + strategyConverter.getValidValues(entry.getKey())); + } + } + } + + // If tests are to be run during build, too, we have to explicitly load the test action context. + if (request.shouldRunTests()) { + String testStrategyValue = request.getOptions(ExecutionOptions.class).testStrategy; + ActionContext context = strategyConverter.getStrategy(TestActionContext.class, + testStrategyValue); + if (context == null) { + throw makeExceptionForInvalidStrategyValue(testStrategyValue, "test", + strategyConverter.getValidValues(TestActionContext.class)); + } + strategies.add(context); + } + } + + private static ExecutorInitException makeExceptionForInvalidStrategyValue(String value, + String strategy, String validValues) { + return new ExecutorInitException(String.format( + "'%s' is an invalid value for %s strategy. Valid values are: %s", value, strategy, + validValues), ExitCode.COMMAND_LINE_ERROR); + } + + Executor getExecutor() throws ExecutorInitException { + if (executor == null) { + executor = createExecutor(); + } + return executor; + } + + /** + * Creates an executor for the current set of blaze runtime, execution options, and request. + */ + private BlazeExecutor createExecutor() + throws ExecutorInitException { + return new BlazeExecutor( + runtime.getDirectories().getExecRoot(), + runtime.getDirectories().getOutputPath(), + getReporter(), + getEventBus(), + runtime.getClock(), + request, + request.getOptions(ExecutionOptions.class).verboseFailures, + request.getOptions(ExecutionOptions.class).showSubcommands, + strategies, + spawnStrategyMap, + actionContextProviders); + } + + void init() throws ExecutorInitException { + createToolsSymlinks(); + getExecutor(); + } + + void shutdown() { + for (ActionContextProvider actionContextProvider : actionContextProviders) { + actionContextProvider.executionPhaseEnding(); + } + } + + /** + * Performs the execution phase (phase 3) of the build, in which the Builder + * is applied to the action graph to bring the targets up to date. (This + * function will return prior to execution-proper if --nobuild was specified.) + * + * @param analysisResult the analysis phase output + * @param buildResult the mutable build result + * @param skyframeExecutor the skyframe executor (if any) + * @param packageRoots package roots collected from loading phase and BuildConfigutaionCollection + * creation + */ + void executeBuild(AnalysisResult analysisResult, + BuildResult buildResult, @Nullable SkyframeExecutor skyframeExecutor, + BuildConfigurationCollection configurations, + ImmutableMap<PathFragment, Path> packageRoots) + throws BuildFailedException, InterruptedException, AbruptExitException, TestExecException, + ViewCreationFailedException { + Stopwatch timer = Stopwatch.createStarted(); + prepare(packageRoots, configurations); + + ActionGraph actionGraph = analysisResult.getActionGraph(); + + // Get top-level artifacts. + ImmutableSet<Artifact> additionalArtifacts = analysisResult.getAdditionalArtifactsToBuild(); + + // If --nobuild is specified, this request completes successfully without + // execution. + if (!request.getBuildOptions().performExecutionPhase) { + return; + } + + // Create symlinks only after we've verified that we're actually + // supposed to build something. + if (getWorkspace().getFileSystem().supportsSymbolicLinks()) { + List<BuildConfiguration> targetConfigurations = + getView().getConfigurationCollection().getTargetConfigurations(); + // TODO(bazel-team): This is not optimal - we retain backwards compatibility in the case where + // there's only a single configuration, but we don't create any symlinks in the multi-config + // case. Can we do better? [multi-config] + if (targetConfigurations.size() == 1) { + OutputDirectoryLinksUtils.createOutputDirectoryLinks( + runtime.getWorkspaceName(), getWorkspace(), getExecRoot(), + runtime.getOutputPath(), getReporter(), targetConfigurations.get(0), + request.getSymlinkPrefix()); + } + } + + OutputService outputService = runtime.getOutputService(); + if (outputService != null) { + outputService.startBuild(); + } else { + startLocalOutputBuild(); // TODO(bazel-team): this could be just another OutputService + } + + ActionCache actionCache = getActionCache(); + Builder builder = createBuilder(request, executor, actionCache, skyframeExecutor); + + // + // Execution proper. All statements below are logically nested in + // begin/end pairs. No early returns or exceptions please! + // + + Collection<ConfiguredTarget> configuredTargets = buildResult.getActualTargets(); + getEventBus().post(new ExecutionStartingEvent(configuredTargets)); + + getReporter().handle(Event.progress("Building...")); + + // Conditionally record dependency-checker log: + ExplanationHandler explanationHandler = + installExplanationHandler(request.getBuildOptions().explanationPath, + request.getOptionsDescription()); + + Set<ConfiguredTarget> builtTargets = new HashSet<>(); + boolean interrupted = false; + try { + Iterable<Artifact> allArtifactsForProviders = Iterables.concat(additionalArtifacts, + TopLevelArtifactHelper.getAllArtifactsToBuild( + analysisResult.getTargetsToBuild(), analysisResult.getTopLevelContext()), + TopLevelArtifactHelper.getAllArtifactsToTest(analysisResult.getTargetsToTest())); + if (request.isRunningInEmacs()) { + // The syntax of this message is tightly constrained by lisp/progmodes/compile.el in emacs + request.getOutErr().printErrLn("blaze: Entering directory `" + getExecRoot() + "/'"); + } + for (ActionContextProvider actionContextProvider : actionContextProviders) { + actionContextProvider.executionPhaseStarting( + fileCache, + actionGraph, + allArtifactsForProviders); + } + executor.executionPhaseStarting(); + skyframeExecutor.drainChangedFiles(); + + if (request.getViewOptions().discardAnalysisCache) { + // Free memory by removing cache entries that aren't going to be needed. Note that in + // skyframe full, this destroys the action graph as well, so we can only do it after the + // action graph is no longer needed. + getView().clearAnalysisCache(analysisResult.getTargetsToBuild()); + actionGraph = null; + } + + configureResourceManager(request); + + Profiler.instance().markPhase(ProfilePhase.EXECUTE); + + builder.buildArtifacts(additionalArtifacts, + analysisResult.getParallelTests(), + analysisResult.getExclusiveTests(), + analysisResult.getTargetsToBuild(), + executor, builtTargets, + request.getBuildOptions().explanationPath != null); + + } catch (InterruptedException e) { + interrupted = true; + throw e; + } finally { + if (request.isRunningInEmacs()) { + request.getOutErr().printErrLn("blaze: Leaving directory `" + getExecRoot() + "/'"); + } + if (!interrupted) { + getReporter().handle(Event.progress("Building complete.")); + } + + // Transfer over source file "last save time" stats so the remote logger can find them. + runtime.getEventBus().post(new ExecutionFinishedEvent(ImmutableMap.<String, Long> of(), 0)); + + // Disable system load polling (noop if it was not enabled). + ResourceManager.instance().setAutoSensing(false); + executor.executionPhaseEnding(); + for (ActionContextProvider actionContextProvider : actionContextProviders) { + actionContextProvider.executionPhaseEnding(); + } + + Profiler.instance().markPhase(ProfilePhase.FINISH); + + if (!interrupted) { + saveCaches(actionCache); + } + + long startTime = Profiler.nanoTimeMaybe(); + determineSuccessfulTargets(buildResult, configuredTargets, builtTargets, timer); + showBuildResult(request, buildResult, configuredTargets); + Preconditions.checkNotNull(buildResult.getSuccessfulTargets()); + Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO, "Show results"); + if (explanationHandler != null) { + uninstallExplanationHandler(explanationHandler); + } + // Finalize output service last, so that if we do throw an exception, we know all the other + // code has already run. + if (runtime.getOutputService() != null) { + boolean isBuildSuccessful = + buildResult.getSuccessfulTargets().size() == configuredTargets.size(); + runtime.getOutputService().finalizeBuild(isBuildSuccessful); + } + } + } + + private void prepare(ImmutableMap<PathFragment, Path> packageRoots, + BuildConfigurationCollection configurations) + throws ViewCreationFailedException { + // Prepare for build. + Profiler.instance().markPhase(ProfilePhase.PREPARE); + + // Create some tools symlinks / cleanup per-build state + createActionLogDirectory(); + + // Plant the symlink forest. + plantSymlinkForest(packageRoots, configurations); + } + + private void createToolsSymlinks() throws ExecutorInitException { + try { + runtime.getBinTools().setupBuildTools(); + } catch (ExecException e) { + throw new ExecutorInitException("Tools symlink creation failed: " + + e.getMessage() + "; build aborted", e); + } + } + + private void plantSymlinkForest(ImmutableMap<PathFragment, Path> packageRoots, + BuildConfigurationCollection configurations) throws ViewCreationFailedException { + try { + FileSystemUtils.deleteTreesBelowNotPrefixed(getExecRoot(), + new String[] { ".", "_", Constants.PRODUCT_NAME + "-"}); + // Delete the build configuration's temporary directories + for (BuildConfiguration configuration : configurations.getTargetConfigurations()) { + configuration.prepareForExecutionPhase(); + } + FileSystemUtils.plantLinkForest(packageRoots, getExecRoot()); + } catch (IOException e) { + throw new ViewCreationFailedException("Source forest creation failed: " + e.getMessage() + + "; build aborted", e); + } + } + + private void createActionLogDirectory() throws ViewCreationFailedException { + Path directory = runtime.getDirectories().getActionConsoleOutputDirectory(); + try { + if (directory.exists()) { + FileSystemUtils.deleteTree(directory); + } + directory.createDirectory(); + } catch (IOException ex) { + throw new ViewCreationFailedException("couldn't delete action output directory: " + + ex.getMessage()); + } + } + + /** + * Prepare for a local output build. + */ + private void startLocalOutputBuild() throws BuildFailedException { + long startTime = Profiler.nanoTimeMaybe(); + + try { + Path outputPath = runtime.getOutputPath(); + Path localOutputPath = runtime.getDirectories().getLocalOutputPath(); + + if (outputPath.isSymbolicLink()) { + // Remove the existing symlink first. + outputPath.delete(); + if (localOutputPath.exists()) { + // Pre-existing local output directory. Move to outputPath. + localOutputPath.renameTo(outputPath); + } + } + } catch (IOException e) { + throw new BuildFailedException(e.getMessage()); + } finally { + Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO, + "Starting local output build"); + } + } + + /** + * If a path is supplied, creates and installs an ExplanationHandler. Returns + * an instance on success. Reports an error and returns null otherwise. + */ + private ExplanationHandler installExplanationHandler(PathFragment explanationPath, + String allOptions) { + if (explanationPath == null) { + return null; + } + ExplanationHandler handler; + try { + handler = new ExplanationHandler( + getWorkspace().getRelative(explanationPath).getOutputStream(), + allOptions); + } catch (IOException e) { + getReporter().handle(Event.warn(String.format( + "Cannot write explanation of rebuilds to file '%s': %s", + explanationPath, e.getMessage()))); + return null; + } + getReporter().handle( + Event.info("Writing explanation of rebuilds to '" + explanationPath + "'")); + getReporter().addHandler(handler); + return handler; + } + + /** + * Uninstalls the specified ExplanationHandler (if any) and closes the log + * file. + */ + private void uninstallExplanationHandler(ExplanationHandler handler) { + if (handler != null) { + getReporter().removeHandler(handler); + handler.log.close(); + } + } + + /** + * An ErrorEventListener implementation that records DEPCHECKER events into a log + * file, iff the --explain flag is specified during a build. + */ + private static class ExplanationHandler implements EventHandler { + + private final PrintWriter log; + + private ExplanationHandler(OutputStream log, String optionsDescription) { + this.log = new PrintWriter(log); + this.log.println("Build options: " + optionsDescription); + } + + + @Override + public void handle(Event event) { + if (event.getKind() == EventKind.DEPCHECKER) { + log.println(event.getMessage()); + } + } + } + + /** + * Computes the result of the build. Sets the list of successful (up-to-date) + * targets in the request object. + * + * @param configuredTargets The configured targets whose artifacts are to be + * built. + * @param timer A timer that was started when the execution phase started. + */ + private void determineSuccessfulTargets(BuildResult result, + Collection<ConfiguredTarget> configuredTargets, Set<ConfiguredTarget> builtTargets, + Stopwatch timer) { + // Maintain the ordering by copying builtTargets into a LinkedHashSet in the same iteration + // order as configuredTargets. + Collection<ConfiguredTarget> successfulTargets = new LinkedHashSet<>(); + for (ConfiguredTarget target : configuredTargets) { + if (builtTargets.contains(target)) { + successfulTargets.add(target); + } + } + getEventBus().post( + new ExecutionPhaseCompleteEvent(timer.stop().elapsed(TimeUnit.MILLISECONDS))); + result.setSuccessfulTargets(successfulTargets); + } + + /** + * Shows the result of the build. Information includes the list of up-to-date + * and failed targets and list of output artifacts for successful targets + * + * @param request The build request, which specifies various options. + * @param configuredTargets The configured targets whose artifacts are to be + * built. + * + * TODO(bazel-team): (2010) refactor into using Reporter and info/progress events + */ + private void showBuildResult(BuildRequest request, BuildResult result, + Collection<ConfiguredTarget> configuredTargets) { + // NOTE: be careful what you print! We don't want to create a consistency + // problem where the summary message and the exit code disagree. The logic + // here is already complex. + + // Filter the targets we care about into two buckets: + Collection<ConfiguredTarget> succeeded = new ArrayList<>(); + Collection<ConfiguredTarget> failed = new ArrayList<>(); + for (ConfiguredTarget target : configuredTargets) { + // TODO(bazel-team): this is quite ugly. Add a marker provider for this check. + if (target instanceof InputFileConfiguredTarget) { + // Suppress display of source files (because we do no work to build them). + continue; + } + if (target.getTarget() instanceof Rule) { + Rule rule = (Rule) target.getTarget(); + if (rule.getRuleClass().contains("$")) { + // Suppress display of hidden rules + continue; + } + } + if (target instanceof OutputFileConfiguredTarget) { + // Suppress display of generated files (because they appear underneath + // their generating rule), EXCEPT those ones which are not part of the + // filesToBuild of their generating rule (e.g. .par, _deploy.jar + // files), OR when a user explicitly requests an output file but not + // its rule. + TransitiveInfoCollection generatingRule = + getView().getGeneratingRule((OutputFileConfiguredTarget) target); + if (CollectionUtils.containsAll( + generatingRule.getProvider(FileProvider.class).getFilesToBuild(), + target.getProvider(FileProvider.class).getFilesToBuild()) && + configuredTargets.contains(generatingRule)) { + continue; + } + } + + Collection<ConfiguredTarget> successfulTargets = result.getSuccessfulTargets(); + (successfulTargets.contains(target) ? succeeded : failed).add(target); + } + + // Suppress summary if --show_result value is exceeded: + if (succeeded.size() + failed.size() > request.getBuildOptions().maxResultTargets) { + return; + } + + OutErr outErr = request.getOutErr(); + + for (ConfiguredTarget target : succeeded) { + Label label = target.getLabel(); + // For up-to-date targets report generated artifacts, but only + // if they have associated action and not middleman artifacts. + boolean headerFlag = true; + for (Artifact artifact : getFilesToBuild(target, request)) { + if (!artifact.isSourceArtifact()) { + if (headerFlag) { + outErr.printErr("Target " + label + " up-to-date:\n"); + headerFlag = false; + } + outErr.printErrLn(" " + + OutputDirectoryLinksUtils.getPrettyPath(artifact.getPath(), + runtime.getWorkspaceName(), getWorkspace(), request.getSymlinkPrefix())); + } + } + if (headerFlag) { + outErr.printErr( + "Target " + label + " up-to-date (nothing to build)\n"); + } + } + + for (ConfiguredTarget target : failed) { + outErr.printErr("Target " + target.getLabel() + " failed to build\n"); + + // For failed compilation, it is still useful to examine temp artifacts, + // (ie, preprocessed and assembler files). + TempsProvider tempsProvider = target.getProvider(TempsProvider.class); + if (tempsProvider != null) { + for (Artifact temp : tempsProvider.getTemps()) { + if (temp.getPath().exists()) { + outErr.printErrLn(" See temp at " + + OutputDirectoryLinksUtils.getPrettyPath(temp.getPath(), + runtime.getWorkspaceName(), getWorkspace(), request.getSymlinkPrefix())); + } + } + } + } + if (!failed.isEmpty() && !request.getOptions(ExecutionOptions.class).verboseFailures) { + outErr.printErr("Use --verbose_failures to see the command lines of failed build steps.\n"); + } + } + + /** + * Gets all the files to build for a given target and build request. + * There may be artifacts that should be built which are not represented in the + * configured target graph. Currently, this only occurs when "--save_temps" is on. + * + * @param target configured target + * @param request the build request + * @return artifacts to build + */ + private static Collection<Artifact> getFilesToBuild(ConfiguredTarget target, + BuildRequest request) { + ImmutableSet.Builder<Artifact> result = ImmutableSet.builder(); + if (request.getBuildOptions().compileOnly) { + FilesToCompileProvider provider = target.getProvider(FilesToCompileProvider.class); + if (provider != null) { + result.addAll(provider.getFilesToCompile()); + } + } else if (request.getBuildOptions().compilationPrerequisitesOnly) { + CompilationPrerequisitesProvider provider = + target.getProvider(CompilationPrerequisitesProvider.class); + if (provider != null) { + result.addAll(provider.getCompilationPrerequisites()); + } + } else { + FileProvider provider = target.getProvider(FileProvider.class); + if (provider != null) { + result.addAll(provider.getFilesToBuild()); + } + } + TempsProvider tempsProvider = target.getProvider(TempsProvider.class); + if (tempsProvider != null) { + result.addAll(tempsProvider.getTemps()); + } + + return result.build(); + } + + private ActionCache getActionCache() throws LocalEnvironmentException { + try { + return runtime.getPersistentActionCache(); + } catch (IOException e) { + // TODO(bazel-team): (2010) Ideally we should just remove all cache data and reinitialize + // caches. + LoggingUtil.logToRemote(Level.WARNING, "Failed to initialize action cache: " + + e.getMessage(), e); + throw new LocalEnvironmentException("couldn't create action cache: " + e.getMessage() + + ". If error persists, use 'blaze clean'"); + } + } + + private Builder createBuilder(BuildRequest request, + Executor executor, + ActionCache actionCache, + SkyframeExecutor skyframeExecutor) { + BuildRequest.BuildRequestOptions options = request.getBuildOptions(); + boolean verboseExplanations = options.verboseExplanations; + boolean keepGoing = request.getViewOptions().keepGoing; + + Path actionOutputRoot = runtime.getDirectories().getActionConsoleOutputDirectory(); + Predicate<Action> executionFilter = CheckUpToDateFilter.fromOptions( + request.getOptions(ExecutionOptions.class)); + + // jobs should have been verified in BuildRequest#validateOptions(). + Preconditions.checkState(options.jobs >= -1); + int actualJobs = options.jobs == 0 ? 1 : options.jobs; // Treat 0 jobs as a single task. + + // Unfortunately, the exec root cache is not shared with caches in the remote execution + // client. + fileCache = createBuildSingleFileCache(executor.getExecRoot()); + skyframeExecutor.setActionOutputRoot(actionOutputRoot); + return new SkyframeBuilder(skyframeExecutor, + new ActionCacheChecker(actionCache, getView().getArtifactFactory(), executionFilter, + verboseExplanations), + keepGoing, actualJobs, options.checkOutputFiles, fileCache, + request.getBuildOptions().progressReportInterval); + } + + private void configureResourceManager(BuildRequest request) { + ResourceManager resourceMgr = ResourceManager.instance(); + ExecutionOptions options = request.getOptions(ExecutionOptions.class); + if (options.availableResources != null) { + resourceMgr.setAvailableResources(options.availableResources); + resourceMgr.setRamUtilizationPercentage(100); + } else { + resourceMgr.setAvailableResources(LocalHostCapacity.getLocalHostCapacity()); + resourceMgr.setRamUtilizationPercentage(options.ramUtilizationPercentage); + if (options.useResourceAutoSense) { + getReporter().handle( + Event.warn("Not using resource autosense due to known responsiveness issues")); + } + ResourceManager.instance().setAutoSensing(/*autosense=*/false); + } + } + + /** + * Writes the cache files to disk, reporting any errors that occurred during + * writing. + */ + private void saveCaches(ActionCache actionCache) { + long actionCacheSizeInBytes = 0; + long actionCacheSaveTime; + + long startTime = BlazeClock.nanoTime(); + try { + LOG.info("saving action cache..."); + actionCacheSizeInBytes = actionCache.save(); + LOG.info("action cache saved"); + } catch (IOException e) { + getReporter().handle(Event.error("I/O error while writing action log: " + e.getMessage())); + } finally { + long stopTime = BlazeClock.nanoTime(); + actionCacheSaveTime = + TimeUnit.MILLISECONDS.convert(stopTime - startTime, TimeUnit.NANOSECONDS); + Profiler.instance().logSimpleTask(startTime, stopTime, + ProfilerTask.INFO, "Saving action cache"); + } + + runtime.getEventBus().post(new CachesSavedEvent( + actionCacheSaveTime, actionCacheSizeInBytes)); + } + + private ActionInputFileCache createBuildSingleFileCache(Path execRoot) { + String cwd = execRoot.getPathString(); + FileSystem fs = runtime.getDirectories().getFileSystem(); + + ActionInputFileCache cache = null; + for (BlazeModule module : runtime.getBlazeModules()) { + ActionInputFileCache pluggable = module.createActionInputCache(cwd, fs); + if (pluggable != null) { + Preconditions.checkState(cache == null); + cache = pluggable; + } + } + + if (cache == null) { + cache = new SingleBuildFileCache(cwd, fs); + } + return cache; + } + + private Reporter getReporter() { + return runtime.getReporter(); + } + + private EventBus getEventBus() { + return runtime.getEventBus(); + } + + private BuildView getView() { + return runtime.getView(); + } + + private Path getWorkspace() { + return runtime.getWorkspace(); + } + + private Path getExecRoot() { + return runtime.getExecRoot(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/LocalEnvironmentException.java b/src/main/java/com/google/devtools/build/lib/buildtool/LocalEnvironmentException.java new file mode 100644 index 0000000..7890b22 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/LocalEnvironmentException.java
@@ -0,0 +1,45 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool; + +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; + +/** + * An exception that signals that something is wrong with the user's environment + * that he can fix. Used to report the problem of having no free space left in + * the blaze output directory. + * + * <p>Note that this is a much higher level exception then the similarly named + * EnvironmentExecException, which is thrown from the base Client and Strategy + * layers of Blaze. + * + * <p>This exception is only thrown when we've decided that the build has, in + * fact, failed and we should exit. + */ +public class LocalEnvironmentException extends AbruptExitException { + + public LocalEnvironmentException(String message) { + super(message, ExitCode.LOCAL_ENVIRONMENTAL_ERROR); + } + + public LocalEnvironmentException(Throwable cause) { + super(ExitCode.LOCAL_ENVIRONMENTAL_ERROR, cause); + } + + public LocalEnvironmentException(String message, Throwable cause) { + super(message, ExitCode.LOCAL_ENVIRONMENTAL_ERROR, cause); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java b/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java new file mode 100644 index 0000000..094b7bc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java
@@ -0,0 +1,184 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.common.base.Joiner; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Symlinks; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Static utilities for managing output directory symlinks. + */ +public class OutputDirectoryLinksUtils { + public static final String OUTPUT_SYMLINK_NAME = Constants.PRODUCT_NAME + "-out"; + + // Used in getPrettyPath() method below. + private static final String[] LINKS = { "bin", "genfiles", "includes" }; + + private static final String NO_CREATE_SYMLINKS_PREFIX = "/"; + + private static String execRootSymlink(String workspaceName) { + return Constants.PRODUCT_NAME + "-" + workspaceName; + } + /** + * Attempts to create convenience symlinks in the workspaceDirectory and in + * execRoot to the output area and to the configuration-specific output + * directories. Issues a warning if it fails, e.g. because workspaceDirectory + * is readonly. + */ + public static void createOutputDirectoryLinks(String workspaceName, + Path workspace, Path execRoot, Path outputPath, + EventHandler eventHandler, BuildConfiguration targetConfig, String symlinkPrefix) { + if (NO_CREATE_SYMLINKS_PREFIX.equals(symlinkPrefix)) { + return; + } + List<String> failures = new ArrayList<>(); + + // Make the two non-specific links from the workspace to the output area, + // and the configuration-specific links in both the workspace and the execution root dirs. + // NB! Keep in sync with removeOutputDirectoryLinks below. + createLink(workspace, OUTPUT_SYMLINK_NAME, outputPath, failures); + + // Points to execroot + createLink(workspace, execRootSymlink(workspaceName), execRoot, failures); + createLink(workspace, symlinkPrefix + "bin", targetConfig.getBinDirectory().getPath(), + failures); + createLink(workspace, symlinkPrefix + "testlogs", targetConfig.getTestLogsDirectory().getPath(), + failures); + createLink(workspace, symlinkPrefix + "genfiles", targetConfig.getGenfilesDirectory().getPath(), + failures); + if (!failures.isEmpty()) { + eventHandler.handle(Event.warn(String.format( + "failed to create one or more convenience symlinks for prefix '%s':\n %s", + symlinkPrefix, Joiner.on("\n ").join(failures)))); + } + } + + /** + * Returns a convenient path to the specified file, relativizing it and using output-dir symlinks + * if possible. Otherwise, return a path relative to the workspace directory if possible. + * Otherwise, return the absolute path. + * + * <p>This method must be called after the symlinks are created at the end of a build. If called + * before, the pretty path may be incorrect if the symlinks end up pointing somewhere new. + */ + public static PathFragment getPrettyPath(Path file, String workspaceName, + Path workspaceDirectory, String symlinkPrefix) { + for (String link : LINKS) { + PathFragment result = relativize(file, workspaceDirectory, symlinkPrefix + link); + if (result != null) { + return result; + } + } + + PathFragment result = relativize(file, workspaceDirectory, execRootSymlink(workspaceName)); + if (result != null) { + return result; + } + + result = relativize(file, workspaceDirectory, OUTPUT_SYMLINK_NAME); + if (result != null) { + return result; + } + + return file.asFragment(); + } + + // Helper to getPrettyPath. Returns file, relativized w.r.t. the referent of + // "linkname", or null if it was a not a child. + private static PathFragment relativize(Path file, Path workspaceDirectory, String linkname) { + PathFragment link = new PathFragment(linkname); + try { + Path dir = workspaceDirectory.getRelative(link); + PathFragment levelOneLinkTarget = dir.readSymbolicLink(); + if (levelOneLinkTarget.isAbsolute() && + file.startsWith(dir = file.getRelative(levelOneLinkTarget))) { + return link.getRelative(file.relativeTo(dir)); + } + } catch (IOException e) { + /* ignore */ + } + return null; + } + + /** + * Attempts to remove the convenience symlinks in the workspace directory. + * + * <p>Issues a warning if it fails, e.g. because workspaceDirectory is readonly. + * Also cleans up any child directories created by a custom prefix. + * + * @param workspace the runtime's workspace + * @param eventHandler the error eventHandler + * @param symlinkPrefix the symlink prefix which should be removed + */ + public static void removeOutputDirectoryLinks(String workspaceName, Path workspace, + EventHandler eventHandler, String symlinkPrefix) { + if (NO_CREATE_SYMLINKS_PREFIX.equals(symlinkPrefix)) { + return; + } + List<String> failures = new ArrayList<>(); + + removeLink(workspace, OUTPUT_SYMLINK_NAME, failures); + removeLink(workspace, execRootSymlink(workspaceName), failures); + removeLink(workspace, symlinkPrefix + "bin", failures); + removeLink(workspace, symlinkPrefix + "testlogs", failures); + removeLink(workspace, symlinkPrefix + "genfiles", failures); + FileSystemUtils.removeDirectoryAndParents(workspace, new PathFragment(symlinkPrefix)); + if (!failures.isEmpty()) { + eventHandler.handle(Event.warn(String.format( + "failed to remove one or more convenience symlinks for prefix '%s':\n %s", symlinkPrefix, + Joiner.on("\n ").join(failures)))); + } + } + + /** + * Helper to createOutputDirectoryLinks that creates a symlink from base + name to target. + */ + private static boolean createLink(Path base, String name, Path target, List<String> failures) { + try { + FileSystemUtils.ensureSymbolicLink(base.getRelative(name), target); + return true; + } catch (IOException e) { + failures.add(String.format("%s -> %s: %s", name, target.getPathString(), e.getMessage())); + return false; + } + } + + /** + * Helper to removeOutputDirectoryLinks that removes one of the Blaze convenience symbolic links. + */ + private static boolean removeLink(Path base, String name, List<String> failures) { + Path link = base.getRelative(name); + try { + if (link.exists(Symlinks.NOFOLLOW)) { + ExecutionTool.LOG.finest("Removing " + link); + link.delete(); + } + return true; + } catch (IOException e) { + failures.add(String.format("%s: %s", name, e.getMessage())); + return false; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java new file mode 100644 index 0000000..779515a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java
@@ -0,0 +1,355 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionCacheChecker; +import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BuildFailedException; +import com.google.devtools.build.lib.actions.BuilderUtils; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceManager; +import com.google.devtools.build.lib.actions.TestExecException; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.TargetCompleteEvent; +import com.google.devtools.build.lib.rules.test.TestProvider; +import com.google.devtools.build.lib.skyframe.ActionExecutionInactivityWatchdog; +import com.google.devtools.build.lib.skyframe.ActionExecutionValue; +import com.google.devtools.build.lib.skyframe.Builder; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.skyframe.TargetCompletionValue; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.skyframe.CycleInfo; +import com.google.devtools.build.skyframe.ErrorInfo; +import com.google.devtools.build.skyframe.EvaluationProgressReceiver; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.text.NumberFormat; +import java.util.Collection; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A {@link Builder} implementation driven by Skyframe. + */ +@VisibleForTesting +public class SkyframeBuilder implements Builder { + + private final SkyframeExecutor skyframeExecutor; + private final boolean keepGoing; + private final int numJobs; + private final boolean checkOutputFiles; + private final ActionInputFileCache fileCache; + private final ActionCacheChecker actionCacheChecker; + private final int progressReportInterval; + + @VisibleForTesting + public SkyframeBuilder(SkyframeExecutor skyframeExecutor, ActionCacheChecker actionCacheChecker, + boolean keepGoing, int numJobs, boolean checkOutputFiles, + ActionInputFileCache fileCache, int progressReportInterval) { + this.skyframeExecutor = skyframeExecutor; + this.actionCacheChecker = actionCacheChecker; + this.keepGoing = keepGoing; + this.numJobs = numJobs; + this.checkOutputFiles = checkOutputFiles; + this.fileCache = fileCache; + this.progressReportInterval = progressReportInterval; + } + + @Override + public void buildArtifacts(Set<Artifact> artifacts, + Set<ConfiguredTarget> parallelTests, + Set<ConfiguredTarget> exclusiveTests, + Collection<ConfiguredTarget> targetsToBuild, + Executor executor, + Set<ConfiguredTarget> builtTargets, + boolean explain) + throws BuildFailedException, AbruptExitException, TestExecException, InterruptedException { + skyframeExecutor.prepareExecution(checkOutputFiles); + skyframeExecutor.setFileCache(fileCache); + // Note that executionProgressReceiver accesses builtTargets concurrently (after wrapping in a + // synchronized collection), so unsynchronized access to this variable is unsafe while it runs. + ExecutionProgressReceiver executionProgressReceiver = + new ExecutionProgressReceiver(Preconditions.checkNotNull(builtTargets), + countTestActions(exclusiveTests), skyframeExecutor.getEventBus()); + ResourceManager.instance().setEventBus(skyframeExecutor.getEventBus()); + + boolean success = false; + EvaluationResult<?> result; + + ActionExecutionStatusReporter statusReporter = ActionExecutionStatusReporter.create( + skyframeExecutor.getReporter(), executor, skyframeExecutor.getEventBus()); + + AtomicBoolean isBuildingExclusiveArtifacts = new AtomicBoolean(false); + ActionExecutionInactivityWatchdog watchdog = new ActionExecutionInactivityWatchdog( + executionProgressReceiver.createInactivityMonitor(statusReporter), + executionProgressReceiver.createInactivityReporter(statusReporter, + isBuildingExclusiveArtifacts), progressReportInterval); + + skyframeExecutor.setActionExecutionProgressReportingObjects(executionProgressReceiver, + executionProgressReceiver, statusReporter); + watchdog.start(); + + try { + result = skyframeExecutor.buildArtifacts(executor, artifacts, targetsToBuild, parallelTests, + /*exclusiveTesting=*/false, keepGoing, explain, numJobs, actionCacheChecker, + executionProgressReceiver); + // progressReceiver is finished, so unsynchronized access to builtTargets is now safe. + success = processResult(result, keepGoing, skyframeExecutor); + + Preconditions.checkState( + !success || result.keyNames().size() + == (artifacts.size() + targetsToBuild.size() + parallelTests.size()), + "Build reported as successful but not all artifacts and targets built: %s, %s", + result, artifacts); + + // Run exclusive tests: either tagged as "exclusive" or is run in an invocation with + // --test_output=streamed. + isBuildingExclusiveArtifacts.set(true); + for (ConfiguredTarget exclusiveTest : exclusiveTests) { + // Since only one artifact is being built at a time, we don't worry about an artifact being + // built and then the build being interrupted. + result = skyframeExecutor.buildArtifacts(executor, ImmutableSet.<Artifact>of(), + targetsToBuild, ImmutableSet.of(exclusiveTest), /*exclusiveTesting=*/true, keepGoing, + explain, numJobs, actionCacheChecker, null); + boolean exclusiveSuccess = processResult(result, keepGoing, skyframeExecutor); + Preconditions.checkState(!exclusiveSuccess || !result.keyNames().isEmpty(), + "Build reported as successful but test %s not executed: %s", + exclusiveTest, result); + success &= exclusiveSuccess; + } + } finally { + watchdog.stop(); + ResourceManager.instance().unsetEventBus(); + skyframeExecutor.setActionExecutionProgressReportingObjects(null, null, null); + statusReporter.unregisterFromEventBus(); + } + + if (!success) { + throw new BuildFailedException(); + } + } + + private static boolean resultHasCatastrophicError(EvaluationResult<?> result) { + for (ErrorInfo errorInfo : result.errorMap().values()) { + if (errorInfo.isCatastrophic()) { + return true; + } + } + // An unreported catastrophe manifests with hasError() being true but no errors visible. + return result.hasError() && result.errorMap().isEmpty(); + } + + /** + * Process the Skyframe update, taking into account the keepGoing setting. + * + * Returns false if the update() failed, but we should continue. Returns true on success. + * Throws on fail-fast failures. + */ + private static boolean processResult(EvaluationResult<?> result, boolean keepGoing, + SkyframeExecutor skyframeExecutor) throws BuildFailedException, TestExecException { + if (result.hasError()) { + boolean hasCycles = false; + for (Map.Entry<SkyKey, ErrorInfo> entry : result.errorMap().entrySet()) { + Iterable<CycleInfo> cycles = entry.getValue().getCycleInfo(); + skyframeExecutor.reportCycles(cycles, entry.getKey()); + hasCycles |= !Iterables.isEmpty(cycles); + } + if (keepGoing && !resultHasCatastrophicError(result)) { + return false; + } + if (hasCycles || result.errorMap().isEmpty()) { + // error map may be empty in the case of a catastrophe. + throw new BuildFailedException(); + } else { + // Need to wrap exception for rethrowCause. + BuilderUtils.rethrowCause( + new Exception(Preconditions.checkNotNull(result.getError().getException()))); + } + } + return true; + } + + private static int countTestActions(Iterable<ConfiguredTarget> testTargets) { + int count = 0; + for (ConfiguredTarget testTarget : testTargets) { + count += TestProvider.getTestStatusArtifacts(testTarget).size(); + } + return count; + } + + /** + * Listener for executed actions and built artifacts. We use a listener so that we have an + * accurate set of successfully run actions and built artifacts, even if the build is interrupted. + */ + private static final class ExecutionProgressReceiver implements EvaluationProgressReceiver, + SkyframeActionExecutor.ProgressSupplier, SkyframeActionExecutor.ActionCompletedReceiver { + private static final NumberFormat PROGRESS_MESSAGE_NUMBER_FORMATTER; + + // Must be thread-safe! + private final Set<ConfiguredTarget> builtTargets; + private final Set<SkyKey> enqueuedActions = Sets.newConcurrentHashSet(); + private final Set<Action> completedActions = Sets.newConcurrentHashSet(); + private final Object activityIndicator = new Object(); + /** Number of exclusive tests. To be accounted for in progress messages. */ + private final int exclusiveTestsCount; + private final EventBus eventBus; + + static { + PROGRESS_MESSAGE_NUMBER_FORMATTER = NumberFormat.getIntegerInstance(Locale.ENGLISH); + PROGRESS_MESSAGE_NUMBER_FORMATTER.setGroupingUsed(true); + } + + /** + * {@code builtTargets} is accessed through a synchronized set, and so no other access to it + * is permitted while this receiver is active. + */ + ExecutionProgressReceiver(Set<ConfiguredTarget> builtTargets, int exclusiveTestsCount, + EventBus eventBus) { + this.builtTargets = Collections.synchronizedSet(builtTargets); + this.exclusiveTestsCount = exclusiveTestsCount; + this.eventBus = eventBus; + } + + @Override + public void invalidated(SkyValue node, InvalidationState state) {} + + @Override + public void enqueueing(SkyKey skyKey) { + if (ActionExecutionValue.isReportWorthyAction(skyKey)) { + // Remember all enqueued actions for the benefit of progress reporting. + // We discover most actions early in the build, well before we start executing them. + // Some of these will be cache hits and won't be executed, so we'll need to account for them + // in the evaluated method too. + enqueuedActions.add(skyKey); + } + } + + @Override + public void evaluated(SkyKey skyKey, SkyValue node, EvaluationState state) { + SkyFunctionName type = skyKey.functionName(); + if (type == SkyFunctions.TARGET_COMPLETION) { + TargetCompletionValue val = (TargetCompletionValue) node; + ConfiguredTarget target = val.getConfiguredTarget(); + builtTargets.add(target); + eventBus.post(TargetCompleteEvent.createSuccessful(target)); + } else if (type == SkyFunctions.ACTION_EXECUTION) { + // Remember all completed actions, regardless of having been cached or really executed. + actionCompleted((Action) skyKey.argument()); + } + } + + /** + * {@inheritDoc} + * + * <p>This method adds the action to {@link #completedActions} and notifies the + * {@link #activityIndicator}. + * + * <p>We could do this only in the {@link #evaluated} method too, but as it happens the action + * executor tells the reporter about the completed action before the node is inserted into the + * graph, so the reporter would find out about the completed action sooner than we could + * have updated {@link #completedActions}, which would result in incorrect numbers on the + * progress messages. However we have to store completed actions in {@link #evaluated} too, + * because that's the only place we get notified about completed cached actions. + */ + @Override + public void actionCompleted(Action a) { + if (ActionExecutionValue.isReportWorthyAction(a)) { + completedActions.add(a); + synchronized (activityIndicator) { + activityIndicator.notifyAll(); + } + } + } + + @Override + public String getProgressString() { + return String.format("[%s / %s]", + PROGRESS_MESSAGE_NUMBER_FORMATTER.format(completedActions.size()), + PROGRESS_MESSAGE_NUMBER_FORMATTER.format(exclusiveTestsCount + enqueuedActions.size())); + } + + ActionExecutionInactivityWatchdog.InactivityMonitor createInactivityMonitor( + final ActionExecutionStatusReporter statusReporter) { + return new ActionExecutionInactivityWatchdog.InactivityMonitor() { + + @Override + public boolean hasStarted() { + return !enqueuedActions.isEmpty(); + } + + @Override + public int getPending() { + return statusReporter.getCount(); + } + + @Override + public int waitForNextCompletion(int timeoutMilliseconds) throws InterruptedException { + synchronized (activityIndicator) { + int before = completedActions.size(); + long startTime = BlazeClock.instance().currentTimeMillis(); + while (true) { + activityIndicator.wait(timeoutMilliseconds); + + int completed = completedActions.size() - before; + long now = 0; + if (completed > 0 || (startTime + timeoutMilliseconds) <= (now = BlazeClock.instance() + .currentTimeMillis())) { + // Some actions completed, or timeout fully elapsed. + return completed; + } else { + // Spurious Wakeup -- no actions completed and there's still time to wait. + timeoutMilliseconds -= now - startTime; // account for elapsed wait time + startTime = now; + } + } + } + } + }; + } + + ActionExecutionInactivityWatchdog.InactivityReporter createInactivityReporter( + final ActionExecutionStatusReporter statusReporter, + final AtomicBoolean isBuildingExclusiveArtifacts) { + return new ActionExecutionInactivityWatchdog.InactivityReporter() { + @Override + public void maybeReportInactivity() { + // Do not report inactivity if we are currently running an exclusive test or a streaming + // action (in practice only tests can stream and it implicitly makes them exclusive). + if (!isBuildingExclusiveArtifacts.get()) { + statusReporter.showCurrentlyExecutingActions( + ExecutionProgressReceiver.this.getProgressString() + " "); + } + } + }; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/TargetValidator.java b/src/main/java/com/google/devtools/build/lib/buildtool/TargetValidator.java new file mode 100644 index 0000000..e6eed80 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/TargetValidator.java
@@ -0,0 +1,37 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.LoadingFailedException; + +import java.util.Collection; + +/** + * Validator for targets. + * + * <p>Used in "blaze run" to make sure that we are building exactly one binary target. + */ +public interface TargetValidator { + + /** + * Hook for subclasses to validate a build request before building begins. + * Implementors should print warnings for invalid targets iff keepGoing. + * + * @param targets The targets to build. + * @throws LoadingFailedException if the request is not valid for some reason. + */ + void validateTargets(Collection<Target> targets, boolean keepGoing) + throws LoadingFailedException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildCompleteEvent.java new file mode 100644 index 0000000..e9278e6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildCompleteEvent.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool.buildevent; + +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.BuildResult; + +/** + * This event is fired from BuildTool#stopRequest(). + */ +public final class BuildCompleteEvent { + private final BuildResult result; + + /** + * Construct the BuildStartingEvent. + * @param request the build request. + */ + public BuildCompleteEvent(BuildRequest request, BuildResult result) { + this.result = result; + } + + /** + * @return the build summary + */ + public BuildResult getResult() { + return result; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildInterruptedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildInterruptedEvent.java new file mode 100644 index 0000000..02a5d8b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildInterruptedEvent.java
@@ -0,0 +1,22 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool.buildevent; + +/** + * This event is fired from {@code AbstractBuildCommand#doBuild} to indicate + * that the user interrupted the build with control-C. + */ +public class BuildInterruptedEvent { +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildStartingEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildStartingEvent.java new file mode 100644 index 0000000..714534d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildStartingEvent.java
@@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool.buildevent; + +import com.google.devtools.build.lib.buildtool.BuildRequest; + +/** + * This event is fired from BuildTool#startRequest(). + * At this point, the set of target patters are known, but have + * yet to be parsed. + */ +public class BuildStartingEvent { + private final String outputFileSystem; + private final BuildRequest request; + + /** + * Construct the BuildStartingEvent. + * @param request the build request. + */ + public BuildStartingEvent(String outputFileSystem, BuildRequest request) { + this.outputFileSystem = outputFileSystem; + this.request = request; + } + + /** + * @return the output file system. + */ + public String getOutputFileSystem() { + return outputFileSystem; + } + + /** + * @return the active BuildRequest. + */ + public BuildRequest getRequest() { + return request; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionPhaseCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionPhaseCompleteEvent.java new file mode 100644 index 0000000..cf57960 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionPhaseCompleteEvent.java
@@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool.buildevent; + +/** + * This event is fired after the execution phase is complete. + */ +public class ExecutionPhaseCompleteEvent { + private final long timeInMs; + + /** + * Construct the event. + * + * @param timeInMs time for execution phase in milliseconds. + */ + public ExecutionPhaseCompleteEvent(long timeInMs) { + this.timeInMs = timeInMs; + } + + public long getTimeInMs() { + return timeInMs; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionStartingEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionStartingEvent.java new file mode 100644 index 0000000..c2b4f77 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionStartingEvent.java
@@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool.buildevent; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.buildtool.ExecutionTool; + +import java.util.Collection; + +/** + * This event is fired from {@link ExecutionTool#executeBuild} to indicate that the execution phase + * of the build is starting. + */ +public class ExecutionStartingEvent { + private final Collection<TransitiveInfoCollection> targets; + + /** + * Construct the event with a set of targets. + * @param targets Remaining active targets. + */ + public ExecutionStartingEvent(Collection<? extends TransitiveInfoCollection> targets) { + this.targets = ImmutableList.copyOf(targets); + } + + /** + * @return The set of active targets remaining, which is a subset + * of the targets in the user request. + */ + public Collection<TransitiveInfoCollection> getTargets() { + return targets; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/TestFilteringCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/TestFilteringCompleteEvent.java new file mode 100644 index 0000000..c380456 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/TestFilteringCompleteEvent.java
@@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool.buildevent; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.rules.test.TestProvider; + +import java.util.Collection; + +import javax.annotation.concurrent.Immutable; + +/** + * This event is fired after test filtering. + * + * The test filtering phase always expands test_suite rules, so + * the set of active targets should never contain test_suites. + */ +@Immutable +public class TestFilteringCompleteEvent { + private final Collection<ConfiguredTarget> targets; + private final Collection<ConfiguredTarget> testTargets; + + /** + * Construct the event. + * @param targets The set of active targets that remain. + * @param testTargets The collection of tests to be run. May be null. + */ + public TestFilteringCompleteEvent( + Collection<? extends ConfiguredTarget> targets, + Collection<? extends ConfiguredTarget> testTargets) { + this.targets = ImmutableList.copyOf(targets); + this.testTargets = testTargets == null ? null : ImmutableList.copyOf(testTargets); + if (testTargets == null) { + return; + } + + for (ConfiguredTarget testTarget : testTargets) { + Preconditions.checkState(testTarget.getProvider(TestProvider.class) != null); + } + } + + /** + * @return The set of active targets remaining. This is a subset of + * the targets that passed analysis, after test_suite expansion. + */ + public Collection<ConfiguredTarget> getTargets() { + return targets; + } + + /** + * @return The set of test targets to be run. May be null. + */ + public Collection<ConfiguredTarget> getTestTargets() { + return testTargets; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/LabelValidator.java b/src/main/java/com/google/devtools/build/lib/cmdline/LabelValidator.java new file mode 100644 index 0000000..50b3379 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/cmdline/LabelValidator.java
@@ -0,0 +1,289 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.cmdline; + +import com.google.common.base.CharMatcher; + +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * The canonical place to parse and validate Blaze labels. + */ +public final class LabelValidator { + + /** + * Matches punctuation in target names which requires quoting in a blaze query. + */ + private static final CharMatcher PUNCTUATION_REQUIRING_QUOTING = CharMatcher.anyOf("+,=~"); + + /** + * Matches punctuation in target names which doesn't require quoting in a blaze query. + * + * Note that . is also allowed in target names, and doesn't require quoting, but has restrictions + * on its surrounding characters; see {@link #validateTargetName(String)}. + */ + private static final CharMatcher PUNCTUATION_NOT_REQUIRING_QUOTING = CharMatcher.anyOf("_-@"); + + /** + * Matches characters allowed in target names regardless of context. + * + * Note that the only other characters allowed in target names are / and . but they have + * restrictions around surrounding characters; see {@link #validateTargetName(String)}. + */ + private static final CharMatcher ALWAYS_ALLOWED_TARGET_CHARACTERS = + CharMatcher.JAVA_LETTER_OR_DIGIT + .or(PUNCTUATION_REQUIRING_QUOTING) + .or(PUNCTUATION_NOT_REQUIRING_QUOTING); + + private static final String PACKAGE_NAME_ERROR = + "package names may contain only A-Z, a-z, 0-9, '/', '-' and '_'"; + + /** + * Performs validity checking of the specified package name. Returns null on success or an error + * message otherwise. + * + * @param packageName the name of the package + * @return null if {@code name} is valid or an error string if any part + * of the package name is invalid + */ + @Nullable + public static String validatePackageName(String packageName) { + int len = packageName.length(); + if (len == 0) { + return "empty package name"; + } + char first = packageName.charAt(0); + if (first < 'a' || first > 'z') { + return "package names must start with a lowercase ASCII letter"; + } + + // Fast path for packages with '.' in their name + if (packageName.lastIndexOf('.') != -1) { + return PACKAGE_NAME_ERROR; + } + + // Check for any character outside of [/0-9A-Z_a-z-]. Try to evaluate the + // conditional quickly (by looking in decreasing order of character class + // likelihood). + for (int i = len - 1; i >= 0; --i) { + char c = packageName.charAt(i); + if ((c < 'a' || c > 'z') && c != '/' && c != '_' && c != '-' && + (c < '0' || c > '9') && (c < 'A' || c > 'Z')) { + return PACKAGE_NAME_ERROR; + } + } + + if (packageName.contains("//")) { + return "package names may not contain '//' path separators"; + } + if (packageName.endsWith("/")) { + return "package names may not end with '/'"; + } + return null; // ok + } + + /** + * Performs validity checking of the specified target name. Returns null on success or an error + * message otherwise. + */ + @Nullable + public static String validateTargetName(String targetName) { + // TODO(bazel-team): (2011) allow labels equaling '.' or ending in '/.' for now. If we ever + // actually configure the target we will report an error, but they will be permitted for + // data directories. + + // TODO(bazel-team): (2011) Get rid of this code once we have reached critical mass and can + // pressure developers to clean up their BUILD files. + + // Code optimized for the common case: success. + int len = targetName.length(); + if (len == 0) { + return "empty target name"; + } + // Forbidden start chars: + char c = targetName.charAt(0); + if (c == '/') { + return "target names may not start with '/'"; + } else if (c == '.') { + if (targetName.startsWith("../") || targetName.equals("..")) { + return "target names may not contain up-level references '..'"; + } else if (targetName.equals(".")) { + return null; // See comment above; ideally should be an error. + } else if (targetName.startsWith("./")) { + return "target names may not contain '.' as a path segment"; + } + } + + // Give a friendly error message on CRs in target names + if (targetName.endsWith("\r")) { + return "target names may not end with carriage returns " + + "(perhaps the input source is CRLF-terminated)"; + } + + for (int ii = 0; ii < len; ++ii) { + c = targetName.charAt(ii); + if (ALWAYS_ALLOWED_TARGET_CHARACTERS.matches(c)) { + continue; + } + if (c == '.') { + continue; + } + if (c == '/') { + if (targetName.substring(ii).startsWith("/../")) { + return "target names may not contain up-level references '..'"; + } else if (targetName.substring(ii).startsWith("/./")) { + return "target names may not contain '.' as a path segment"; + } else if (targetName.substring(ii).startsWith("//")) { + return "target names may not contain '//' path separators"; + } + continue; + } + if (CharMatcher.JAVA_ISO_CONTROL.matches(c)) { + return "target names may not contain non-printable characters: '" + + String.format("\\x%02X", (int) c) + "'"; + } + return "target names may not contain '" + c + "'"; + } + // Forbidden end chars: + if (c == '.' && targetName.endsWith("/..")) { + return "target names may not contain up-level references '..'"; + } else if (c == '.' && targetName.endsWith("/.")) { + return null; // See comment above; ideally should be an error. + } + if (c == '/') { + return "target names may not end with '/'"; + } + return null; // ok + } + + /** + * Validate the label and parse it into a pair of package name and target name. If the label is + * not valid, it throws an {@link BadLabelException}. + * + * <p>It accepts these forms of labels: + * <pre> + * //foo/bar + * //foo/bar:quux + * //foo/bar: (undocumented, but accepted) + * </pre> + */ + public static PackageAndTarget validateAbsoluteLabel(String absName) throws BadLabelException { + PackageAndTarget result = parseAbsoluteLabel(absName); + String packageName = result.getPackageName(); + String targetName = result.getTargetName(); + String error = validatePackageName(packageName); + if (error != null) { + error = "invalid package name '" + packageName + "': " + error; + // This check is just for a more helpful error message, + // i.e. valid target name, invalid package name, colon-free label form + // used => probably they meant "//foo:bar.c" not "//foo/bar.c". + if (packageName.endsWith("/" + targetName)) { + error += " (perhaps you meant \":" + targetName + "\"?)"; + } + throw new BadLabelException(error); + } + error = validateTargetName(targetName); + if (error != null) { + error = "invalid target name '" + targetName + "': " + error; + throw new BadLabelException(error); + } + return result; + } + + /** + * Parses the given absolute label by verifying that it starts with "//". If it contains a ':', + * then the part after that is the target name within the package, and the part before that (but + * without the leading "//") is the package name. However, it performs no validation on these two + * pieces. + * + * <p>Use of this method is generally not recommended. + * + * @throws NullPointerException if {@code absName} is {@code null} + * @throws BadLabelException if {@code absName} starts with "//" + */ + public static PackageAndTarget parseAbsoluteLabel(String absName) throws BadLabelException { + if (!absName.startsWith("//")) { + throw new BadLabelException("invalid label: " + absName); + } + // Find the package/suffix separation: + int colonIndex = absName.indexOf(':'); + int splitAt = colonIndex >= 0 ? colonIndex : absName.length(); + String packageName = absName.substring("//".length(), splitAt); + String suffix = absName.substring(splitAt); + // ('suffix' is empty, or starts with a colon.) + + // "If packagename and version are elided, the colon is not necessary." + String targetName = suffix.isEmpty() + // Target name is last package segment: (works in slash-free case too.) + ? packageName.substring(packageName.lastIndexOf('/') + 1) + // Target name is what's after colon: + : suffix.substring(1); + + return new PackageAndTarget(packageName, targetName); + } + + /** + * A pair of package and target names. Note that having an instance of this does not imply that + * the package or target names are actually valid. + */ + public static class PackageAndTarget { + private final String packageName; + private final String targetName; + + public PackageAndTarget(String packageName, String targetName) { + this.packageName = packageName; + this.targetName = targetName; + } + + public String getPackageName() { + return packageName; + } + + public String getTargetName() { + return targetName; + } + + @Override + public String toString() { + return "//" + packageName + ":" + targetName; + } + + @Override + public int hashCode() { + return Objects.hash(packageName, targetName); + } + + @Override + public boolean equals(Object o) { + if (o == null || o.getClass() != getClass()) { + return false; + } + PackageAndTarget otherTarget = (PackageAndTarget) o; + return Objects.equals(otherTarget.targetName, targetName) + && Objects.equals(otherTarget.packageName, packageName); + } + } + + /** + * An exception to notify the caller that a label could not be parsed. + */ + public static class BadLabelException extends Exception { + public BadLabelException(String msg) { + super(msg); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/ResolvedTargets.java b/src/main/java/com/google/devtools/build/lib/cmdline/ResolvedTargets.java new file mode 100644 index 0000000..806cd61 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/cmdline/ResolvedTargets.java
@@ -0,0 +1,170 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.cmdline; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; + +import java.util.Collection; +import java.util.Set; + +import javax.annotation.concurrent.Immutable; + +/** + * Contains the result of the target pattern evaluation. This is a specialized container class for + * the result of target pattern resolution. There is no restriction on the element type, but it will + * usually be {@code Target}. + */ +@Immutable +public final class ResolvedTargets<T> { + private static final ResolvedTargets<?> FAILED_RESULT = + new ResolvedTargets<>(ImmutableSet.of(), ImmutableSet.of(), true); + + private static final ResolvedTargets<?> EMPTY_RESULT = + new ResolvedTargets<>(ImmutableSet.of(), ImmutableSet.of(), false); + + @SuppressWarnings("unchecked") + public static <T> ResolvedTargets<T> failed() { + return (ResolvedTargets<T>) FAILED_RESULT; + } + + @SuppressWarnings("unchecked") + public static <T> ResolvedTargets<T> empty() { + return (ResolvedTargets<T>) EMPTY_RESULT; + } + + public static <T> ResolvedTargets<T> of(T target) { + return new ResolvedTargets<>(ImmutableSet.<T>of(target), false); + } + + private final boolean hasError; + private final ImmutableSet<T> targets; + private final ImmutableSet<T> filteredTargets; + + public ResolvedTargets(Set<T> targets, Set<T> filteredTargets, boolean hasError) { + this.targets = ImmutableSet.copyOf(targets); + this.filteredTargets = ImmutableSet.copyOf(filteredTargets); + this.hasError = hasError; + } + + public ResolvedTargets(Set<T> targets, boolean hasError) { + this.targets = ImmutableSet.copyOf(targets); + this.filteredTargets = ImmutableSet.of(); + this.hasError = hasError; + } + + public boolean hasError() { + return hasError; + } + + public ImmutableSet<T> getTargets() { + return targets; + } + + public ImmutableSet<T> getFilteredTargets() { + return filteredTargets; + } + + /** + * Returns a builder using concurrent sets, as long as you don't call filter. + */ + public static <T> ResolvedTargets.Builder<T> concurrentBuilder() { + return new ResolvedTargets.Builder<>( + Sets.<T>newConcurrentHashSet(), + Sets.<T>newConcurrentHashSet()); + } + + public static <T> ResolvedTargets.Builder<T> builder() { + return new ResolvedTargets.Builder<>(); + } + + public static final class Builder<T> { + private Set<T> targets; + private Set<T> filteredTargets; + private volatile boolean hasError = false; + + private Builder() { + this(Sets.<T>newLinkedHashSet(), Sets.<T>newLinkedHashSet()); + } + + private Builder(Set<T> targets, Set<T> filteredTargets) { + this.targets = targets; + this.filteredTargets = filteredTargets; + } + + public ResolvedTargets<T> build() { + return new ResolvedTargets<>(targets, filteredTargets, hasError); + } + + public Builder<T> merge(ResolvedTargets<T> other) { + removeAll(other.filteredTargets); + addAll(other.targets); + if (other.hasError) { + hasError = true; + } + return this; + } + + public Builder<T> add(T target) { + targets.add(target); + filteredTargets.remove(target); + return this; + } + + public Builder<T> addAll(Collection<T> targets) { + this.targets.addAll(targets); + this.filteredTargets.removeAll(targets); + return this; + } + + public void remove(T target) { + targets.remove(target); + filteredTargets.add(target); + } + + public Builder<T> removeAll(Collection<T> targets) { + this.filteredTargets.addAll(targets); + this.targets.removeAll(targets); + return this; + } + + public Builder<T> filter(Predicate<T> predicate) { + Set<T> oldTargets = targets; + targets = Sets.newLinkedHashSet(); + for (T target : oldTargets) { + if (predicate.apply(target)) { + add(target); + } else { + remove(target); + } + } + return this; + } + + public Builder<T> setError() { + this.hasError = true; + return this; + } + + public Builder<T> mergeError(boolean hasError) { + this.hasError |= hasError; + return this; + } + + public boolean isEmpty() { + return targets.isEmpty(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/TargetParsingException.java b/src/main/java/com/google/devtools/build/lib/cmdline/TargetParsingException.java new file mode 100644 index 0000000..044bcca --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/cmdline/TargetParsingException.java
@@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.cmdline; + +import com.google.common.base.Preconditions; + +/** + * Indicates that a target label cannot be parsed. + */ +public class TargetParsingException extends Exception { + public TargetParsingException(String message) { + super(Preconditions.checkNotNull(message)); + } + + public TargetParsingException(String message, Throwable cause) { + super(Preconditions.checkNotNull(message), cause); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/TargetPattern.java b/src/main/java/com/google/devtools/build/lib/cmdline/TargetPattern.java new file mode 100644 index 0000000..6677970 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/cmdline/TargetPattern.java
@@ -0,0 +1,464 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.cmdline; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.cmdline.LabelValidator.BadLabelException; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import javax.annotation.concurrent.Immutable; + +/** + * Represents a target pattern. Target patterns are a generalization of labels to include + * wildcards for finding all packages recursively beneath some root, and for finding all targets + * within a package. + * + * <p>Note that this class does not handle negative patterns ("-//foo/bar"); these must be handled + * one level up. In particular, the query language comes with built-in support for negative + * patterns. + * + * <p>In order to resolve target patterns, you need an implementation of {@link + * TargetPatternResolver}. This class is thread-safe if the corresponding instance is thread-safe. + * + * <p>See lib/blaze/commands/target-syntax.txt for details. + */ +public abstract class TargetPattern { + + private static final Splitter SLASH_SPLITTER = Splitter.on('/'); + private static final Joiner SLASH_JOINER = Joiner.on('/'); + + private static final Parser DEFAULT_PARSER = new Parser(""); + + private final Type type; + + /** + * Returns a parser with no offset. Note that the Parser class is immutable, so this method may + * return the same instance on subsequent calls. + */ + public static Parser defaultParser() { + return DEFAULT_PARSER; + } + + private static String removeSuffix(String s, String suffix) { + if (s.endsWith(suffix)) { + return s.substring(0, s.length() - suffix.length()); + } else { + throw new IllegalArgumentException(s + ", " + suffix); + } + } + + /** + * Normalizes the given relative path by resolving {@code //}, {@code /./} and {@code x/../} + * pieces. Note that leading {@code ".."} segments are not removed, so the returned string can + * have leading {@code ".."} segments. + * + * @throws IllegalArgumentException if the path is absolute, i.e. starts with a @{code '/'} + */ + @VisibleForTesting + static String normalize(String path) { + Preconditions.checkArgument(!path.startsWith("/")); + Iterator<String> it = SLASH_SPLITTER.split(path).iterator(); + List<String> pieces = new ArrayList<>(); + while (it.hasNext()) { + String piece = it.next(); + if (".".equals(piece) || piece.isEmpty()) { + continue; + } + if ("..".equals(piece)) { + if (pieces.isEmpty()) { + pieces.add(piece); + continue; + } + String predecessor = pieces.remove(pieces.size() - 1); + if ("..".equals(predecessor)) { + pieces.add(piece); + pieces.add(piece); + } + continue; + } + pieces.add(piece); + } + return SLASH_JOINER.join(pieces); + } + + private TargetPattern(Type type) { + // Don't allow inheritance outside this class. + this.type = type; + } + + /** + * Return the type of the pattern. Examples include "below package" like "foo/..." and "single + * target" like "//x:y". + */ + public Type getType() { + return type; + } + + /** + * Evaluates the current target pattern and returns the result. + */ + public abstract <T> ResolvedTargets<T> eval(TargetPatternResolver<T> resolver) + throws TargetParsingException, InterruptedException, + TargetPatternResolver.MissingDepException; + + private static final class SingleTarget extends TargetPattern { + + private final String targetName; + + private SingleTarget(String targetName) { + super(Type.SINGLE_TARGET); + this.targetName = targetName; + } + + @Override + public <T> ResolvedTargets<T> eval(TargetPatternResolver<T> resolver) + throws TargetParsingException, InterruptedException, + TargetPatternResolver.MissingDepException { + return resolver.getExplicitTarget(targetName); + } + } + + private static final class InterpretPathAsTarget extends TargetPattern { + + private final String path; + + private InterpretPathAsTarget(String path) { + super(Type.PATH_AS_TARGET); + this.path = normalize(path); + } + + @Override + public <T> ResolvedTargets<T> eval(TargetPatternResolver<T> resolver) + throws TargetParsingException, InterruptedException, + TargetPatternResolver.MissingDepException { + if (resolver.isPackage(path)) { + // User has specified a package name. Issue a helpful error message. + throw new TargetParsingException("ambiguous target pattern: '" + path + "' is " + + "the name of a package; use '" + path + ":all' to mean \"all " + + "rules in this package\", '" + path + "/...' to mean \"all rules recursively " + + "beneath this package\", or '//" + path + "' to mean \"the default rule in this " + + "package\""); + } + + List<String> pieces = SLASH_SPLITTER.splitToList(path); + + // Interprets the label as a file target. This loop stops as soon as the + // first BUILD file is found (i.e. longest prefix match). + for (int i = pieces.size() - 1; i > 0; i--) { + String packageName = SLASH_JOINER.join(pieces.subList(0, i)); + if (resolver.isPackage(packageName)) { + String targetName = SLASH_JOINER.join(pieces.subList(i, pieces.size())); + return resolver.getExplicitTarget("//" + packageName + ":" + targetName); + } + } + + throw new TargetParsingException( + "couldn't determine target from filename '" + path + "'"); + } + } + + private static final class TargetsInPackage extends TargetPattern { + + private final String originalPattern; + private final String pattern; + private final String suffix; + private final boolean isAbsolute; + private final boolean rulesOnly; + private final boolean checkWildcardConflict; + + private TargetsInPackage(String originalPattern, String pattern, String suffix, + boolean isAbsolute, boolean rulesOnly, boolean checkWildcardConflict) { + super(Type.TARGETS_IN_PACKAGE); + this.originalPattern = originalPattern; + this.pattern = pattern; + this.suffix = suffix; + this.isAbsolute = isAbsolute; + this.rulesOnly = rulesOnly; + this.checkWildcardConflict = checkWildcardConflict; + } + + @Override + public <T> ResolvedTargets<T> eval(TargetPatternResolver<T> resolver) + throws TargetParsingException, InterruptedException, + TargetPatternResolver.MissingDepException { + if (checkWildcardConflict) { + ResolvedTargets<T> targets = getWildcardConflict(resolver); + if (targets != null) { + return targets; + } + } + return resolver.getTargetsInPackage(originalPattern, removeSuffix(pattern, suffix), + rulesOnly); + } + + /** + * There's a potential ambiguity if '//foo/bar:all' refers to an actual target. In this case, we + * use the the target but print a warning. + * + * @return the Target corresponding to the given pattern, if the pattern is absolute and there + * is such a target. Otherwise, return null. + */ + private <T> ResolvedTargets<T> getWildcardConflict(TargetPatternResolver<T> resolver) + throws InterruptedException, TargetPatternResolver.MissingDepException { + if (!isAbsolute) { + return null; + } + + T target = resolver.getTargetOrNull("//" + pattern); + if (target != null) { + String name = pattern.lastIndexOf(':') != -1 + ? pattern.substring(pattern.lastIndexOf(':') + 1) + : pattern.substring(pattern.lastIndexOf('/') + 1); + resolver.warn(String.format("The Blaze target pattern '%s' is ambiguous: '%s' is " + + "both a wildcard, and the name of an existing %s; " + + "using the latter interpretation", + "//" + pattern, ":" + name, + resolver.getTargetKind(target))); + try { + return resolver.getExplicitTarget("//" + pattern); + } catch (TargetParsingException e) { + throw new IllegalStateException( + "getTargetOrNull() returned non-null, so target should exist", e); + } + } + return null; + } + } + + private static final class TargetsBelowPackage extends TargetPattern { + + private final String originalPattern; + private final String pathPrefix; + private final boolean rulesOnly; + + private TargetsBelowPackage(String originalPattern, String pathPrefix, boolean rulesOnly) { + super(Type.TARGETS_BELOW_PACKAGE); + this.originalPattern = originalPattern; + this.pathPrefix = pathPrefix; + this.rulesOnly = rulesOnly; + } + + @Override + public <T> ResolvedTargets<T> eval(TargetPatternResolver<T> resolver) + throws TargetParsingException, InterruptedException, + TargetPatternResolver.MissingDepException { + return resolver.findTargetsBeneathDirectory(originalPattern, pathPrefix, rulesOnly); + } + } + + @Immutable + public static final class Parser { + // TODO(bazel-team): Merge the Label functionality that requires similar constants into this + // class. + /** + * The set of target-pattern suffixes which indicate wildcards over all <em>rules</em> in a + * single package. + */ + private static final List<String> ALL_RULES_IN_SUFFIXES = ImmutableList.of( + "all"); + + /** + * The set of target-pattern suffixes which indicate wildcards over all <em>targets</em> in a + * single package. + */ + private static final List<String> ALL_TARGETS_IN_SUFFIXES = ImmutableList.of( + "*", + "all-targets"); + + private static final List<String> SUFFIXES; + + static { + SUFFIXES = ImmutableList.<String>builder() + .addAll(ALL_RULES_IN_SUFFIXES) + .addAll(ALL_TARGETS_IN_SUFFIXES) + .add("/...") + .build(); + } + + /** + * Returns whether the given pattern is simple, i.e., not starting with '-' and using none of + * the target matching suffixes. + */ + public static boolean isSimpleTargetPattern(String pattern) { + if (pattern.startsWith("-")) { + return false; + } + + for (String suffix : SUFFIXES) { + if (pattern.endsWith(":" + suffix)) { + return false; + } + } + return true; + } + + /** + * Directory prefix to use when resolving relative labels (rather than absolute ones). For + * example, if the working directory is "<workspace root>/foo", then this should be "foo", + * which will make patterns such as "bar:bar" be resolved as "//foo/bar:bar". This makes the + * command line a bit more convenient to use. + */ + private final String relativeDirectory; + + /** + * Creates a new parser with the given offset for relative patterns. + */ + public Parser(String relativeDirectory) { + this.relativeDirectory = relativeDirectory; + } + + /** + * Parses the given pattern, and throws an exception if the pattern is invalid. + * + * @return a target pattern corresponding to the pattern parsed + * @throws TargetParsingException if the pattern is invalid + */ + public TargetPattern parse(String pattern) throws TargetParsingException { + // The structure of this method is by cases, according to the usage string + // constant (see lib/blaze/commands/target-syntax.txt). + + String originalPattern = pattern; + final boolean isAbsolute = pattern.startsWith("//"); + + // We now absolutize non-absolute target patterns. + pattern = isAbsolute ? pattern.substring(2) : absolutize(pattern); + // Check for common errors. + if (pattern.startsWith("/")) { + throw new TargetParsingException("not a relative path or label: '" + pattern + "'"); + } + if (pattern.isEmpty()) { + throw new TargetParsingException("the empty string is not a valid target"); + } + + // Transform "/BUILD" suffix into ":BUILD" to accept //foo/bar/BUILD + // syntax as a synonym to //foo/bar:BUILD. + if (pattern.endsWith("/BUILD")) { + pattern = pattern.substring(0, pattern.length() - 6) + ":BUILD"; + } + + int colonIndex = pattern.lastIndexOf(':'); + String packagePart = colonIndex < 0 ? pattern : pattern.substring(0, colonIndex); + String targetPart = colonIndex < 0 ? "" : pattern.substring(colonIndex + 1); + + if (packagePart.equals("...")) { + packagePart = "/..."; // special case this for easier parsing + } + + if (packagePart.endsWith("/")) { + throw new TargetParsingException("The package part of '" + originalPattern + + "' should not end in a slash"); + } + + if (packagePart.endsWith("/...")) { + String realPackagePart = removeSuffix(packagePart, "/..."); + if (targetPart.isEmpty() || ALL_RULES_IN_SUFFIXES.contains(targetPart)) { + return new TargetsBelowPackage(originalPattern, realPackagePart, true); + } else if (ALL_TARGETS_IN_SUFFIXES.contains(targetPart)) { + return new TargetsBelowPackage(originalPattern, realPackagePart, false); + } + } + + if (ALL_RULES_IN_SUFFIXES.contains(targetPart)) { + return new TargetsInPackage( + originalPattern, pattern, ":" + targetPart, isAbsolute, true, true); + } + + if (ALL_TARGETS_IN_SUFFIXES.contains(targetPart)) { + return new TargetsInPackage( + originalPattern, pattern, ":" + targetPart, isAbsolute, false, true); + } + + + if (isAbsolute || pattern.contains(":")) { + String fullLabel = "//" + pattern; + try { + LabelValidator.validateAbsoluteLabel(fullLabel); + } catch (BadLabelException e) { + String error = "invalid target format '" + originalPattern + "': " + e.getMessage(); + throw new TargetParsingException(error); + } + return new SingleTarget(fullLabel); + } + + // This is a stripped-down version of interpretPathAsTarget that does no I/O. We have a basic + // relative path. e.g. "foo/bar/Wiz.java". The strictest correct check we can do here (without + // I/O) is just to ensure that there is *some* prefix that is a valid package-name. It's + // sufficient to test the first segment. This is really a rather weak check; perhaps we should + // just eliminate it. + int slashIndex = pattern.indexOf('/'); + if (slashIndex < 0) { + throw new TargetParsingException("ambiguous target pattern: '" + pattern + "' could " + + "potentially be the name of a package; use '" + pattern + ":all' to mean \"all " + + "rules in this package\", '" + pattern + "/...' to mean \"all rules recursively " + + "beneath this package\", or '//" + pattern + "' to mean \"the default rule in this " + + "package\""); + } + String errorMessage = LabelValidator.validatePackageName(pattern.substring(0, slashIndex)); + if (errorMessage != null) { + throw new TargetParsingException("Bad target pattern '" + originalPattern + "': " + + errorMessage); + } + return new InterpretPathAsTarget(pattern); + } + + /** + * Absolutizes the target pattern to the offset. + * Patterns starting with "/" are absolute and not modified. + * + * If the offset is "foo": + * absolutize(":bar") --> "foo:bar" + * absolutize("bar") --> "foo/bar" + * absolutize("/biz/bar") --> "biz/bar" (absolute) + * absolutize("biz:bar") --> "foo/biz:bar" + * + * @param pattern The target pattern to parse. + * @return the pattern, absolutized to the offset if approprate. + */ + private String absolutize(String pattern) { + if (relativeDirectory.isEmpty() || pattern.startsWith("/")) { + return pattern; + } + + // It seems natural to use {@link PathFragment#getRelative()} here, + // but it doesn't work when the pattern starts with ":". + // "foo".getRelative(":all") would return "foo/:all", where we + // really want "foo:all". + return pattern.startsWith(":") + ? relativeDirectory + pattern + : relativeDirectory + "/" + pattern; + } + } + + /** + * The target pattern type (targets below package, in package, explicit target, etc.) + */ + public enum Type { + /** A path interpreted as a target, eg "foo/bar/baz" */ + PATH_AS_TARGET, + /** An explicit target, eg "//foo:bar." */ + SINGLE_TARGET, + /** Targets below a package, eg "foo/...". */ + TARGETS_BELOW_PACKAGE, + /** Target in a package, eg "foo:all". */ + TARGETS_IN_PACKAGE; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/TargetPatternResolver.java b/src/main/java/com/google/devtools/build/lib/cmdline/TargetPatternResolver.java new file mode 100644 index 0000000..109179f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/cmdline/TargetPatternResolver.java
@@ -0,0 +1,94 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.cmdline; + +/** + * A callback interface that is used during the process of converting target patterns (such as + * <code>//foo:all</code>) into one or more lists of targets (such as <code>//foo:foo, + * //foo:bar</code>). During a call to {@link TargetPattern#eval}, the {@link TargetPattern} makes + * calls to this interface to implement the target pattern semantics. The generic type {@code T} is + * only for compile-time type safety; there are no requirements to the actual type. + */ +public interface TargetPatternResolver<T> { + + /** + * Reports the given warning. + */ + void warn(String msg); + + /** + * Returns a single target corresponding to the given name, or null. This method may only throw an + * exception if the current thread was interrupted. + */ + T getTargetOrNull(String targetName) throws InterruptedException, MissingDepException; + + /** + * Returns a single target corresponding to the given name, or an empty or failed result. + */ + ResolvedTargets<T> getExplicitTarget(String targetName) + throws TargetParsingException, InterruptedException, MissingDepException; + + /** + * Returns the set containing the targets found in the given package. The specified directory is + * not necessarily a valid package name. If {@code rulesOnly} is true, then this method should + * only return rules in the given package. + * + * @param originalPattern the original target pattern for error reporting purposes + * @param packageName the name of the package + * @param rulesOnly whether to return rules only + */ + ResolvedTargets<T> getTargetsInPackage(String originalPattern, String packageName, + boolean rulesOnly) throws TargetParsingException, InterruptedException, MissingDepException; + + /** + * Returns the set containing the targets found below the given {@code pathPrefix}. Conceptually, + * this method should look for all packages that start with the {@code pathPrefix} (as a proper + * prefix directory, i.e., "foo/ba" is not a proper prefix of "foo/bar/"), and then collect all + * targets in each such package (subject to {@code rulesOnly}) as if calling {@link + * #getTargetsInPackage}. The specified directory is not necessarily a valid package name. + * + * <p>Note that the {@code pathPrefix} can be empty, which corresponds to the "//..." pattern. + * Implementations may choose not to support this case and throw an exception instead, or may + * restrict the set of directories that are considered by default. + * + * <p>If the {@code pathPrefix} points to a package, then that package should also be part of the + * result. + * + * @param originalPattern the original target pattern for error reporting purposes + * @param pathPrefix the directory in which to look for packages + * @param rulesOnly whether to return rules only + */ + ResolvedTargets<T> findTargetsBeneathDirectory(String originalPattern, String pathPrefix, + boolean rulesOnly) throws TargetParsingException, InterruptedException, MissingDepException; + + /** + * Returns true, if and only if the given name corresponds to a package, i.e., a file with the + * name {@code packageName/BUILD} exists. + */ + boolean isPackage(String packageName) throws MissingDepException; + + /** + * Returns the target kind of the given target, for example {@code cc_library rule}. + */ + String getTargetKind(T target); + + /** + * A missing dependency is needed before target parsing can proceed. Currently used only in + * skyframe to notify the framework of missing dependencies. + */ + // TODO(bazel-team): Avoid this use of exception for expected control flow management. + public class MissingDepException extends Exception { + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/CollectionUtils.java b/src/main/java/com/google/devtools/build/lib/collect/CollectionUtils.java new file mode 100644 index 0000000..d7b04bb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/CollectionUtils.java
@@ -0,0 +1,225 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Utilities for collection classes. + */ +public final class CollectionUtils { + + private CollectionUtils() {} + + /** + * Given a collection of elements and an equivalence relation, returns a new + * unordered collection of the disjoint subsets of those elements which are + * equivalent under the specified relation. + * + * <p>Note: the Comparator needs only to implement the less-strict contract + * of EquivalenceRelation (q.v.). (Hopefully this will one day be a + * superinterface of Comparator.) + * + * @param elements the collection of elements to be partitioned. May + * contain duplicates. + * @param equivalenceRelation an equivalence relation over the elements. + * @return a collection of sets of elements that are equivalent under the + * specified relation. + */ + public static <T> Collection<Set<T>> partition(Collection<T> elements, + Comparator<T> equivalenceRelation) { + // TODO(bazel-team): (2009) O(n*m) where n=|elements| and m=|eqClasses|; i.e., + // quadratic. Use Tarjan's algorithm instead. + List<Set<T>> eqClasses = new ArrayList<>(); + for (T element : elements) { + boolean found = false; + for (Set<T> eqClass : eqClasses) { + if (equivalenceRelation.compare(eqClass.iterator().next(), + element) == 0) { + eqClass.add(element); + found = true; + break; + } + } + if (!found) { + Set<T> eqClass = new HashSet<>(); + eqClass.add(element); + eqClasses.add(eqClass); + } + } + return eqClasses; + } + + /** + * See partition(Collection, Comparator). + */ + public static <T> Collection<Set<T>> partition(Collection<T> elements, + final EquivalenceRelation<T> equivalenceRelation) { + return partition(elements, new Comparator<T>() { + @Override + public int compare(T o1, T o2) { + return equivalenceRelation.compare(o1, o2); + } + }); + } + + /** + * Returns the set of all elements in the given collection that appear more than once. + * @param input some collection. + * @return the set of repeated elements. May return an empty set, but never null. + */ + public static <T> Set<T> duplicatedElementsOf(Collection<T> input) { + Set<T> duplicates = new HashSet<>(); + Set<T> elementSet = new HashSet<>(); + for (T el : input) { + if (!elementSet.add(el)) { + duplicates.add(el); + } + } + return duplicates; + } + + /** + * Returns an immutable list of all non-null parameters in the order in which + * they are specified. + */ + @SuppressWarnings("unchecked") + public static <T> ImmutableList<T> asListWithoutNulls(T... elements) { + ImmutableList.Builder<T> builder = ImmutableList.builder(); + for (T element : elements) { + if (element != null) { + builder.add(element); + } + } + return builder.build(); + } + + /** + * Returns true if the given iterable can be verified to be immutable. + * + * <p>Note that if this method returns false, that does not mean that the iterable is mutable. + */ + public static <T> boolean isImmutable(Iterable<T> iterable) { + return iterable instanceof ImmutableList<?> + || iterable instanceof ImmutableSet<?> + || iterable instanceof IterablesChain<?> + || iterable instanceof NestedSet<?> + || iterable instanceof ImmutableIterable<?>; + } + + /** + * Throws a runtime exception if the given iterable can not be verified to be immutable. + */ + public static <T> void checkImmutable(Iterable<T> iterable) { + Preconditions.checkState(isImmutable(iterable), iterable.getClass()); + } + + /** + * Given an iterable, returns an immutable iterable with the same contents. + */ + public static <T> Iterable<T> makeImmutable(Iterable<T> iterable) { + if (isImmutable(iterable)) { + return iterable; + } else { + return ImmutableList.copyOf(iterable); + } + } + + /** + * Converts a set of enum values to a bit field. Requires that the enum contains at most 32 + * elements. + */ + public static <T extends Enum<T>> int toBits(Set<T> values) { + int result = 0; + for (T value : values) { + // <p>Note that when the 32. bit is set, the integer becomes negative (because that is the + // sign bit). This does not affect the function of the bitwise operators, so it is fine. + Preconditions.checkArgument(value.ordinal() < 32); + result |= (1 << value.ordinal()); + } + + return result; + } + + /** + * Converts a set of enum values to a bit field. Requires that the enum contains at most 32 + * elements. + */ + @SuppressWarnings("unchecked") + public static <T extends Enum<T>> int toBits(T... values) { + return toBits(ImmutableSet.copyOf(values)); + } + + /** + * Converts a bit field to a set of enum values. Requires that the enum contains at most 32 + * elements. + */ + public static <T extends Enum<T>> EnumSet<T> fromBits(int value, Class<T> clazz) { + T[] elements = clazz.getEnumConstants(); + Preconditions.checkArgument(elements.length <= 32); + ArrayList<T> result = new ArrayList<>(); + for (T element : elements) { + if ((value & (1 << element.ordinal())) != 0) { + result.add(element); + } + } + + return result.isEmpty() ? EnumSet.noneOf(clazz) : EnumSet.copyOf(result); + } + + /** + * Returns whether an {@link Iterable} is a superset of another one. + */ + public static <T> boolean containsAll(Iterable<T> superset, Iterable<T> subset) { + return ImmutableSet.copyOf(superset).containsAll(ImmutableList.copyOf(subset)); + } + + /** + * Returns an ImmutableMap of ImmutableMaps created from the Map of Maps parameter. + */ + public static <KEY_1, KEY_2, VALUE> ImmutableMap<KEY_1, ImmutableMap<KEY_2, VALUE>> toImmutable( + Map<KEY_1, Map<KEY_2, VALUE>> map) { + ImmutableMap.Builder<KEY_1, ImmutableMap<KEY_2, VALUE>> builder = ImmutableMap.builder(); + for (Map.Entry<KEY_1, Map<KEY_2, VALUE>> entry : map.entrySet()) { + builder.put(entry.getKey(), ImmutableMap.copyOf(entry.getValue())); + } + return builder.build(); + } + + /** + * Returns a copy of the Map of Maps parameter. + */ + public static <KEY_1, KEY_2, VALUE> Map<KEY_1, Map<KEY_2, VALUE>> copyOf( + Map<KEY_1, ? extends Map<KEY_2, VALUE>> map) { + Map<KEY_1, Map<KEY_2, VALUE>> result = new HashMap<>(); + for (Map.Entry<KEY_1, ? extends Map<KEY_2, VALUE>> entry : map.entrySet()) { + result.put(entry.getKey(), new HashMap<>(entry.getValue())); + } + return result; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/CompactHashSet.java b/src/main/java/com/google/devtools/build/lib/collect/CompactHashSet.java new file mode 100644 index 0000000..5ee2417 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/CompactHashSet.java
@@ -0,0 +1,604 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/* + * Copyright (C) 2012 The Guava Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.build.lib.collect; + +import com.google.common.base.Preconditions; +import com.google.common.primitives.Ints; + +import java.io.IOException; +import java.io.InvalidObjectException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.lang.reflect.Array; +import java.util.AbstractSet; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.ConcurrentModificationException; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * CompactHashSet is an implementation of a Set. All optional operations (adding and + * removing) are supported. The elements can be any objects. + * + * <p>{@code contains(x)}, {@code add(x)} and {@code remove(x)}, are all (expected and amortized) + * constant time operations. Expected in the hashtable sense (depends on the hash function + * doing a good job of distributing the elements to the buckets to a distribution not far from + * uniform), and amortized since some operations can trigger a hash table resize. + * + * <p>Unlike {@code java.util.HashSet}, iteration is only proportional to the actual + * {@code size()}, which is optimal, and <i>not</i> the size of the internal hashtable, + * which could be much larger than {@code size()}. Furthermore, this structure only depends + * on a fixed number of arrays; {@code add(x)} operations <i>do not</i> create objects + * for the garbage collector to deal with, and for every element added, the garbage collector + * will have to traverse {@code 1.5} references on average, in the marking phase, not {@code 5.0} + * as in {@code java.util.HashSet}. + * + * <p>If there are no removals, then {@link #iterator iteration} order is the same as insertion + * order. Any removal invalidates any ordering guarantees. + */ +// TODO(bazel-team): This was branched of an internal version of guava. If the class is released, we +// should remove this again. +public class CompactHashSet<E> extends AbstractSet<E> implements Serializable { + // TODO(bazel-team): cache all field accesses in local vars + + // A partial copy of com.google.common.collect.Hashing. + private static final int C1 = 0xcc9e2d51; + private static final int C2 = 0x1b873593; + + /* + * This method was rewritten in Java from an intermediate step of the Murmur hash function in + * http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp, which contained the + * following header: + * + * MurmurHash3 was written by Austin Appleby, and is placed in the public domain. The author + * hereby disclaims copyright to this source code. + */ + private static int smear(int hashCode) { + return C2 * Integer.rotateLeft(hashCode * C1, 15); + } + + private static int smearedHash(@Nullable Object o) { + return smear((o == null) ? 0 : o.hashCode()); + } + + private static final int MAX_TABLE_SIZE = Ints.MAX_POWER_OF_TWO; + + private static int closedTableSize(int expectedEntries, double loadFactor) { + // Get the recommended table size. + // Round down to the nearest power of 2. + expectedEntries = Math.max(expectedEntries, 2); + int tableSize = Integer.highestOneBit(expectedEntries); + // Check to make sure that we will not exceed the maximum load factor. + if (expectedEntries > (int) (loadFactor * tableSize)) { + tableSize <<= 1; + return (tableSize > 0) ? tableSize : MAX_TABLE_SIZE; + } + return tableSize; + } + + /** + * Creates an empty {@code CompactHashSet} instance. + */ + public static <E> CompactHashSet<E> create() { + return new CompactHashSet<E>(); + } + + /** + * Creates a <i>mutable</i> {@code CompactHashSet} instance containing the elements + * of the given collection in unspecified order. + * + * @param collection the elements that the set should contain + * @return a new {@code CompactHashSet} containing those elements (minus duplicates) + */ + public static <E> CompactHashSet<E> create(Collection<? extends E> collection) { + CompactHashSet<E> set = createWithExpectedSize(collection.size()); + set.addAll(collection); + return set; + } + + /** + * Creates a <i>mutable</i> {@code CompactHashSet} instance containing the given + * elements in unspecified order. + * + * @param elements the elements that the set should contain + * @return a new {@code CompactHashSet} containing those elements (minus duplicates) + */ + @SafeVarargs + public static <E> CompactHashSet<E> create(E... elements) { + CompactHashSet<E> set = createWithExpectedSize(elements.length); + Collections.addAll(set, elements); + return set; + } + + /** + * Creates a {@code CompactHashSet} instance, with a high enough "initial capacity" + * that it <i>should</i> hold {@code expectedSize} elements without growth. + * + * @param expectedSize the number of elements you expect to add to the returned set + * @return a new, empty {@code CompactHashSet} with enough capacity to hold {@code + * expectedSize} elements without resizing + * @throws IllegalArgumentException if {@code expectedSize} is negative + */ + public static <E> CompactHashSet<E> createWithExpectedSize(int expectedSize) { + return new CompactHashSet<E>(expectedSize); + } + + private static final int MAXIMUM_CAPACITY = 1 << 30; + + // TODO(bazel-team): decide, and inline, load factor. 0.75? + private static final float DEFAULT_LOAD_FACTOR = 1.0f; + + /** + * Bitmask that selects the low 32 bits. + */ + private static final long NEXT_MASK = (1L << 32) - 1; + + /** + * Bitmask that selects the high 32 bits. + */ + private static final long HASH_MASK = ~NEXT_MASK; + + // TODO(bazel-team): decide default size + private static final int DEFAULT_SIZE = 3; + + static final int UNSET = -1; + + /** + * The hashtable. Its values are indexes to both the elements and entries arrays. + * + * Currently, the UNSET value means "null pointer", and any non negative value x is + * the actual index. + * + * Its size must be a power of two. + */ + private transient int[] table; + + /** + * Contains the logical entries, in the range of [0, size()). The high 32 bits of each + * long is the smeared hash of the element, whereas the low 32 bits is the "next" pointer + * (pointing to the next entry in the bucket chain). The pointers in [size(), entries.length) + * are all "null" (UNSET). + */ + private transient long[] entries; + + /** + * The elements contained in the set, in the range of [0, size()). + */ + transient Object[] elements; + + /** + * The load factor. + */ + transient float loadFactor; + + /** + * Keeps track of modifications of this set, to make it possible to throw + * ConcurrentModificationException in the iterator. Note that we choose not to + * make this volatile, so we do less of a "best effort" to track such errors, + * for better performance. + */ + transient int modCount; + + /** + * When we have this many elements, resize the hashtable. + */ + private transient int threshold; + + /** + * The number of elements contained in the set. + */ + private transient int size; + + /** + * Constructs a new empty instance of {@code CompactHashSet}. + */ + CompactHashSet() { + init(DEFAULT_SIZE, DEFAULT_LOAD_FACTOR); + } + + /** + * Constructs a new instance of {@code CompactHashSet} with the specified capacity. + * + * @param expectedSize the initial capacity of this {@code CompactHashSet}. + */ + CompactHashSet(int expectedSize) { + init(expectedSize, DEFAULT_LOAD_FACTOR); + } + + /** + * Pseudoconstructor for serialization support. + */ + void init(int expectedSize, float loadFactor) { + Preconditions.checkArgument(expectedSize >= 0, "Initial capacity must be non-negative"); + Preconditions.checkArgument(loadFactor > 0, "Illegal load factor"); + int buckets = closedTableSize(expectedSize, loadFactor); + this.table = newTable(buckets); + this.loadFactor = loadFactor; + this.elements = new Object[expectedSize]; + this.entries = newEntries(expectedSize); + this.threshold = Math.max(1, (int) (buckets * loadFactor)); + } + + private static int[] newTable(int size) { + int[] array = new int[size]; + Arrays.fill(array, UNSET); + return array; + } + + private static long[] newEntries(int size) { + long[] array = new long[size]; + Arrays.fill(array, UNSET); + return array; + } + + private static int getHash(long entry) { + return (int) (entry >>> 32); + } + + /** + * Returns the index, or UNSET if the pointer is "null" + */ + private static int getNext(long entry) { + return (int) entry; + } + + /** + * Returns a new entry value by changing the "next" index of an existing entry + */ + private static long swapNext(long entry, int newNext) { + return (HASH_MASK & entry) | (NEXT_MASK & newNext); + } + + private int hashTableMask() { + return table.length - 1; + } + + @Override + public boolean add(@Nullable E object) { + long[] entries = this.entries; + Object[] elements = this.elements; + int hash = smearedHash(object); + int tableIndex = hash & hashTableMask(); + int newEntryIndex = this.size; // current size, and pointer to the entry to be appended + int next = table[tableIndex]; + if (next == UNSET) { // uninitialized bucket + table[tableIndex] = newEntryIndex; + } else { + int last; + long entry; + do { + last = next; + entry = entries[next]; + if (getHash(entry) == hash && Objects.equals(object, elements[next])) { + return false; + } + next = getNext(entry); + } while (next != UNSET); + entries[last] = swapNext(entry, newEntryIndex); + } + if (newEntryIndex == Integer.MAX_VALUE) { + throw new IllegalStateException("Cannot contain more than Integer.MAX_VALUE elements!"); + } + int newSize = newEntryIndex + 1; + resizeMeMaybe(newSize); + insertEntry(newEntryIndex, object, hash); + this.size = newSize; + if (newEntryIndex >= threshold) { + resizeTable(2 * table.length); + } + modCount++; + return true; + } + + /** + * Creates a fresh entry with the specified object at the specified position in the entry + * arrays. + */ + void insertEntry(int entryIndex, E object, int hash) { + this.entries[entryIndex] = ((long) hash << 32) | (NEXT_MASK & UNSET); + this.elements[entryIndex] = object; + } + + /** + * Returns currentSize + 1, after resizing the entries storage if necessary. + */ + private void resizeMeMaybe(int newSize) { + int entriesSize = entries.length; + if (newSize > entriesSize) { + int newCapacity = entriesSize + Math.max(1, entriesSize >>> 1); + if (newCapacity < 0) { + newCapacity = Integer.MAX_VALUE; + } + if (newCapacity != entriesSize) { + resizeEntries(newCapacity); + } + } + } + + /** + * Resizes the internal entries array to the specified capacity, which may be greater or less + * than the current capacity. + */ + void resizeEntries(int newCapacity) { + this.elements = Arrays.copyOf(elements, newCapacity); + long[] entries = this.entries; + int oldSize = entries.length; + entries = Arrays.copyOf(entries, newCapacity); + if (newCapacity > oldSize) { + Arrays.fill(entries, oldSize, newCapacity, UNSET); + } + this.entries = entries; + } + + private void resizeTable(int newCapacity) { // newCapacity always a power of two + int[] oldTable = table; + int oldCapacity = oldTable.length; + if (oldCapacity >= MAXIMUM_CAPACITY) { + threshold = Integer.MAX_VALUE; + return; + } + int newThreshold = 1 + (int) (newCapacity * loadFactor); + int[] newTable = newTable(newCapacity); + long[] entries = this.entries; + + int mask = newTable.length - 1; + for (int i = 0; i < size; i++) { + long oldEntry = entries[i]; + int hash = getHash(oldEntry); + int tableIndex = hash & mask; + int next = newTable[tableIndex]; + newTable[tableIndex] = i; + entries[i] = ((long) hash << 32) | (NEXT_MASK & next); + } + + this.threshold = newThreshold; + this.table = newTable; + } + + @Override + public boolean contains(@Nullable Object object) { + int hash = smearedHash(object); + int next = table[hash & hashTableMask()]; + while (next != UNSET) { + long entry = entries[next]; + if (getHash(entry) == hash && Objects.equals(object, elements[next])) { + return true; + } + next = getNext(entry); + } + return false; + } + + @Override + public boolean remove(@Nullable Object object) { + return remove(object, smearedHash(object)); + } + + private boolean remove(Object object, int hash) { + int tableIndex = hash & hashTableMask(); + int next = table[tableIndex]; + if (next == UNSET) { + return false; + } + int last = UNSET; + do { + if (getHash(entries[next]) == hash && Objects.equals(object, elements[next])) { + if (last == UNSET) { + // we need to update the root link from table[] + table[tableIndex] = getNext(entries[next]); + } else { + // we need to update the link from the chain + entries[last] = swapNext(entries[last], getNext(entries[next])); + } + + moveEntry(next); + size--; + modCount++; + return true; + } + last = next; + next = getNext(entries[next]); + } while (next != UNSET); + return false; + } + + /** + * Moves the last entry in the entry array into {@code dstIndex}, and nulls out its old position. + */ + void moveEntry(int dstIndex) { + int srcIndex = size() - 1; + if (dstIndex < srcIndex) { + // move last entry to deleted spot + elements[dstIndex] = elements[srcIndex]; + elements[srcIndex] = null; + + // move the last entry to the removed spot, just like we moved the element + long lastEntry = entries[srcIndex]; + entries[dstIndex] = lastEntry; + entries[srcIndex] = UNSET; + + // also need to update whoever's "next" pointer was pointing to the last entry place + // reusing "tableIndex" and "next"; these variables were no longer needed + int tableIndex = getHash(lastEntry) & hashTableMask(); + int lastNext = table[tableIndex]; + if (lastNext == srcIndex) { + // we need to update the root pointer + table[tableIndex] = dstIndex; + } else { + // we need to update a pointer in an entry + int previous; + long entry; + do { + previous = lastNext; + lastNext = getNext(entry = entries[lastNext]); + } while (lastNext != srcIndex); + // here, entries[previous] points to the old entry location; update it + entries[previous] = swapNext(entry, dstIndex); + } + } else { + elements[dstIndex] = null; + entries[dstIndex] = UNSET; + } + } + + @Override + public Iterator<E> iterator() { + return new Iterator<E>() { + int expectedModCount = modCount; + boolean nextCalled = false; + int index = 0; + + @Override + public boolean hasNext() { + return index < size; + } + + @Override + @SuppressWarnings("unchecked") + public E next() { + checkForConcurrentModification(); + if (!hasNext()) { + throw new NoSuchElementException(); + } + nextCalled = true; + return (E) elements[index++]; + } + + @Override + public void remove() { + checkForConcurrentModification(); + Preconditions.checkState(nextCalled, "no calls to next() since the last call to remove()"); + expectedModCount++; + index--; + CompactHashSet.this.remove(elements[index], getHash(entries[index])); + nextCalled = false; + } + + private void checkForConcurrentModification() { + if (modCount != expectedModCount) { + throw new ConcurrentModificationException(); + } + } + }; + } + + @Override + public int size() { + return size; + } + + @Override + public boolean isEmpty() { + return size == 0; + } + + @Override + public Object[] toArray() { + return Arrays.copyOf(elements, size); + } + + @SuppressWarnings("unchecked") + @Override + public <T> T[] toArray(T[] a) { + if (a.length < size) { + a = (T[]) Array.newInstance(a.getClass().getComponentType(), size); + } + System.arraycopy(elements, 0, a, 0, size); + return a; + } + + /** + * Ensures that this {@code CompactHashSet} has the smallest representation in memory, + * given its current size. + */ + public void trimToSize() { + int size = this.size; + if (size < entries.length) { + resizeEntries(size); + } + // size / loadFactor gives the table size of the appropriate load factor, + // but that may not be a power of two. We floor it to a power of two by + // keeping its highest bit. But the smaller table may have a load factor + // larger than what we want; then we want to go to the next power of 2 if we can + int minimumTableSize = Math.max(1, Integer.highestOneBit((int) (size / loadFactor))); + if (minimumTableSize < MAXIMUM_CAPACITY) { + double load = (double) size / minimumTableSize; + if (load > loadFactor) { + minimumTableSize <<= 1; // increase to next power if possible + } + } + + if (minimumTableSize < table.length) { + resizeTable(minimumTableSize); + } + } + + @Override + public void clear() { + modCount++; + Arrays.fill(elements, 0, size, null); + Arrays.fill(table, UNSET); + Arrays.fill(entries, UNSET); + this.size = 0; + } + + private void writeObject(ObjectOutputStream stream) throws IOException { + stream.defaultWriteObject(); + stream.writeInt(table.length); + stream.writeFloat(loadFactor); + stream.writeInt(size); + for (E e : this) { + stream.writeObject(e); + } + } + + @SuppressWarnings("unchecked") + private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { + stream.defaultReadObject(); + int length = stream.readInt(); + float loadFactor = stream.readFloat(); + int elementCount = stream.readInt(); + try { + init(length, loadFactor); + } catch (IllegalArgumentException e) { + throw new InvalidObjectException(e.getMessage()); + } + for (int i = elementCount; --i >= 0;) { + E element = (E) stream.readObject(); + add(element); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/EquivalenceRelation.java b/src/main/java/com/google/devtools/build/lib/collect/EquivalenceRelation.java new file mode 100644 index 0000000..1596523 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/EquivalenceRelation.java
@@ -0,0 +1,93 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.collect; + +/** + * A comparison function, which imposes an equivalence relation on some + * collection of objects. + * + * <p>The ordering imposed by an EquivalenceRelation <tt>e</tt> on a set of + * elements <tt>S</tt> is said to be <i>consistent with equals</i> if and only + * if <tt>(compare((Object)e1, (Object)e2)==0)</tt> has the same boolean value + * as <tt>e1.equals((Object)e2)</tt> for every <tt>e1</tt> and <tt>e2</tt> in + * <tt>S</tt>.<p> + * + * <p>Unlike {@link java.util.Comparator}, whose implementations are often + * consistent with equals, the applications for which EquivalenceRelation + * instances are used means that its implementations rarely are. They may are + * usually more or less discriminative than the default equivalence relation + * for the type. + * + * <p>For example, consider possible equivalence relations for {@link + * java.lang.Integer}: the default equivalence defined by Integer.equals() is + * based on the integer value is represents, but two alternative equivalences + * would be {@link EquivalenceRelation#IDENTITY} (object identity—a more + * discriminative relation) or <i>parity</i> (under which all even numbers, odd + * numbers are considered equivalent to each other—a less discriminative + * relation). + */ +public interface EquivalenceRelation<T> { + // This should be a superinterface of Comparator. + + /** + * Compares its two arguments for equivalence. Returns zero if they are + * considered equivalent, or non-zero otherwise.<p> + * + * The implementor must ensure that the relation is + * + * reflexive (<tt>compare(x,x)==0</tt> for all x), + * + * symmetric (<tt>compare(x,y)==compare(y,x)<tt> for all x, y), + * + * and transitive <tt>(compare(x, y)==0 && compare(y, + * z)==0</tt> implies <tt>compare(x, z)==0</tt>.<p> + * + * @param o1 the first object to be compared. + * @param o2 the second object to be compared. + * @return zero if the two objects are equivalent; some other integer value + * otherwise. + * @throws ClassCastException if the arguments' types prevent them from + * being compared by this EquivalenceRelation. + */ + int compare(T o1, T o2); + + /** + * The object-identity equivalence relation. This is the strictest possible + * equivalence relation for objects, and considers two values equal iff they + * are references to the same object instance. + */ + public static final EquivalenceRelation<?> IDENTITY = + new EquivalenceRelation<Object>() { + @Override + public int compare(Object o1, Object o2) { + return o1 == o2 ? 0 : -1; + } + }; + + /** + * The default equivalence relation for type T, using T.equals(). This + * relation considers two values equivalent if either they are both null, or + * o1.equals(o2). + */ + public static final EquivalenceRelation<?> DEFAULT = + new EquivalenceRelation<Object>() { + @Override + public int compare(Object o1, Object o2) { + return (o1 == null ? o2 == null : o1.equals(o2)) + ? 0 + : -1; + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/ImmutableIterable.java b/src/main/java/com/google/devtools/build/lib/collect/ImmutableIterable.java new file mode 100644 index 0000000..74fab83 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/ImmutableIterable.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.collect; + +import java.util.Iterator; + +/** + * A wrapper that signals the immutability of a certain iterable. + * + * <p>Intended for use in scenarios when you have an iterable that is de facto immutable, + * but is not recognized as such by {@link CollectionUtils#checkImmutable(Iterable)}. + * + * <p>Only use this when you know that the contents of the underlying iterable will never change, + * or you will be setting yourself up for aliasing bugs. + */ +public final class ImmutableIterable<T> implements Iterable<T> { + + private final Iterable<T> iterable; + + private ImmutableIterable(Iterable<T> iterable) { + this.iterable = iterable; + } + + @Override + public Iterator<T> iterator() { + return iterable.iterator(); + } + + /** + * Creates an {@link ImmutableIterable} instance. + */ + // Use a factory method in order to avoid having to specify generic arguments. + public static <T> ImmutableIterable<T> from(Iterable<T> iterable) { + return new ImmutableIterable<>(iterable); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyListMultimap.java b/src/main/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyListMultimap.java new file mode 100644 index 0000000..2163378 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyListMultimap.java
@@ -0,0 +1,394 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect; + +import com.google.common.base.Preconditions; +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMultiset; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multiset; + +import java.util.AbstractCollection; +import java.util.AbstractMap; +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A immutable multimap implementation for multimaps with comparable keys. It uses a sorted array + * and binary search to return the correct values. It's only purpose is to save memory - it consumes + * only about half the memory of the equivalent ImmutableListMultimap. Only a few methods are + * efficiently implemented: {@link #isEmpty} is O(1), {@link #get} and {@link #containsKey} are + * O(log(n)), and {@link #asMap} and {@link #values} refer to the parent instance. All other methods + * can take O(n) or even make a copy of the contents. + * + * <p>This implementation supports neither {@code null} keys nor {@code null} values. + */ +public final class ImmutableSortedKeyListMultimap<K extends Comparable<K>, V> + implements ListMultimap<K, V> { + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static final ImmutableSortedKeyListMultimap EMPTY_MULTIMAP = + new ImmutableSortedKeyListMultimap(new Comparable<?>[0], new List<?>[0]); + + /** Returns the empty multimap. */ + @SuppressWarnings("unchecked") + public static <K extends Comparable<K>, V> ImmutableSortedKeyListMultimap<K, V> of() { + // Safe because the multimap will never hold any elements. + return EMPTY_MULTIMAP; + } + + @SuppressWarnings("unchecked") + public static <K extends Comparable<K>, V> ImmutableSortedKeyListMultimap<K, V> copyOf( + Multimap<K, V> data) { + if (data.isEmpty()) { + return EMPTY_MULTIMAP; + } + if (data instanceof ImmutableSortedKeyListMultimap) { + return (ImmutableSortedKeyListMultimap<K, V>) data; + } + Set<K> keySet = data.keySet(); + int size = keySet.size(); + K[] sortedKeys = (K[]) new Comparable<?>[size]; + int index = 0; + for (K key : keySet) { + sortedKeys[index++] = Preconditions.checkNotNull(key); + } + Arrays.sort(sortedKeys); + List<V>[] values = (List<V>[]) new List<?>[size]; + for (int i = 0; i < size; i++) { + values[i] = ImmutableList.copyOf(data.get(sortedKeys[i])); + } + return new ImmutableSortedKeyListMultimap<>(sortedKeys, values); + } + + public static <K extends Comparable<K>, V> Builder<K, V> builder() { + return new Builder<>(); + } + + /** + * A builder class for ImmutableSortedKeyListMultimap<K, V> instances. + */ + public static final class Builder<K extends Comparable<K>, V> { + private final Multimap<K, V> builderMultimap = ArrayListMultimap.create(); + + Builder() { + // Not public so you must call builder() instead. + } + + public ImmutableSortedKeyListMultimap<K, V> build() { + return ImmutableSortedKeyListMultimap.copyOf(builderMultimap); + } + + public Builder<K, V> put(K key, V value) { + builderMultimap.put(Preconditions.checkNotNull(key), Preconditions.checkNotNull(value)); + return this; + } + + public Builder<K, V> putAll(K key, Collection<? extends V> values) { + Collection<V> valueList = builderMultimap.get(Preconditions.checkNotNull(key)); + for (V value : values) { + valueList.add(Preconditions.checkNotNull(value)); + } + return this; + } + + @SuppressWarnings("unchecked") + public Builder<K, V> putAll(K key, V... values) { + return putAll(Preconditions.checkNotNull(key), Arrays.asList(values)); + } + + public Builder<K, V> putAll(Multimap<? extends K, ? extends V> multimap) { + for (Map.Entry<? extends K, ? extends Collection<? extends V>> entry + : multimap.asMap().entrySet()) { + putAll(entry.getKey(), entry.getValue()); + } + return this; + } + } + + /** + * An implementation for the Multimap.asMap method. Note that AbstractMap already provides + * implementations for all methods except {@link #entrySet}, but we override a few here because we + * can do it much faster than the existing entrySet-based implementations. Also note that it + * inherits the type parameters K and V from the parent class. + */ + private class AsMap extends AbstractMap<K, Collection<V>> { + + AsMap() { + } + + @Override + public int size() { + return sortedKeys.length; + } + + @Override + public boolean containsKey(Object key) { + return ImmutableSortedKeyListMultimap.this.containsKey(key); + } + + @Override + public Collection<V> get(Object key) { + int index = Arrays.binarySearch(sortedKeys, key); + // Note the different semantic between Map and Multimap. + return index >= 0 ? values[index] : null; + } + + @Override + public Collection<V> remove(Object key) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + @Override + public Set<Entry<K, Collection<V>>> entrySet() { + ImmutableSet.Builder<Entry<K, Collection<V>>> builder = ImmutableSet.builder(); + for (int i = 0; i < sortedKeys.length; i++) { + builder.add(new SimpleImmutableEntry<K, Collection<V>>(sortedKeys[i], values[i])); + } + return builder.build(); + } + } + + private class ValuesCollection extends AbstractCollection<V> { + + ValuesCollection() { + } + + @Override + public int size() { + return ImmutableSortedKeyListMultimap.this.size(); + } + + @Override + public boolean isEmpty() { + return sortedKeys.length == 0; + } + + @Override + public boolean contains(Object o) { + return ImmutableSortedKeyListMultimap.this.containsValue(o); + } + + @Override + public Iterator<V> iterator() { + if (isEmpty()) { + return Collections.emptyIterator(); + } + return new AbstractIterator<V>() { + private int currentList = 0; + private int currentIndex = 0; + + @Override + protected V computeNext() { + if (currentList >= values.length) { + return endOfData(); + } + V result = values[currentList].get(currentIndex); + // Find the next list/index pair. + currentIndex++; + if (currentIndex >= values[currentList].size()) { + currentIndex = 0; + currentList++; + } + return result; + } + }; + } + + @Override + public boolean remove(Object o) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean removeAll(Collection<?> c) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean retainAll(Collection<?> c) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + } + + private final K[] sortedKeys; + private final List<V>[] values; + + private ImmutableSortedKeyListMultimap(K[] sortedKeys, List<V>[] values) { + this.sortedKeys = sortedKeys; + this.values = values; + } + + @Override + public int size() { + int result = 0; + for (List<V> list : values) { + result += list.size(); + } + return result; + } + + @Override + public boolean isEmpty() { + return sortedKeys.length == 0; + } + + @Override + public boolean containsKey(Object key) { + int index = Arrays.binarySearch(sortedKeys, key); + return index >= 0; + } + + @Override + public boolean containsValue(Object value) { + for (List<V> list : values) { + if (list.contains(value)) { + return true; + } + } + return false; + } + + @Override + public boolean containsEntry(Object key, Object value) { + int index = Arrays.binarySearch(sortedKeys, key); + if (index >= 0) { + return values[index].contains(value); + } + return false; + } + + @Override + public boolean put(K key, V value) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean remove(Object key, Object value) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean putAll(K key, Iterable<? extends V> values) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean putAll(Multimap<? extends K, ? extends V> multimap) { + throw new UnsupportedOperationException(); + } + + @Override + public List<V> replaceValues(K key, Iterable<? extends V> values) { + throw new UnsupportedOperationException(); + } + + @Override + public List<V> removeAll(Object key) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + @Override + public List<V> get(K key) { + int index = Arrays.binarySearch(sortedKeys, key); + return index >= 0 ? values[index] : ImmutableList.<V>of(); + } + + @Override + public Set<K> keySet() { + return ImmutableSet.copyOf(sortedKeys); + } + + @Override + public Multiset<K> keys() { + return ImmutableMultiset.copyOf(sortedKeys); + } + + @Override + public Collection<V> values() { + return new ValuesCollection(); + } + + @Override + public Collection<Entry<K, V>> entries() { + ImmutableList.Builder<Entry<K, V>> builder = ImmutableList.builder(); + for (int i = 0; i < sortedKeys.length; i++) { + for (V value : values[i]) { + builder.add(new SimpleImmutableEntry<K, V>(sortedKeys[i], value)); + } + } + return builder.build(); + } + + /** + * {@inheritDoc} + * + * <p>Note that only {@code get} and {@code containsKey} are implemented efficiently on the + * returned map. + */ + @Override + public Map<K, Collection<V>> asMap() { + return new AsMap(); + } + + @Override + public String toString() { + return asMap().toString(); + } + + @Override + public int hashCode() { + return asMap().hashCode(); + } + + @Override + public boolean equals(@Nullable Object object) { + if (this == object) { + return true; + } + if (object instanceof Multimap) { + Multimap<?, ?> that = (Multimap<?, ?>) object; + return asMap().equals(that.asMap()); + } + return false; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyMap.java b/src/main/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyMap.java new file mode 100644 index 0000000..e8f4621 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyMap.java
@@ -0,0 +1,310 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect; + +import com.google.common.base.Preconditions; +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ImmutableSet; + +import java.util.AbstractCollection; +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A immutable map implementation for maps with comparable keys. It uses a sorted array + * and binary search to return the correct values. Its only purpose is to save memory - for n + * entries, it consumes 8n + 64 bytes, much less than a normal HashMap (43n + 128) or an + * ImmutableMap (35n + 81). + * + * <p>Only a few methods are efficiently implemented: {@link #isEmpty} is O(1), {@link #get} and + * {@link #containsKey} are O(log(n)), using binary search; {@link #keySet} and {@link #values} + * refer to the parent instance. All other methods can take O(n) or even make a copy of the + * contents. + * + * <p>This implementation supports neither {@code null} keys nor {@code null} values. + * + * @param <K> the type of keys maintained by this map; keys must be comparable + * @param <V> the type of mapped values + */ +public final class ImmutableSortedKeyMap<K extends Comparable<K>, V> implements Map<K, V> { + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static final ImmutableSortedKeyMap EMPTY_MAP = + new ImmutableSortedKeyMap(new Comparable<?>[0], new Object[0]); + + /** Returns the empty multimap. */ + @SuppressWarnings("unchecked") + public static <K extends Comparable<K>, V> ImmutableSortedKeyMap<K, V> of() { + // Safe because the multimap will never hold any elements. + return EMPTY_MAP; + } + + public static <K extends Comparable<K>, V> ImmutableSortedKeyMap<K, V> of(K key0, V value0) { + return ImmutableSortedKeyMap.<K, V>builder() + .put(key0, value0) + .build(); + } + + public static <K extends Comparable<K>, V> ImmutableSortedKeyMap<K, V> of( + K key0, V value0, K key1, V value1) { + return ImmutableSortedKeyMap.<K, V>builder() + .put(key0, value0) + .put(key1, value1) + .build(); + } + + @SuppressWarnings("unchecked") + public static <K extends Comparable<K>, V> ImmutableSortedKeyMap<K, V> copyOf(Map<K, V> data) { + if (data.isEmpty()) { + return EMPTY_MAP; + } + if (data instanceof ImmutableSortedKeyMap) { + return (ImmutableSortedKeyMap<K, V>) data; + } + Set<K> keySet = data.keySet(); + int size = keySet.size(); + K[] sortedKeys = (K[]) new Comparable<?>[size]; + int index = 0; + for (K key : keySet) { + sortedKeys[index] = Preconditions.checkNotNull(key); + index++; + } + Arrays.sort(sortedKeys); + V[] values = (V[]) new Object[size]; + for (int i = 0; i < size; i++) { + values[i] = data.get(sortedKeys[i]); + } + return new ImmutableSortedKeyMap<>(sortedKeys, values); + } + + public static <K extends Comparable<K>, V> Builder<K, V> builder() { + return new Builder<>(); + } + + /** + * A builder class for ImmutableSortedKeyListMultimap<K, V> instances. + */ + public static final class Builder<K extends Comparable<K>, V> { + private final Map<K, V> builderMap = new HashMap<>(); + + Builder() { + // Not public so you must call builder() instead. + } + + public ImmutableSortedKeyMap<K, V> build() { + return ImmutableSortedKeyMap.copyOf(builderMap); + } + + public Builder<K, V> put(K key, V value) { + builderMap.put(Preconditions.checkNotNull(key), Preconditions.checkNotNull(value)); + return this; + } + + public Builder<K, V> putAll(Map<? extends K, ? extends V> map) { + for (Map.Entry<? extends K, ? extends V> entry : map.entrySet()) { + put(entry.getKey(), entry.getValue()); + } + return this; + } + } + + private class ValuesCollection extends AbstractCollection<V> { + + ValuesCollection() { + } + + @Override + public int size() { + return ImmutableSortedKeyMap.this.size(); + } + + @Override + public boolean isEmpty() { + return sortedKeys.length == 0; + } + + @Override + public boolean contains(Object o) { + return ImmutableSortedKeyMap.this.containsValue(o); + } + + @Override + public Iterator<V> iterator() { + if (isEmpty()) { + return Collections.emptyIterator(); + } + return new AbstractIterator<V>() { + private int currentIndex = 0; + + @Override + protected V computeNext() { + if (currentIndex >= values.length) { + return endOfData(); + } + return values[currentIndex++]; + } + }; + } + + @Override + public boolean remove(Object o) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean removeAll(Collection<?> c) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean retainAll(Collection<?> c) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + } + + private final K[] sortedKeys; + private final V[] values; + + private ImmutableSortedKeyMap(K[] sortedKeys, V[] values) { + this.sortedKeys = sortedKeys; + this.values = values; + } + + @Override + public int size() { + return sortedKeys.length; + } + + @Override + public boolean isEmpty() { + return sortedKeys.length == 0; + } + + @Override + public boolean containsKey(@Nullable Object key) { + if (key == null) { + return false; + } + int index = Arrays.binarySearch(sortedKeys, key); + return index >= 0; + } + + @Override + public boolean containsValue(@Nullable Object value) { + if (value == null) { + return false; + } + for (V v : values) { + if (v.equals(value)) { + return true; + } + } + return false; + } + + @Override + public V put(K key, V value) { + throw new UnsupportedOperationException(); + } + + @Override + public V remove(Object key) { + throw new UnsupportedOperationException(); + } + + @Override + public void putAll(Map<? extends K, ? extends V> map) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + @Override + public V get(@Nullable Object key) { + if (key == null) { + return null; + } + int index = Arrays.binarySearch(sortedKeys, key); + return index >= 0 ? values[index] : null; + } + + @Override + public Set<K> keySet() { + return ImmutableSet.copyOf(sortedKeys); + } + + @Override + public Collection<V> values() { + return new ValuesCollection(); + } + + @Override + public Set<Entry<K, V>> entrySet() { + ImmutableSet.Builder<Entry<K, V>> builder = ImmutableSet.builder(); + for (int i = 0; i < sortedKeys.length; i++) { + builder.add(new SimpleImmutableEntry<K, V>(sortedKeys[i], values[i])); + } + return builder.build(); + } + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append('{'); + for (int i = 0; i < sortedKeys.length; i++) { + if (i != 0) { + result.append(", "); + } + result.append(sortedKeys[i]).append('=').append(values[i]); + } + result.append('}'); + return result.toString(); + } + + @Override + public int hashCode() { + int h = 0; + for (Entry<K, V> entry : entrySet()) { + h += entry.hashCode(); + } + return h; + } + + @Override + public boolean equals(@Nullable Object object) { + if (this == object) { + return true; + } + if (object instanceof Map) { + throw new UnsupportedOperationException(); + } + return false; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/IterablesChain.java b/src/main/java/com/google/devtools/build/lib/collect/IterablesChain.java new file mode 100644 index 0000000..15182b6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/IterablesChain.java
@@ -0,0 +1,159 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.collect; + +import com.google.common.base.Joiner; +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; + +/** + * An immutable chain of immutable Iterables. + * + * <p>This class is defined for the sole purpose of being able to check for immutability + * (using instanceof). Otherwise, we could use a plain Iterable (as returned by + * {@code Iterables.concat()}). + * + * @see CollectionUtils#checkImmutable(Iterable) + */ +public final class IterablesChain<T> implements Iterable<T> { + + private final Iterable<T> chain; + + private IterablesChain(Iterable<T> chain) { + this.chain = chain; + } + + @Override + public Iterator<T> iterator() { + return chain.iterator(); + } + + public static <T> Builder<T> builder() { + return new Builder<>(); + } + + @Override + public String toString() { + return "[" + Joiner.on(", ").join(this) + "]"; + } + + /** + * Builder for IterablesChain. + * + * + */ + public static class Builder<T> { + private List<Iterable<T>> iterables = new ArrayList<>(); + private boolean deduplicate; + + private Builder() { + } + + /** + * Adds an immutable iterable to the end of the chain. + * + * <p>If the iterable can not be confirmed to be immutable, a runtime error is thrown. + */ + public Builder<T> add(Iterable<T> iterable) { + CollectionUtils.checkImmutable(iterable); + if (!Iterables.isEmpty(iterable)) { + iterables.add(iterable); + } + return this; + } + + /** + * Adds a single element to the chain. + */ + public Builder<T> addElement(T element) { + iterables.add(ImmutableList.of(element)); + return this; + } + + /** + * Returns true if the chain is empty. + */ + public boolean isEmpty() { + return iterables.isEmpty(); + } + + /** + * If this is called, the the resulting {@link IterablesChain} object uses a hash set to remove + * duplicate elements. + */ + public Builder<T> deduplicate() { + this.deduplicate = true; + return this; + } + + /** + * Builds an iterable that iterates through all elements in this chain. + */ + public IterablesChain<T> build() { + if (isEmpty()) { + return new IterablesChain<>(ImmutableList.<T>of()); + } + Iterable<T> concat = Iterables.concat(ImmutableList.copyOf(iterables)); + return new IterablesChain<>(deduplicate ? new Deduper<>(concat) : concat); + } + } + + /** + * An iterable implementation that removes duplicate elements (as determined by equals), using a + * hash set. + */ + private static final class Deduper<T> implements Iterable<T> { + private final Iterable<T> iterable; + + public Deduper(Iterable<T> iterable) { + this.iterable = iterable; + } + + @Override + public Iterator<T> iterator() { + return new DedupingIterator<T>(iterable.iterator()); + } + } + + /** + * An iterator implementation that removes duplicate elements (as determined by equals), using a + * hash set. + */ + private static final class DedupingIterator<T> extends AbstractIterator<T> { + private final HashSet<T> set = new HashSet<>(); + private final Iterator<T> it; + + public DedupingIterator(Iterator<T> it) { + this.it = it; + } + + @Override + protected T computeNext() { + while (it.hasNext()) { + T next = it.next(); + if (set.add(next)) { + return next; + } + } + return endOfData(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderExpander.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderExpander.java new file mode 100644 index 0000000..5ac8610 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderExpander.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableCollection; + +/** + * A nested set expander that implements left-to-right postordering. + * + * <p>For example, for the nested set {B, D, {A, C}}, the iteration order is "A C B D" + * (child-first). + * + * <p>This type of set would typically be used for artifacts where elements of nested sets go before + * the direct members of a set, for example in the case of constructing Java classpaths. + */ +final class CompileOrderExpander<E> implements NestedSetExpander<E> { + + // We suppress unchecked warning so that we can access the internal raw structure of the + // NestedSet. + @SuppressWarnings("unchecked") + @Override + public void expandInto(NestedSet<E> set, Uniqueifier uniqueifier, + ImmutableCollection.Builder<E> builder) { + for (NestedSet<E> subset : set.transitiveSets()) { + if (!subset.isEmpty() && uniqueifier.isUnique(subset)) { + expandInto(subset, uniqueifier, builder); + } + } + + // This switch is here to compress the memo used by the uniqueifier + for (Object e : set.directMembers()) { + if (uniqueifier.isUnique(e)) { + builder.add((E) e); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderNestedSetFactory.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderNestedSetFactory.java new file mode 100644 index 0000000..178f9a5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderNestedSetFactory.java
@@ -0,0 +1,152 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableList; + +/** + * Compile order {@code NestedSet} factory. + */ +final class CompileOrderNestedSetFactory implements NestedSetFactory { + + @Override + public <E> NestedSet<E> onlyDirects(Object[] directs) { + return new CompileOnlyDirectsNestedSet<>(directs); + } + + @Override + public <E> NestedSet<E> onlyDirects(ImmutableList<E> directs) { + return new CompileOrderImmutableListDirectsNestedSet<>(directs); + } + + @Override + public <E> NestedSet<E> oneDirectOneTransitive(E direct, NestedSet<E> transitive) { + return new CompileOneDirectOneTransitiveNestedSet<>(direct, transitive); + } + + @Override + public <E> NestedSet<E> manyDirectsOneTransitive(Object[] direct, + NestedSet<E> transitive) { + return new CompileManyDirectOneTransitiveNestedSet<>(direct, transitive); + } + + @Override + public <E> NestedSet<E> onlyOneTransitive(NestedSet<E> transitive) { + return new CompileOnlyOneTransitiveNestedSet<>(transitive); + } + + @Override + public <E> NestedSet<E> onlyManyTransitives(NestedSet[] transitives) { + return new CompileOnlyTransitivesNestedSet<>(transitives); + } + + @Override + public <E> NestedSet<E> oneDirectManyTransitive(Object direct, NestedSet[] transitives) { + return new CompileOneDirectManyTransitive<>(direct, transitives); + } + + @Override + public <E> NestedSet<E> manyDirectManyTransitive(Object[] directs, NestedSet[] transitives) { + return new CompileManyDirectManyTransitive<>(directs, transitives); + } + + @Override + public <E> NestedSet<E> oneDirect(E element) { + return new CompileSingleDirectNestedSet<>(element); + } + + private static class CompileOnlyDirectsNestedSet<E> extends OnlyDirectsNestedSet<E> { + + CompileOnlyDirectsNestedSet(Object[] directs) { super(directs); } + + @Override + public Order getOrder() { return Order.COMPILE_ORDER; } + } + + private static class CompileOneDirectOneTransitiveNestedSet<E> extends + OneDirectOneTransitiveNestedSet<E> { + + private CompileOneDirectOneTransitiveNestedSet(E direct, NestedSet<E> transitive) { + super(direct, transitive); + } + + @Override + public Order getOrder() { return Order.COMPILE_ORDER; } + } + + private static class CompileOneDirectManyTransitive<E> extends OneDirectManyTransitive<E> { + + private CompileOneDirectManyTransitive(Object direct, NestedSet[] transitive) { + super(direct, transitive); + } + + @Override + public Order getOrder() { return Order.COMPILE_ORDER; } + } + + private static class CompileManyDirectManyTransitive<E> extends ManyDirectManyTransitive<E> { + + private CompileManyDirectManyTransitive(Object[] directs, NestedSet[] transitives) { + super(directs, transitives); + } + + @Override + public Order getOrder() { return Order.COMPILE_ORDER; } + } + + private static class CompileOnlyOneTransitiveNestedSet<E> extends OnlyOneTransitiveNestedSet<E> { + + private CompileOnlyOneTransitiveNestedSet(NestedSet<E> transitive) { super(transitive); } + + @Override + public Order getOrder() { return Order.COMPILE_ORDER; } + } + + private static class CompileManyDirectOneTransitiveNestedSet<E> extends + ManyDirectOneTransitiveNestedSet<E> { + + private CompileManyDirectOneTransitiveNestedSet(Object[] direct, + NestedSet<E> transitive) { super(direct, transitive); } + + @Override + public Order getOrder() { return Order.COMPILE_ORDER; } + } + + private static class CompileOnlyTransitivesNestedSet<E> extends OnlyTransitivesNestedSet<E> { + + private CompileOnlyTransitivesNestedSet(NestedSet[] transitives) { super(transitives); } + + @Override + public Order getOrder() { return Order.COMPILE_ORDER; } + } + + private static class CompileOrderImmutableListDirectsNestedSet<E> extends + ImmutableListDirectsNestedSet<E> { + + private CompileOrderImmutableListDirectsNestedSet(ImmutableList<E> directs) { super(directs); } + + @Override + public Order getOrder() { + return Order.COMPILE_ORDER; + } + } + + private static class CompileSingleDirectNestedSet<E> extends SingleDirectNestedSet<E> { + + private CompileSingleDirectNestedSet(E element) { super(element); } + + @Override + public Order getOrder() { return Order.COMPILE_ORDER; } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/EmptyNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/EmptyNestedSet.java new file mode 100644 index 0000000..5889ba8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/EmptyNestedSet.java
@@ -0,0 +1,87 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * An empty nested set. + */ +final class EmptyNestedSet<E> extends NestedSet<E> { + private static final NestedSet[] EMPTY_NESTED_SET = new NestedSet[0]; + private static final Object[] EMPTY_ELEMENTS = new Object[0]; + private final Order order; + + EmptyNestedSet(Order type) { + this.order = type; + } + + @Override + public Iterator<E> iterator() { + return ImmutableList.<E>of().iterator(); + } + + @Override + Object[] directMembers() { + return EMPTY_ELEMENTS; + } + + @Override + NestedSet[] transitiveSets() { + return EMPTY_NESTED_SET; + } + + @Override + public boolean isEmpty() { + return true; + } + + @Override + public List<E> toList() { + return ImmutableList.of(); + } + + @Override + public Set<E> toSet() { + return ImmutableSet.of(); + } + + @Override + public String toString() { + return "{}"; + } + + @Override + public Order getOrder() { + return order; + } + + @Override + public boolean shallowEquals(@Nullable NestedSet<? extends E> other) { + return other != null && getOrder() == other.getOrder() && other.isEmpty(); + } + + @Override + public int shallowHashCode() { + return Objects.hash(getOrder()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/ImmutableListDirectsNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/ImmutableListDirectsNestedSet.java new file mode 100644 index 0000000..12bf222 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/ImmutableListDirectsNestedSet.java
@@ -0,0 +1,88 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Memory-optimized NestedSet implementation for NestedSets without transitive dependencies that + * allows us to share an ImmutableList. + */ +abstract class ImmutableListDirectsNestedSet<E> extends NestedSet<E> { + + private static final NestedSet[] EMPTY = new NestedSet[0]; + private final ImmutableList<E> directDeps; + + public ImmutableListDirectsNestedSet(ImmutableList<E> directDeps) { + this.directDeps = directDeps; + } + + @Override + public abstract Order getOrder(); + + @Override + Object[] directMembers() { + return directDeps.toArray(); + } + + @Override + NestedSet[] transitiveSets() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return directDeps.isEmpty(); + } + + /** + * Currently all the Order implementations return the direct elements in the same order if they do + * not have transitive elements. So we skip calling order.getExpander(). + */ + @SuppressWarnings("unchecked") + @Override + public List<E> toList() { + return directDeps; + } + + @SuppressWarnings("unchecked") + @Override + public Set<E> toSet() { + return ImmutableSet.copyOf(directDeps); + } + + @Override + public boolean shallowEquals(@Nullable NestedSet<? extends E> other) { + if (this == other) { + return true; + } + if (other == null) { + return false; + } + return getOrder().equals(other.getOrder()) + && other instanceof ImmutableListDirectsNestedSet + && directDeps.equals(((ImmutableListDirectsNestedSet) other).directDeps); + } + + @Override + public int shallowHashCode() { + return directDeps.hashCode(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderExpander.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderExpander.java new file mode 100644 index 0000000..603ac15 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderExpander.java
@@ -0,0 +1,105 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; + +/** + * A nested set expander that implements a variation of left-to-right preordering. + * + * <p>For example, for the nested set {A, C, {B, D}}, the iteration order is "A C B D" + * (parent-first). + * + * <p>This type of set would typically be used for artifacts where elements of + * nested sets go after the direct members of a set, for example when providing + * a list of libraries to the C++ compiler. + * + * <p>The custom ordering has the property that elements of nested sets always come + * before elements of descendant nested sets. Left-to-right order is preserved if + * possible, both for items and for references to nested sets. + * + * <p>The left-to-right pre-order-like ordering is implemented by running a + * right-to-left postorder traversal and then reversing the result. + * + * <p>The reason naive left-to left-to-right preordering is not used here is that + * it does not handle diamond-like structures properly. For example, take the + * following structure (nesting downwards): + * + * <pre> + * A + * / \ + * B C + * \ / + * D + * </pre> + * + * <p>Naive preordering would produce "A B D C", which does not preserve the + * "parent before child" property: C is a parent of D, so C should come before + * D. Either "A B C D" or "A C B D" would be acceptable. This implementation + * returns the first option of the two so that left-to-right order is preserved. + * + * <p>In case the nested sets form a tree, the ordering algorithm is equivalent to + * standard left-to-right preorder. + * + * <p>Sometimes it may not be possible to preserve left-to-right order: + * + * <pre> + * A + * / \ + * B C + * / \ / \ + * \ E / + * \ / + * \ / + * D + * </pre> + * + * <p>The left branch (B) would indicate "D E" ordering and the right branch (C) + * dictates "E D". In such cases ordering is decided by the rightmost branch + * because of the list reversing behind the scenes, so the ordering in the final + * enumeration will be "E D". + */ + +final class LinkOrderExpander<E> implements NestedSetExpander<E> { + @Override + public void expandInto(NestedSet<E> nestedSet, Uniqueifier uniqueifier, + ImmutableCollection.Builder<E> builder) { + ImmutableList.Builder<E> result = ImmutableList.builder(); + internalEnumerate(nestedSet, uniqueifier, result); + builder.addAll(result.build().reverse()); + } + + // We suppress unchecked warning so that we can access the internal raw structure of the + // NestedSet. + @SuppressWarnings("unchecked") + private void internalEnumerate(NestedSet<E> set, Uniqueifier uniqueifier, + ImmutableCollection.Builder<E> builder) { + NestedSet[] transitiveSets = set.transitiveSets(); + for (int i = transitiveSets.length - 1; i >= 0; i--) { + NestedSet<E> subset = transitiveSets[i]; + if (!subset.isEmpty() && uniqueifier.isUnique(subset)) { + internalEnumerate(subset, uniqueifier, builder); + } + } + + Object[] directMembers = set.directMembers(); + for (int i = directMembers.length - 1; i >= 0; i--) { + Object e = directMembers[i]; + if (uniqueifier.isUnique(e)) { + builder.add((E) e); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderNestedSetFactory.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderNestedSetFactory.java new file mode 100644 index 0000000..9e23793 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderNestedSetFactory.java
@@ -0,0 +1,152 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableList; + +/** + * Link order {@code NestedSet} factory. + */ +final class LinkOrderNestedSetFactory implements NestedSetFactory { + + @Override + public <E> NestedSet<E> onlyDirects(Object[] directs) { + return new LinkOnlyDirectsNestedSet<>(directs); + } + + @Override + public <E> NestedSet<E> onlyDirects(ImmutableList<E> directs) { + return new LinkImmutableListDirectsNestedSet<>(directs); + } + + @Override + public <E> NestedSet<E> oneDirectOneTransitive(E direct, NestedSet<E> transitive) { + return new LinkOneDirectOneTransitiveNestedSet<>(direct, transitive); + } + + @Override + public <E> NestedSet<E> manyDirectsOneTransitive(Object[] direct, + NestedSet<E> transitive) { + return new LinkManyDirectOneTransitiveNestedSet<>(direct, transitive); + } + + @Override + public <E> NestedSet<E> onlyOneTransitive(NestedSet<E> transitive) { + return new LinkOnlyOneTransitiveNestedSet<>(transitive); + } + + @Override + public <E> NestedSet<E> onlyManyTransitives(NestedSet[] transitives) { + return new LinkOnlyTransitivesNestedSet<>(transitives); + } + + @Override + public <E> NestedSet<E> oneDirectManyTransitive(Object direct, NestedSet[] transitives) { + return new LinkOneDirectManyTransitive<>(direct, transitives); + } + + @Override + public <E> NestedSet<E> manyDirectManyTransitive(Object[] directs, NestedSet[] transitives) { + return new LinkManyDirectManyTransitive<>(directs, transitives); + } + + @Override + public <E> NestedSet<E> oneDirect(E element) { + return new LinkSingleDirectNestedSet<>(element); + } + + private static class LinkOnlyDirectsNestedSet<E> extends OnlyDirectsNestedSet<E> { + + LinkOnlyDirectsNestedSet(Object[] directs) { super(directs); } + + @Override + public Order getOrder() { return Order.LINK_ORDER; } + } + + private static class LinkOneDirectOneTransitiveNestedSet<E> extends + OneDirectOneTransitiveNestedSet<E> { + + private LinkOneDirectOneTransitiveNestedSet(E direct, NestedSet<E> transitive) { + super(direct, transitive); + } + + @Override + public Order getOrder() { return Order.LINK_ORDER; } + } + + private static class LinkOneDirectManyTransitive<E> extends OneDirectManyTransitive<E> { + + private LinkOneDirectManyTransitive(Object direct, NestedSet[] transitive) { + super(direct, transitive); + } + + @Override + public Order getOrder() { return Order.LINK_ORDER; } + } + + private static class LinkManyDirectManyTransitive<E> extends ManyDirectManyTransitive<E> { + + private LinkManyDirectManyTransitive(Object[] directs, NestedSet[] transitives) { + super(directs, transitives); + } + + @Override + public Order getOrder() { return Order.LINK_ORDER; } + } + + private static class LinkOnlyOneTransitiveNestedSet<E> extends OnlyOneTransitiveNestedSet<E> { + + private LinkOnlyOneTransitiveNestedSet(NestedSet<E> transitive) { super(transitive); } + + @Override + public Order getOrder() { return Order.LINK_ORDER; } + } + + private static class LinkManyDirectOneTransitiveNestedSet<E> extends + ManyDirectOneTransitiveNestedSet<E> { + + private LinkManyDirectOneTransitiveNestedSet(Object[] direct, + NestedSet<E> transitive) { super(direct, transitive); } + + @Override + public Order getOrder() { return Order.LINK_ORDER; } + } + + private static class LinkOnlyTransitivesNestedSet<E> extends OnlyTransitivesNestedSet<E> { + + private LinkOnlyTransitivesNestedSet(NestedSet[] transitives) { super(transitives); } + + @Override + public Order getOrder() { return Order.LINK_ORDER; } + } + + private static class LinkImmutableListDirectsNestedSet<E> extends + ImmutableListDirectsNestedSet<E> { + + private LinkImmutableListDirectsNestedSet(ImmutableList<E> directs) { super(directs); } + + @Override + public Order getOrder() { + return Order.LINK_ORDER; + } + } + + private static class LinkSingleDirectNestedSet<E> extends SingleDirectNestedSet<E> { + + private LinkSingleDirectNestedSet(E element) { super(element); } + + @Override + public Order getOrder() { return Order.LINK_ORDER; } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/ManyDirectManyTransitive.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/ManyDirectManyTransitive.java new file mode 100644 index 0000000..05ba2e8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/ManyDirectManyTransitive.java
@@ -0,0 +1,63 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import java.util.Arrays; +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * NestedSet implementation that can have many direct elements and many transitive + * {@code NestedSet}s. + */ +abstract class ManyDirectManyTransitive<E> extends MemoizedUniquefierNestedSet<E> { + + private final Object[] directs; + private final NestedSet[] transitives; + private Object memo; + + ManyDirectManyTransitive(Object[] directs, NestedSet[] transitives) { + this.directs = directs; + this.transitives = transitives; + } + + @Override + Object getMemo() { return memo; } + + @Override + void setMemo(Object memo) { this.memo = memo; } + + @Override + Object[] directMembers() { return directs; } + + @Override + NestedSet[] transitiveSets() { return transitives; } + + @Override + public boolean shallowEquals(@Nullable NestedSet<? extends E> other) { + if (this == other) { + return true; + } + return other != null + && getOrder().equals(other.getOrder()) + && other instanceof ManyDirectManyTransitive + && Arrays.equals(directs, ((ManyDirectManyTransitive) other).directs) + && Arrays.equals(transitives, ((ManyDirectManyTransitive) other).transitives); + } + + @Override + public int shallowHashCode() { + return Objects.hash(getOrder(), Arrays.hashCode(directs), Arrays.hashCode(transitives)); } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/ManyDirectOneTransitiveNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/ManyDirectOneTransitiveNestedSet.java new file mode 100644 index 0000000..cdb4f04 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/ManyDirectOneTransitiveNestedSet.java
@@ -0,0 +1,63 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import java.util.Arrays; +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * Memory-efficient implementation for the case where we have many direct elements and one + * transitive NestedSet. + */ +abstract class ManyDirectOneTransitiveNestedSet<E> extends MemoizedUniquefierNestedSet<E> { + + private final Object[] directs; + private final NestedSet<E> transitive; + private Object memo; + + public ManyDirectOneTransitiveNestedSet(Object[] directs, NestedSet<E> transitive) { + this.directs = directs; + this.transitive = transitive; + } + + @Override + Object getMemo() { return memo; } + + @Override + void setMemo(Object memo) { this.memo = memo; } + + @Override + Object[] directMembers() { return directs; } + + @Override + NestedSet[] transitiveSets() { return new NestedSet[]{transitive}; } + + @Override + public boolean shallowEquals(@Nullable NestedSet<? extends E> other) { + if (this == other) { + return true; + } + return other != null + && getOrder().equals(other.getOrder()) + && other instanceof ManyDirectOneTransitiveNestedSet + && Arrays.equals(directs, ((ManyDirectOneTransitiveNestedSet) other).directs) + && transitive == ((ManyDirectOneTransitiveNestedSet) other).transitive; + } + + @Override + public int shallowHashCode() { + return Objects.hash(getOrder(), Arrays.hashCode(directs), transitive); } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/MemoizedUniquefierNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/MemoizedUniquefierNestedSet.java new file mode 100644 index 0000000..2a7f1b6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/MemoizedUniquefierNestedSet.java
@@ -0,0 +1,74 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.Set; + +/** + * A NestedSet that keeps a memoized uniquifier so that it is faster to fill a set. + * + * <p>This class does not keep the memoized object itself so that we can take advantage of the + * memory field alignment (Memory alignment does not put in the same structure the fields of a + * class and its extensions). + */ +public abstract class MemoizedUniquefierNestedSet<E> extends NestedSet<E> { + + @Override + public List<E> toList() { + ImmutableList.Builder<E> builder = new ImmutableList.Builder<>(); + memoizedFill(builder); + return builder.build(); + } + + @Override + public Set<E> toSet() { + ImmutableSet.Builder<E> builder = new ImmutableSet.Builder<>(); + memoizedFill(builder); + return builder.build(); + } + + /** + * It does not make sense to have a {@code MemoizedUniquefierNestedSet} if it is empty. + */ + @Override + public boolean isEmpty() { return false; } + + abstract Object getMemo(); + + abstract void setMemo(Object object); + + /** + * Fill a collection builder by using a memoized {@code Uniqueifier} for faster uniqueness check. + */ + final void memoizedFill(ImmutableCollection.Builder<E> builder) { + Uniqueifier memoed; + synchronized (this) { + Object memo = getMemo(); + if (memo == null) { + RecordingUniqueifier uniqueifier = new RecordingUniqueifier(); + getOrder().<E>expander().expandInto(this, uniqueifier, builder); + setMemo(uniqueifier.getMemo()); + return; + } else { + memoed = RecordingUniqueifier.createReplayUniqueifier(memo); + } + } + getOrder().<E>expander().expandInto(this, memoed, builder); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderExpander.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderExpander.java new file mode 100644 index 0000000..6c49103 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderExpander.java
@@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableCollection; + +/** + * A nested set expander that implements naive left-to-right preordering. + * + * <p>For example, for the nested set {B, D, {A, C}}, the iteration order is "B D A C". + * + * <p>This implementation is intended for backwards-compatible nested set replacements of code that + * uses naive preordering. + * + * <p>The implementation is called naive because it does no special treatment of dependency graphs + * that are not trees. For such graphs the property of parent-before-dependencies in the iteration + * order will not be upheld. For example, the diamond-shape graph A->{B, C}, B->{D}, C->{D} will be + * enumerated as "A B D C" rather than "A B C D" or "A C B D". + * + * <p>The difference from {@link LinkOrderNestedSet} is that this implementation gives priority to + * left-to-right order over dependencies-after-parent ordering. Note that the latter is usually more + * important, so please use {@link LinkOrderNestedSet} whenever possible. + */ +final class NaiveLinkOrderExpander<E> implements NestedSetExpander<E> { + + @SuppressWarnings("unchecked") + @Override + public void expandInto(NestedSet<E> set, Uniqueifier uniqueifier, + ImmutableCollection.Builder<E> builder) { + + for (Object e : set.directMembers()) { + if (uniqueifier.isUnique(e)) { + builder.add((E) e); + } + } + + for (NestedSet<E> subset : set.transitiveSets()) { + if (!subset.isEmpty() && uniqueifier.isUnique(subset)) { + expandInto(subset, uniqueifier, builder); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderNestedSetFactory.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderNestedSetFactory.java new file mode 100644 index 0000000..4677938 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderNestedSetFactory.java
@@ -0,0 +1,153 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableList; + +/** + * NaiveLink order {@code NestedSet} factory. + */ +final class NaiveLinkOrderNestedSetFactory implements NestedSetFactory { + + @Override + public <E> NestedSet<E> onlyDirects(Object[] directs) { + return new NaiveLinkOnlyDirectsNestedSet<>(directs); + } + + @Override + public <E> NestedSet<E> onlyDirects(ImmutableList<E> directs) { + return new NaiveLinkImmutableListDirectsNestedSet<>(directs); + } + + @Override + public <E> NestedSet<E> oneDirectOneTransitive(E direct, NestedSet<E> transitive) { + return new NaiveLinkOneDirectOneTransitiveNestedSet<>(direct, transitive); + } + + @Override + public <E> NestedSet<E> manyDirectsOneTransitive(Object[] direct, + NestedSet<E> transitive) { + return new NaiveLinkManyDirectOneTransitiveNestedSet<>(direct, transitive); + } + + @Override + public <E> NestedSet<E> onlyOneTransitive(NestedSet<E> transitive) { + return new NaiveLinkOnlyOneTransitiveNestedSet<>(transitive); + } + + @Override + public <E> NestedSet<E> onlyManyTransitives(NestedSet[] transitives) { + return new NaiveLinkOnlyTransitivesNestedSet<>(transitives); + } + + @Override + public <E> NestedSet<E> oneDirectManyTransitive(Object direct, NestedSet[] transitives) { + return new NaiveLinkOneDirectManyTransitive<>(direct, transitives); + } + + @Override + public <E> NestedSet<E> manyDirectManyTransitive(Object[] directs, NestedSet[] transitives) { + return new NaiveLinkManyDirectManyTransitive<>(directs, transitives); + } + + @Override + public <E> NestedSet<E> oneDirect(final E element) { + return new NaiveLinkSingleDirectNestedSet<>(element); + } + + private static class NaiveLinkOnlyDirectsNestedSet<E> extends OnlyDirectsNestedSet<E> { + + NaiveLinkOnlyDirectsNestedSet(Object[] directs) { super(directs); } + + @Override + public Order getOrder() { return Order.NAIVE_LINK_ORDER; } + } + + private static class NaiveLinkOneDirectOneTransitiveNestedSet<E> extends + OneDirectOneTransitiveNestedSet<E> { + + private NaiveLinkOneDirectOneTransitiveNestedSet(E direct, NestedSet<E> transitive) { + super(direct, transitive); + } + + @Override + public Order getOrder() { return Order.NAIVE_LINK_ORDER; } + } + + private static class NaiveLinkOneDirectManyTransitive<E> extends OneDirectManyTransitive<E> { + + private NaiveLinkOneDirectManyTransitive(Object direct, NestedSet[] transitive) { + super(direct, transitive); + } + + @Override + public Order getOrder() { return Order.NAIVE_LINK_ORDER; } + } + + private static class NaiveLinkManyDirectManyTransitive<E> extends ManyDirectManyTransitive<E> { + + private NaiveLinkManyDirectManyTransitive(Object[] directs, NestedSet[] transitives) { + super(directs, transitives); + } + + @Override + public Order getOrder() { return Order.NAIVE_LINK_ORDER; } + } + + private static class NaiveLinkOnlyOneTransitiveNestedSet<E> + extends OnlyOneTransitiveNestedSet<E> { + + private NaiveLinkOnlyOneTransitiveNestedSet(NestedSet<E> transitive) { super(transitive); } + + @Override + public Order getOrder() { return Order.NAIVE_LINK_ORDER; } + } + + private static class NaiveLinkManyDirectOneTransitiveNestedSet<E> extends + ManyDirectOneTransitiveNestedSet<E> { + + private NaiveLinkManyDirectOneTransitiveNestedSet(Object[] direct, + NestedSet<E> transitive) { super(direct, transitive); } + + @Override + public Order getOrder() { return Order.NAIVE_LINK_ORDER; } + } + + private static class NaiveLinkOnlyTransitivesNestedSet<E> extends OnlyTransitivesNestedSet<E> { + + private NaiveLinkOnlyTransitivesNestedSet(NestedSet[] transitives) { super(transitives); } + + @Override + public Order getOrder() { return Order.NAIVE_LINK_ORDER; } + } + + private static class NaiveLinkImmutableListDirectsNestedSet<E> extends + ImmutableListDirectsNestedSet<E> { + + private NaiveLinkImmutableListDirectsNestedSet(ImmutableList<E> directs) { super(directs); } + + @Override + public Order getOrder() { + return Order.NAIVE_LINK_ORDER; + } + } + + private static class NaiveLinkSingleDirectNestedSet<E> extends SingleDirectNestedSet<E> { + + private NaiveLinkSingleDirectNestedSet(E element) { super(element); } + + @Override + public Order getOrder() { return Order.NAIVE_LINK_ORDER; } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSet.java new file mode 100644 index 0000000..da074e0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSet.java
@@ -0,0 +1,127 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.base.Joiner; + +import java.io.Serializable; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A list-like iterable that supports efficient nesting. + * + * @see NestedSetBuilder + */ +public abstract class NestedSet<E> implements Iterable<E>, Serializable { + + NestedSet() {} + + /** + * Returns the ordering of this nested set. + */ + public abstract Order getOrder(); + + /** + * Returns a collection of elements added to this specific set in an implementation-specified + * order. + * + * <p>Elements from subsets are not taken into account. + * + * <p>The reason for using Object[] instead of E[] is that when we build the NestedSet we + * would need to have access to the specific class that E represents in order to create an E + * array. Since this method is only designed to be used internally it is fine to keep it as + * Object[]. + * + * <p>Callers of this method should only consume the objects and not modify the array. + */ + abstract Object[] directMembers(); + + /** + * Returns the collection of sets included as subsets in this set. + * + * <p>Callers of this method should only consume the objects and not modify the array. + */ + abstract NestedSet[] transitiveSets(); + + /** + * Returns true if the set is empty. + */ + public abstract boolean isEmpty(); + + /** + * Returns a collection of all unique elements of this set (including subsets) + * in an implementation-specified order as a {@code Collection}. + * + * <p>If you do not need a Collection and an Iterable is enough, use the + * nested set itself as an Iterable. + */ + public Collection<E> toCollection() { + return toList(); + } + + /** + * Returns a collection of all unique elements of this set (including subsets) + * in an implementation-specified order as a {code List}. + * + * <p>Use {@link #toCollection} when possible for better efficiency. + */ + public abstract List<E> toList(); + + /** + * Returns a collection of all unique elements of this set (including subsets) + * in an implementation-specified order as a {@code Set}. + * + * <p>Use {@link #toCollection} when possible for better efficiency. + */ + public abstract Set<E> toSet(); + + /** + * Returns true if this set is equal to {@code other} based on the top-level + * elements and object identity (==) of direct subsets. As such, this function + * can fail to equate {@code this} with another {@code NestedSet} that holds + * the same elements. It will never fail to detect that two {@code NestedSet}s + * are different, however. + * + * @param other the {@code NestedSet} to compare against. + */ + public abstract boolean shallowEquals(@Nullable NestedSet<? extends E> other); + + /** + * Returns a hash code that produces a notion of identity that is consistent with + * {@link #shallowEquals}. In other words, if two {@code NestedSet}s are equal according + * to {@code #shallowEquals}, then they return the same {@code shallowHashCode}. + * + * <p>The main reason for having these separate functions instead of reusing + * the standard equals/hashCode is to minimize accidental use, since they are + * different from both standard Java objects and collection-like objects. + */ + public abstract int shallowHashCode(); + + @Override + public String toString() { + String members = Joiner.on(", ").join(directMembers()); + String nestedSets = Joiner.on(", ").join(transitiveSets()); + String separator = members.length() > 0 && nestedSets.length() > 0 ? ", " : ""; + return "{" + members + separator + nestedSets + "}"; + } + + @Override + public Iterator<E> iterator() { return new NestedSetLazyIterator<>(this); } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetBuilder.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetBuilder.java new file mode 100644 index 0000000..327fcdb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetBuilder.java
@@ -0,0 +1,253 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import static com.google.common.collect.Iterables.getOnlyElement; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + +import java.util.LinkedHashSet; + +/** + * A builder for nested sets. + * + * <p>The builder supports the standard builder interface (that is, {@code #add}, {@code #addAll} + * and {@code #addTransitive} followed by {@code build}), in addition to shortcut methods + * {@code #wrap} and {@code #of}. + */ +public final class NestedSetBuilder<E> { + + private final Order order; + private final LinkedHashSet<E> items = new LinkedHashSet<>(); + private final LinkedHashSet<NestedSet<? extends E>> transitiveSets = new LinkedHashSet<>(); + + public NestedSetBuilder(Order order) { + this.order = order; + } + + /** Returns whether the set to be built is empty. */ + public boolean isEmpty() { + return items.isEmpty() && transitiveSets.isEmpty(); + } + + /** + * Add an element. + * + * <p>Preserves ordering of added elements. Discards duplicate values. + * Throws an exception if a null value is passed in. + * + * <p>The collections of the direct members of the set and the nested sets are + * kept separate, so the order between multiple add/addAll calls matters, + * and the order between multiple addTransitive calls matters, but the order + * between add/addAll and addTransitive does not. + * + * @return the builder. + */ + @SuppressWarnings("unchecked") // B is the type of the concrete subclass + public NestedSetBuilder<E> add(E element) { + Preconditions.checkNotNull(element); + items.add(element); + return this; + } + + /** + * Adds a collection of elements to the set. + * + * <p>This is equivalent to invoking {@code add} for every item of the collection in iteration + * order. + * + * <p>The collections of the direct members of the set and the nested sets are kept separate, so + * the order between multiple add/addAll calls matters, and the order between multiple + * addTransitive calls matters, but the order between add/addAll and addTransitive does not. + * + * @return the builder. + */ + @SuppressWarnings("unchecked") // B is the type of the concrete subclass + public NestedSetBuilder<E> addAll(Iterable<? extends E> elements) { + Preconditions.checkNotNull(elements); + Iterables.addAll(items, elements); + return this; + } + + /** + * @deprecated Use {@link #addTransitive} to avoid excessive memory use. + */ + @Deprecated + public NestedSetBuilder<E> addAll(NestedSet<E> elements) { + // Do not delete this method, or else addAll(Iterable) calls with a NestedSet argument + // will not be flagged. + Iterable<E> it = elements; + addAll(it); + return this; + } + + /** + * Adds another nested set to this set. + * + * <p>Preserves ordering of added nested sets. Discards duplicate values. Throws an exception if + * a null value is passed in. + * + * <p>The collections of the direct members of the set and the nested sets are kept separate, so + * the order between multiple add/addAll calls matters, and the order between multiple + * addTransitive calls matters, but the order between add/addAll and addTransitive does not. + * + * <p>An error will be thrown if the ordering of {@code subset} is incompatible with the ordering + * of this set. Either they must match or this set must be a {@code STABLE_ORDER} set. + * + * @return the builder. + */ + public NestedSetBuilder<E> addTransitive(NestedSet<? extends E> subset) { + Preconditions.checkNotNull(subset); + if (subset.getOrder() != order && order != Order.STABLE_ORDER + && subset.getOrder() != Order.STABLE_ORDER) { + // Note that this check is not strictly necessary, although keeping the nested set types + // consistent helps readability and protects against bugs. The polymorphism regarding + // STABLE_ORDER is allowed in order to be able to, e.g., include an arbitrary nested set in + // the inputs of an action, or include a nested set that is indifferent to its order in + // multiple nested sets. + throw new IllegalStateException(subset.getOrder() + " != " + order); + } + if (!subset.isEmpty()) { + transitiveSets.add(subset); + } + return this; + } + + /** + * Builds the actual nested set. + * + * <p>This method may be called multiple times with interleaved {@link #add}, {@link #addAll} and + * {@link #addTransitive} calls. + */ + // Casting from LinkedHashSet<NestedSet<? extends E>> to LinkedHashSet<NestedSet<E>> by way of + // LinkedHashSet<?>. + @SuppressWarnings("unchecked") + public NestedSet<E> build() { + if (isEmpty()) { + return order.emptySet(); + } + + // This cast is safe because NestedSets are immutable -- we will never try to add an element to + // these nested sets, only to retrieve elements from them. Thus, treating them as NestedSet<E> + // is safe. + LinkedHashSet<NestedSet<E>> transitiveSetsCast = + (LinkedHashSet<NestedSet<E>>) (LinkedHashSet<?>) transitiveSets; + if (items.isEmpty() && (transitiveSetsCast.size() == 1)) { + NestedSet<E> candidate = getOnlyElement(transitiveSetsCast); + if (candidate.getOrder().equals(order)) { + return candidate; + } + } + int transitiveSize = transitiveSets.size(); + int directSize = items.size(); + + switch (transitiveSize) { + case 0: + switch (directSize) { + case 0: + return order.emptySet(); + case 1: + return order.factory.oneDirect(getOnlyElement(items)); + default: + return order.factory.onlyDirects(items.toArray()); + } + case 1: + switch (directSize) { + case 0: + return order.factory.onlyOneTransitive(getOnlyElement(transitiveSetsCast)); + case 1: + return order.factory.oneDirectOneTransitive(getOnlyElement(items), + getOnlyElement(transitiveSetsCast)); + default: + return order.factory.manyDirectsOneTransitive(items.toArray(), + getOnlyElement(transitiveSetsCast)); + } + default: + switch (directSize) { + case 0: + return order.factory.onlyManyTransitives( + transitiveSetsCast.toArray(new NestedSet[transitiveSize])); + case 1: + return order.factory.oneDirectManyTransitive(getOnlyElement(items), transitiveSetsCast + .toArray(new NestedSet[transitiveSize])); + default: + return order.factory.manyDirectManyTransitive(items.toArray(), + transitiveSetsCast.toArray(new NestedSet[transitiveSize])); + } + } + } + + /** + * Creates a nested set from a given list of items. + * + * <p>If the list of items is an {@link ImmutableList}, reuses the list as the backing store for + * the nested set. + */ + public static <E> NestedSet<E> wrap(Order order, Iterable<E> wrappedItems) { + ImmutableList<E> wrappedList = ImmutableList.copyOf(wrappedItems); + if (wrappedList.isEmpty()) { + return order.emptySet(); + } else if (wrappedList.size() == 1) { + return order.factory.oneDirect(getOnlyElement(wrappedItems)); + } else { + return order.factory.onlyDirects(wrappedList); + } + } + + + /** + * Creates a nested set with the given list of items as its elements. + */ + @SuppressWarnings("unchecked") + public static <E> NestedSet<E> create(Order order, E... elems) { + return wrap(order, ImmutableList.copyOf(elems)); + } + + /** + * Creates an empty nested set. + */ + public static <E> NestedSet<E> emptySet(Order order) { + return order.emptySet(); + } + + /** + * Creates a builder for stable order nested sets. + */ + public static <E> NestedSetBuilder<E> stableOrder() { + return new NestedSetBuilder<>(Order.STABLE_ORDER); + } + + /** + * Creates a builder for compile order nested sets. + */ + public static <E> NestedSetBuilder<E> compileOrder() { + return new NestedSetBuilder<>(Order.COMPILE_ORDER); + } + + /** + * Creates a builder for link order nested sets. + */ + public static <E> NestedSetBuilder<E> linkOrder() { + return new NestedSetBuilder<>(Order.LINK_ORDER); + } + + /** + * Creates a builder for naive link order nested sets. + */ + public static <E> NestedSetBuilder<E> naiveLinkOrder() { + return new NestedSetBuilder<>(Order.NAIVE_LINK_ORDER); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetExpander.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetExpander.java new file mode 100644 index 0000000..c04c39d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetExpander.java
@@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableCollection; + +/** + * An expander that converts a nested set into a flattened collection. + * + * <p>Expanders are initialized statically (there is one for each order), so they should + * contain no state and all methods must be threadsafe. + */ +interface NestedSetExpander<E> { + /** + * Flattens the NestedSet into the builder. + */ + void expandInto(NestedSet<E> nestedSet, Uniqueifier uniqueifier, + ImmutableCollection.Builder<E> builder); +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetFactory.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetFactory.java new file mode 100644 index 0000000..99fb8bb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetFactory.java
@@ -0,0 +1,55 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableList; + +/** + * Factory methods for creating {@link NestedSet}s of specific shapes. This allows the + * implementation to be memory efficient (e.g. a specialized implementation for the case where + * there are only direct elements, etc). + * + * <p>It's intended for each {@link Order} to have its own factory implementation. That way we can + * be even more efficient since the {@link NestedSet}s instances don't need to store their + * {@link Order}. + */ +interface NestedSetFactory { + + /** Create a NestedSet with just one direct element and not transitive elements. */ + <E> NestedSet<E> oneDirect(E element); + + /** Create a NestedSet with only direct elements. */ + <E> NestedSet<E> onlyDirects(Object[] directs); + + /** Create a NestedSet with only direct elements potentially sharing the ImmutableList. */ + <E> NestedSet<E> onlyDirects(ImmutableList<E> directs); + + /** Create a NestedSet with one direct element and one transitive {@code NestedSet}. */ + <E> NestedSet<E> oneDirectOneTransitive(E direct, NestedSet<E> transitive); + + /** Create a NestedSet with many direct elements and one transitive {@code NestedSet}. */ + <E> NestedSet<E> manyDirectsOneTransitive(Object[] direct, NestedSet<E> transitive); + + /** Create a NestedSet with no direct elements and one transitive {@code NestedSet.} */ + <E> NestedSet<E> onlyOneTransitive(NestedSet<E> transitive); + + /** Create a NestedSet with no direct elements and many transitive {@code NestedSet}s. */ + <E> NestedSet<E> onlyManyTransitives(NestedSet[] transitives); + + /** Create a NestedSet with one direct elements and many transitive {@code NestedSet}s. */ + <E> NestedSet<E> oneDirectManyTransitive(Object direct, NestedSet[] transitive); + + /** Create a NestedSet with many direct elements and many transitive {@code NestedSet}s. */ + <E> NestedSet<E> manyDirectManyTransitive(Object[] directs, NestedSet[] transitive); +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetLazyIterator.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetLazyIterator.java new file mode 100644 index 0000000..c873d56 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetLazyIterator.java
@@ -0,0 +1,53 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import java.util.Iterator; + +/** + * A NestedSet iterator that only expands the NestedSet when the first element is requested. This + * allows code that calls unconditionally to {@code hasNext} to check if the iterator is empty + * to not expand the nested set. + */ +final class NestedSetLazyIterator<E> implements Iterator<E> { + + private NestedSet<E> nestedSet; + private Iterator<E> delegate = null; + + NestedSetLazyIterator(NestedSet<E> nestedSet) { + this.nestedSet = nestedSet; + } + + @Override + public boolean hasNext() { + if (delegate == null) { + return !nestedSet.isEmpty(); + } + return delegate.hasNext(); + } + + @Override + public E next() { + if (delegate == null) { + delegate = nestedSet.toCollection().iterator(); + nestedSet = null; + } + return delegate.next(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetVisitor.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetVisitor.java new file mode 100644 index 0000000..ea8b810 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetVisitor.java
@@ -0,0 +1,96 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; + +import java.util.Set; + +/** + * NestedSetVisitor facilitates a transitive visitation over a NestedSet, which must be in STABLE + * order. The callback may be called from multiple threads, and must be thread-safe. + * + * <p>The visitation is iterative: The caller may invoke a NestedSet within the top-level NestedSet + * in any order. + * + * <p>Currently this class is only used in Skyframe to facilitate iterative replay of transitive + * warnings/errors. + * + * @param <E> the data type + */ +// @ThreadSafety.ThreadSafe +public final class NestedSetVisitor<E> { + + /** + * For each element of the NestedSet the {@code Reciver} will receive one element during the + * visitation. + */ + public interface Receiver<E> { + void accept(E arg); + } + + private final Receiver<E> callback; + + private final VisitedState<E> visited; + + public NestedSetVisitor(Receiver<E> callback, VisitedState<E> visited) { + this.callback = Preconditions.checkNotNull(callback); + this.visited = Preconditions.checkNotNull(visited); + } + + /** + * Transitively visit a nested set. + * + * @param nestedSet the nested set to visit transitively. + * + */ + @SuppressWarnings("unchecked") + public void visit(NestedSet<E> nestedSet) { + // This method suppresses the unchecked warning so that it can access the internal NestedSet + // raw structure. + Preconditions.checkArgument(nestedSet.getOrder() == Order.STABLE_ORDER); + if (!visited.add(nestedSet)) { + return; + } + + for (NestedSet<E> subset : nestedSet.transitiveSets()) { + visit(subset); + } + for (Object member : nestedSet.directMembers()) { + if (visited.add((E) member)) { + callback.accept((E) member); + } + } + } + + /** A class that allows us to keep track of the seen nodes and transitive sets. */ + public static class VisitedState<E> { + private final Set<NestedSet<E>> seenSets = Sets.newConcurrentHashSet(); + private final Set<E> seenNodes = Sets.newConcurrentHashSet(); + + public void clear() { + seenSets.clear(); + seenNodes.clear(); + } + + private boolean add(E node) { + return seenNodes.add(node); + } + + private boolean add(NestedSet<E> set) { + return seenSets.add(set); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/OneDirectManyTransitive.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OneDirectManyTransitive.java new file mode 100644 index 0000000..a99883c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OneDirectManyTransitive.java
@@ -0,0 +1,63 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import java.util.Arrays; +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * Memory-optimized NestedSet implementation for NestedSets with one direct element and + * many transitive dependencies. + */ +abstract class OneDirectManyTransitive<E> extends MemoizedUniquefierNestedSet<E> { + + private final Object direct; + private final NestedSet[] transitives; + private Object memo; + + OneDirectManyTransitive(Object direct, NestedSet[] transitives) { + this.direct = direct; + this.transitives = transitives; + } + + @Override + Object getMemo() { return memo; } + + @Override + void setMemo(Object memo) { this.memo = memo; } + + @Override + Object[] directMembers() { return new Object[]{direct}; } + + @Override + NestedSet[] transitiveSets() { return transitives; } + + @Override + public boolean shallowEquals(@Nullable NestedSet<? extends E> other) { + if (this == other) { + return true; + } + return other != null + && getOrder().equals(other.getOrder()) + && other instanceof OneDirectManyTransitive + && direct.equals(((OneDirectManyTransitive) other).direct) + && Arrays.equals(transitives, ((OneDirectManyTransitive) other).transitives); + } + + @Override + public int shallowHashCode() { + return Objects.hash(getOrder(), direct, Arrays.hashCode(transitives)); } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/OneDirectOneTransitiveNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OneDirectOneTransitiveNestedSet.java new file mode 100644 index 0000000..9acdba1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OneDirectOneTransitiveNestedSet.java
@@ -0,0 +1,61 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * Memory-efficient implementation for the case where we have one direct element and one + * transitive NestedSet. + */ +abstract class OneDirectOneTransitiveNestedSet<E> extends MemoizedUniquefierNestedSet<E> { + + private final E direct; + private final NestedSet<E> transitive; + private Object memo; + + OneDirectOneTransitiveNestedSet(E direct, NestedSet<E> transitive) { + this.direct = direct; + this.transitive = transitive; + } + + @Override + Object getMemo() { return memo; } + + @Override + void setMemo(Object memo) { this.memo = memo; } + + @Override + Object[] directMembers() { return new Object[]{direct}; } + + @Override + NestedSet[] transitiveSets() { return new NestedSet[]{transitive}; } + + @Override + public boolean shallowEquals(@Nullable NestedSet<? extends E> other) { + if (this == other) { + return true; + } + return other != null + && getOrder().equals(other.getOrder()) + && other instanceof OneDirectOneTransitiveNestedSet + && direct.equals(((OneDirectOneTransitiveNestedSet) other).direct) + && transitive == ((OneDirectOneTransitiveNestedSet) other).transitive; + } + + @Override + public int shallowHashCode() { return Objects.hash(getOrder(), direct, transitive); } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyDirectsNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyDirectsNestedSet.java new file mode 100644 index 0000000..9f7588d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyDirectsNestedSet.java
@@ -0,0 +1,88 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Memory-optimized NestedSet implementation for NestedSets without transitive dependencies. + * + */ +abstract class OnlyDirectsNestedSet<E> extends NestedSet<E> { + + private static final NestedSet[] EMPTY = new NestedSet[0]; + private final Object[] directDeps; + + public OnlyDirectsNestedSet(Object[] directDeps) { this.directDeps = directDeps; } + + @Override + public abstract Order getOrder(); + + @Override + Object[] directMembers() { + return directDeps; + } + + @Override + NestedSet[] transitiveSets() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return false; + } + + /** + * Currently all the Order implementations return the direct elements in the same order if they + * do not have transitive elements. So we skip calling order.getExpander()... and return a view + * of the array. + */ + @SuppressWarnings("unchecked") + @Override + public List<E> toList() { + return (List<E>) ImmutableList.copyOf(directDeps); + } + + @SuppressWarnings("unchecked") + @Override + public Set<E> toSet() { + return (Set<E>) ImmutableSet.copyOf(directDeps); + } + + @Override + public boolean shallowEquals(@Nullable NestedSet<? extends E> other) { + if (this == other) { + return true; + } + if (other == null) { + return false; + } + return getOrder().equals(other.getOrder()) + && other instanceof OnlyDirectsNestedSet + && Arrays.equals(directDeps, ((OnlyDirectsNestedSet) other).directDeps); + } + + @Override + public int shallowHashCode() { + return Arrays.hashCode(directDeps); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyOneTransitiveNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyOneTransitiveNestedSet.java new file mode 100644 index 0000000..a3e2d1d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyOneTransitiveNestedSet.java
@@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * Memory-efficient implementation for the case where we only have one transitive NestedSet. + * + * <p>Note that we cannot simply delegate to the inner NestedSet because the order could be + * different and the top-level order is the correct one. + */ +abstract class OnlyOneTransitiveNestedSet<E> extends MemoizedUniquefierNestedSet<E> { + + private static final Object[] EMPTY = new Object[0]; + + private final NestedSet<E> transitive; + private Object memo; + + public OnlyOneTransitiveNestedSet(NestedSet<E> transitive) { + this.transitive = transitive; + } + + @Override + Object getMemo() { return memo; } + + @Override + void setMemo(Object memo) { this.memo = memo; } + + @Override + Object[] directMembers() { + return EMPTY; + } + + @Override + NestedSet[] transitiveSets() { + return new NestedSet[]{transitive}; + } + + @Override + public boolean shallowEquals(@Nullable NestedSet<? extends E> other) { + if (this == other) { + return true; + } + return other != null + && getOrder().equals(other.getOrder()) + && other instanceof OnlyOneTransitiveNestedSet + && transitive == ((OnlyOneTransitiveNestedSet) other).transitive; + } + + @Override + public int shallowHashCode() { + return Objects.hash(getOrder(), transitive); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyTransitivesNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyTransitivesNestedSet.java new file mode 100644 index 0000000..5a57053 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/OnlyTransitivesNestedSet.java
@@ -0,0 +1,62 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import java.util.Arrays; +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * Memory-efficient implementation for the case where we have one direct element and one + * transitive NestedSet. + */ +abstract class OnlyTransitivesNestedSet<E> extends MemoizedUniquefierNestedSet<E> { + + private static final NestedSet[] EMPTY = new NestedSet[0]; + + private final NestedSet[] transitives; + private Object memo; + + OnlyTransitivesNestedSet(NestedSet[] transitives) { + this.transitives = transitives; + } + + @Override + Object getMemo() { return memo; } + + @Override + void setMemo(Object memo) { this.memo = memo; } + + @Override + Object[] directMembers() { return EMPTY; } + + @Override + NestedSet[] transitiveSets() { return transitives; } + + @Override + public boolean shallowEquals(@Nullable NestedSet<? extends E> other) { + if (this == other) { + return true; + } + return other != null + && getOrder().equals(other.getOrder()) + && other instanceof OnlyTransitivesNestedSet + && Arrays.equals(transitives, ((OnlyTransitivesNestedSet) other).transitives); + } + + @Override + public int shallowHashCode() { + return Objects.hash(getOrder(), Arrays.hashCode(transitives)); } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/Order.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/Order.java new file mode 100644 index 0000000..38e6633 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/Order.java
@@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +/** + * Type of a nested set (defines order). + */ +public enum Order { + + STABLE_ORDER(new CompileOrderExpander<>(), new StableOrderNestedSetFactory()), + COMPILE_ORDER(new CompileOrderExpander<>(), new CompileOrderNestedSetFactory()), + LINK_ORDER(new LinkOrderExpander<>(), new LinkOrderNestedSetFactory()), + NAIVE_LINK_ORDER(new NaiveLinkOrderExpander<>(), new NaiveLinkOrderNestedSetFactory()); + + private final NestedSetExpander<?> expander; + final NestedSetFactory factory; + private final NestedSet<?> emptySet; + + private Order(NestedSetExpander<?> expander, NestedSetFactory factory) { + this.expander = expander; + this.factory = factory; + this.emptySet = new EmptyNestedSet<Object>(this); + } + + /** + * Returns an empty set of the given ordering. + */ + @SuppressWarnings("unchecked") // Nested sets are immutable, so a downcast is fine. + <E> NestedSet<E> emptySet() { + return (NestedSet<E>) emptySet; + } + + /** + * Returns an empty set of the given ordering. + */ + @SuppressWarnings("unchecked") // Nested set expanders contain no data themselves. + <E> NestedSetExpander<E> expander() { + return (NestedSetExpander<E>) expander; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/RecordingUniqueifier.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/RecordingUniqueifier.java new file mode 100644 index 0000000..c71f200 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/RecordingUniqueifier.java
@@ -0,0 +1,140 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; + +import java.util.BitSet; +import java.util.HashSet; +import java.util.Set; + +/** + * A Uniqueifier that records a transcript of its interactions with the underlying Set. A memo can + * then be retrieved in the form of another Uniqueifier, and given the same sequence of isUnique + * calls, the Set interactions can be avoided. + */ +class RecordingUniqueifier implements Uniqueifier { + /** + * Unshared byte memos under this length are not constructed. + */ + @VisibleForTesting static final int LENGTH_THRESHOLD = 4096 / 8; // bits -> bytes + + /** + * Returned as a marker that memoization was not worthwhile. + */ + private static final Object NO_MEMO = new Object(); + + /** + * Shared one-byte memos. + */ + private static final byte[][] SHARED_SMALL_MEMOS_1; + + /** + * Shared two-byte memos. + */ + private static final byte[][] SHARED_SMALL_MEMOS_2; + + static { + // Create interned arrays for one and two byte memos + // The memos always start with 0x3, so some one and two byte arrays can be skipped. + + byte[][] memos1 = new byte[64][1]; + for (int i = 0; i < 64; i++) { + memos1[i][0] = (byte) ((i << 2) | 0x3); + } + SHARED_SMALL_MEMOS_1 = memos1; + + byte[][] memos2 = new byte[16384][2]; + for (int i = 0; i < 64; i++) { + byte iAdj = (byte) (0x3 | (i << 2)); + for (int j = 0; j < 256; j++) { + int idx = i | (j << 6); + memos2[idx][0] = iAdj; + memos2[idx][1] = (byte) j; + } + } + SHARED_SMALL_MEMOS_2 = memos2; + } + + private final Set<Object> witnessed = new HashSet<>(256); + private final BitSet memo = new BitSet(); + private int idx = 0; + + static Uniqueifier createReplayUniqueifier(Object memo) { + if (memo == NO_MEMO) { + return new RecordingUniqueifier(); + } else if (memo instanceof Integer) { + BitSet bs = new BitSet(); + bs.set(0, (Integer) memo); + return new ReplayUniqueifier(bs); + } + return new ReplayUniqueifier(BitSet.valueOf((byte[]) memo)); + } + + @Override + public boolean isUnique(Object o) { + boolean firstInstance = witnessed.add(o); + memo.set(idx++, firstInstance); + return firstInstance; + } + + /** + * Gets the memo of the set interactions. Do not call isUnique after this point. + */ + Object getMemo() { + this.idx = -1; // will cause failures if isUnique is called after getMemo. + + // If the bitset is just a contiguous block of ones, use a length memo + int length = memo.length(); + if (memo.cardinality() == length) { + return length; // length-based memo + } + + byte[] ba = memo.toByteArray(); + + Preconditions.checkState( + (length < 2) || ((ba[0] & 3) == 3), + "The memo machinery expects memos to always begin with two 1 bits, " + + "but instead, this memo starts with %X.", ba[0]); + + // For short memos, use an interned array for the memo + if (ba.length == 1) { + return SHARED_SMALL_MEMOS_1[(0xFF & ba[0]) >>> 2]; // shared memo + } else if (ba.length == 2) { + return SHARED_SMALL_MEMOS_2[((ba[1] & 0xFF) << 6) | ((0xFF & ba[0]) >>> 2)]; + } + + // For mid-sized cases, skip the memo since it is not worthwhile + if (ba.length < LENGTH_THRESHOLD) { + return NO_MEMO; // skipped memo + } + + return ba; // normal memo + } + + private static final class ReplayUniqueifier implements Uniqueifier { + private final BitSet memo; + private int idx = 0; + + ReplayUniqueifier(BitSet memo) { + this.memo = memo; + } + + @Override + public boolean isUnique(Object o) { + return memo.get(idx++); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/SingleDirectNestedSet.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/SingleDirectNestedSet.java new file mode 100644 index 0000000..dc4a5fb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/SingleDirectNestedSet.java
@@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterators; + +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Memory-efficient implementation for nested sets with one element. + */ +public abstract class SingleDirectNestedSet<E> extends NestedSet<E> { + + private static final NestedSet[] EMPTY = new NestedSet[0]; + private final E e; + + public SingleDirectNestedSet(E e) { this.e = Preconditions.checkNotNull(e); } + + @Override + public Iterator<E> iterator() { return Iterators.singletonIterator(e); } + + @Override + Object[] directMembers() { return new Object[]{e}; } + + @Override + NestedSet[] transitiveSets() { return EMPTY; } + + @Override + public boolean isEmpty() { return false; } + + @Override + public List<E> toList() { return ImmutableList.of(e); } + + @Override + public Set<E> toSet() { return ImmutableSet.of(e); } + + @Override + public boolean shallowEquals(@Nullable NestedSet<? extends E> other) { + if (this == other) { + return true; + } + return other instanceof SingleDirectNestedSet + && e.equals(((SingleDirectNestedSet) other).e); + } + + @Override + public int shallowHashCode() { + return e.hashCode(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/StableOrderNestedSetFactory.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/StableOrderNestedSetFactory.java new file mode 100644 index 0000000..cd0b618 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/StableOrderNestedSetFactory.java
@@ -0,0 +1,152 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableList; + +/** + * Stable order {@code NestedSet} factory. + */ +final class StableOrderNestedSetFactory implements NestedSetFactory { + + @Override + public <E> NestedSet<E> onlyDirects(Object[] directs) { + return new StableOnlyDirectsNestedSet<>(directs); + } + + @Override + public <E> NestedSet<E> onlyDirects(ImmutableList<E> directs) { + return new StableImmutableListDirectsNestedSet<>(directs); + } + + @Override + public <E> NestedSet<E> oneDirectOneTransitive(E direct, NestedSet<E> transitive) { + return new StableOneDirectOneTransitiveNestedSet<>(direct, transitive); + } + + @Override + public <E> NestedSet<E> manyDirectsOneTransitive(Object[] direct, + NestedSet<E> transitive) { + return new StableManyDirectOneTransitiveNestedSet<>(direct, transitive); + } + + @Override + public <E> NestedSet<E> onlyOneTransitive(NestedSet<E> transitive) { + return new StableOnlyOneTransitiveNestedSet<>(transitive); + } + + @Override + public <E> NestedSet<E> onlyManyTransitives(NestedSet[] transitives) { + return new StableOnlyTransitivesNestedSet<>(transitives); + } + + @Override + public <E> NestedSet<E> oneDirectManyTransitive(Object direct, NestedSet[] transitives) { + return new StableOneDirectManyTransitive<>(direct, transitives); + } + + @Override + public <E> NestedSet<E> manyDirectManyTransitive(Object[] directs, NestedSet[] transitives) { + return new StableManyDirectManyTransitive<>(directs, transitives); + } + + @Override + public <E> NestedSet<E> oneDirect(final E element) { + return new StableSingleDirectNestedSet<>(element); + } + + private static class StableOnlyDirectsNestedSet<E> extends OnlyDirectsNestedSet<E> { + + StableOnlyDirectsNestedSet(Object[] directs) { super(directs); } + + @Override + public Order getOrder() { return Order.STABLE_ORDER; } + } + + private static class StableOneDirectOneTransitiveNestedSet<E> extends + OneDirectOneTransitiveNestedSet<E> { + + private StableOneDirectOneTransitiveNestedSet(E direct, NestedSet<E> transitive) { + super(direct, transitive); + } + + @Override + public Order getOrder() { return Order.STABLE_ORDER; } + } + + private static class StableOneDirectManyTransitive<E> extends OneDirectManyTransitive<E> { + + private StableOneDirectManyTransitive(Object direct, NestedSet[] transitive) { + super(direct, transitive); + } + + @Override + public Order getOrder() { return Order.STABLE_ORDER; } + } + + private static class StableManyDirectManyTransitive<E> extends ManyDirectManyTransitive<E> { + + private StableManyDirectManyTransitive(Object[] directs, NestedSet[] transitives) { + super(directs, transitives); + } + + @Override + public Order getOrder() { return Order.STABLE_ORDER; } + } + + private static class StableOnlyOneTransitiveNestedSet<E> extends OnlyOneTransitiveNestedSet<E> { + + private StableOnlyOneTransitiveNestedSet(NestedSet<E> transitive) { super(transitive); } + + @Override + public Order getOrder() { return Order.STABLE_ORDER; } + } + + private static class StableManyDirectOneTransitiveNestedSet<E> extends + ManyDirectOneTransitiveNestedSet<E> { + + private StableManyDirectOneTransitiveNestedSet(Object[] direct, + NestedSet<E> transitive) { super(direct, transitive); } + + @Override + public Order getOrder() { return Order.STABLE_ORDER; } + } + + private static class StableOnlyTransitivesNestedSet<E> extends OnlyTransitivesNestedSet<E> { + + private StableOnlyTransitivesNestedSet(NestedSet[] transitives) { super(transitives); } + + @Override + public Order getOrder() { return Order.STABLE_ORDER; } + } + + private static class StableImmutableListDirectsNestedSet<E> extends + ImmutableListDirectsNestedSet<E> { + + private StableImmutableListDirectsNestedSet(ImmutableList<E> directs) { super(directs); } + + @Override + public Order getOrder() { + return Order.STABLE_ORDER; + } + } + + private static class StableSingleDirectNestedSet<E> extends SingleDirectNestedSet<E> { + + private StableSingleDirectNestedSet(E element) { super(element); } + + @Override + public Order getOrder() { return Order.STABLE_ORDER; } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/Uniqueifier.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/Uniqueifier.java new file mode 100644 index 0000000..9421a57 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/Uniqueifier.java
@@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +/** + * Helps reduce a sequence of potentially duplicated elements to a sequence of unique elements. + */ +interface Uniqueifier { + + /** + * Returns true if-and-only-if this is the first time that this {@link Uniqueifier}'s method has + * been called with this Object. This uses Object.equals-type equality for the comparison. + */ + public boolean isUnique(Object o); +}
diff --git a/src/main/java/com/google/devtools/build/lib/concurrent/AbstractQueueVisitor.java b/src/main/java/com/google/devtools/build/lib/concurrent/AbstractQueueVisitor.java new file mode 100644 index 0000000..a937d17 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/concurrent/AbstractQueueVisitor.java
@@ -0,0 +1,578 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.concurrent; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.collect.Maps; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * AbstractQueueVisitor is a wrapper around {@link ThreadPoolExecutor} which + * delays thread pool shutdown until entire visitation is complete. + * This is useful for cases in which worker tasks may submit additional tasks. + * + * <p>Consider the following example: + * <pre> + * ThreadPoolExecutor executor = <...> + * executor.submit(myRunnableTask); + * executor.shutdown(); + * executor.awaitTermination(); + * </pre> + * + * <p>This won't work properly if {@code myRunnableTask} submits additional + * tasks to the executor, because it may already have shut down + * by that point. + * + * <p>AbstractQueueVisitor supports interruption. If the main thread is + * interrupted, tasks will no longer be added to the queue, and the + * {@link #work(boolean)} method will throw {@link InterruptedException}. + */ +public class AbstractQueueVisitor { + + /** + * The first unhandled exception thrown by a worker thread. We save it + * and re-throw it from the main thread to detect bugs faster; + * otherwise worker threads just quietly die. + * + * Field updates are synchronized; it's + * important to save the first one as it may be more informative than a + * subsequent one, and this is not a performance-critical path. + */ + private Throwable unhandled = null; + + /** + * An uncaught exception when submitting a job to the ThreadPool is catastrophic, and usually + * indicates a lack of stack space on which to allocate a native thread. The JDK + * ThreadPoolExecutor may reach an inconsistent state in such circumstances, so we avoid blocking + * on its termination when this field is non-null. + */ + private volatile Throwable catastrophe; + + /** + * Enables concurrency. For debugging or testing, set this to false + * to avoid thread creation and concurrency. Any deviation in observed + * behaviour is a bug. + */ + private final boolean concurrent; + + // Condition variable for remainingTasks==0, and a lock for it. + private final Object zeroRemainingTasks = new Object(); + private long remainingTasks = 0; + + // Map of thread ==> number of jobs executing in the thread. + // Currently used only for interrupt handling. + private final Map<Thread, Long> jobs = Maps.newConcurrentMap(); + + /** + * The thread pool. If !concurrent, always null. Created lazily on first + * call to {@link #enqueue(Runnable)}, and removed after call to + * {@link #work(boolean)}. + */ + private final ThreadPoolExecutor pool; + + /** + * Flag used to record when the main thread (the thread which called + * {@link #work(boolean)}) is interrupted. + * + * When this is true, adding tasks to the thread pool will + * fail quietly as a part of the process of shutting down the + * worker threads. + */ + private volatile boolean threadInterrupted = false; + + /** + * Latches used to signal when the visitor has been interrupted or + * seen an exception. Used only for testing. + */ + private final CountDownLatch interruptedLatch = new CountDownLatch(1); + private final CountDownLatch exceptionLatch = new CountDownLatch(1); + + /** + * If true, don't run new actions after an uncaught exception. + */ + private final boolean failFastOnException; + + /** + * If true, don't run new actions after an interrupt. + */ + private final boolean failFastOnInterrupt; + + /** + * If true, we must shut down the thread pool on completion. + */ + private final boolean ownThreadPool; + + /** + * Flag used to record when all threads were killed by failed action execution + */ + private boolean jobsMustBeStopped = false; + + /** + * Create the AbstractQueueVisitor. + * + * @param concurrent true if concurrency should be enabled. Only set to + * false for debugging. + * @param corePoolSize the core pool size of the thread pool. See + * {@link ThreadPoolExecutor#ThreadPoolExecutor(int, int, long, TimeUnit, java.util.concurrent.BlockingQueue)} + * @param maxPoolSize the max number of threads in the pool. + * @param keepAliveTime the keep-alive time for the thread pool. + * @param units the time units of keepAliveTime. + * @param failFastOnException if true, don't run new actions after + * an uncaught exception. + * @param failFastOnInterrupt if true, don't run new actions after interrupt. + * @param poolName sets the name of threads spawn by this thread pool. If {@code null}, default + * thread naming will be used. + */ + public AbstractQueueVisitor(boolean concurrent, int corePoolSize, int maxPoolSize, + long keepAliveTime, TimeUnit units, boolean failFastOnException, + boolean failFastOnInterrupt, String poolName) { + Preconditions.checkNotNull(poolName); + + this.concurrent = concurrent; + this.failFastOnException = failFastOnException; + this.failFastOnInterrupt = failFastOnInterrupt; + this.ownThreadPool = true; + this.pool = concurrent + ? new ThreadPoolExecutor(corePoolSize, maxPoolSize, keepAliveTime, units, getWorkQueue(), + new ThreadFactoryBuilder().setNameFormat(poolName + " %d").build()) + : null; + } + + /** + * Create the AbstractQueueVisitor. + * + * @param concurrent true if concurrency should be enabled. Only set to + * false for debugging. + * @param corePoolSize the core pool size of the thread pool. See + * {@link ThreadPoolExecutor#ThreadPoolExecutor(int, int, long, TimeUnit, java.util.concurrent.BlockingQueue)} + * @param maxPoolSize the max number of threads in the pool. + * @param keepAliveTime the keep-alive time for the thread pool. + * @param units the time units of keepAliveTime. + * @param failFastOnException if true, don't run new actions after + * an uncaught exception. + * @param poolName sets the name of threads spawn by this thread pool. If {@code null}, default + * thread naming will be used. + */ + public AbstractQueueVisitor(boolean concurrent, int corePoolSize, int maxPoolSize, + long keepAliveTime, TimeUnit units, boolean failFastOnException, String poolName) { + this(concurrent, corePoolSize, maxPoolSize, keepAliveTime, units, failFastOnException, true, + poolName); + } + + /** + * Create the AbstractQueueVisitor. + * + * @param executor The ThreadPool to use. + * @param shutdownOnCompletion If true, pass ownership of the Threadpool to + * this class. The pool will be shut down after a + * call to work(). Callers must not shutdown the + * threadpool while queue visitors use it. + * @param failFastOnException if true, don't run new actions after + * an uncaught exception. + * @param failFastOnInterrupt if true, don't run new actions after interrupt. + */ + public AbstractQueueVisitor(ThreadPoolExecutor executor, boolean shutdownOnCompletion, + boolean failFastOnException, boolean failFastOnInterrupt) { + this(/*concurrent=*/true, executor, shutdownOnCompletion, failFastOnException, + failFastOnInterrupt); + } + + /** + * Create the AbstractQueueVisitor. + * + * @param concurrent if false, run tasks inline instead of using the thread pool. + * @param executor The ThreadPool to use. + * @param shutdownOnCompletion If true, pass ownership of the Threadpool to + * this class. The pool will be shut down after a + * call to work(). Callers must not shut down the + * threadpool while queue visitors use it. + * @param failFastOnException if true, don't run new actions after + * an uncaught exception. + * @param failFastOnInterrupt if true, don't run new actions after interrupt. + */ + public AbstractQueueVisitor(boolean concurrent, ThreadPoolExecutor executor, + boolean shutdownOnCompletion, boolean failFastOnException, + boolean failFastOnInterrupt) { + this.concurrent = concurrent; + this.failFastOnException = failFastOnException; + this.failFastOnInterrupt = failFastOnInterrupt; + this.pool = executor; + this.ownThreadPool = shutdownOnCompletion; + } + + public AbstractQueueVisitor(ThreadPoolExecutor executor, boolean failFastOnException) { + this(executor, true, failFastOnException, true); + } + + /** + * Create the AbstractQueueVisitor. + * + * @param concurrent true if concurrency should be enabled. Only set to + * false for debugging. + * @param corePoolSize the core pool size of the thread pool. See + * {@link ThreadPoolExecutor#ThreadPoolExecutor(int, int, long, TimeUnit, java.util.concurrent.BlockingQueue)} + * @param maxPoolSize the max number of threads in the pool. + * @param keepAliveTime the keep-alive time for the thread pool. + * @param units the time units of keepAliveTime. + * @param poolName sets the name of threads spawn by this thread pool. If {@code null}, default + * thread naming will be used. + */ + public AbstractQueueVisitor(boolean concurrent, int corePoolSize, int maxPoolSize, + long keepAliveTime, TimeUnit units, String poolName) { + this(concurrent, corePoolSize, maxPoolSize, keepAliveTime, units, false, poolName); + } + + /** + * Create the AbstractQueueVisitor with concurrency enabled. + * + * @param corePoolSize the core pool size of the thread pool. See + * {@link ThreadPoolExecutor#ThreadPoolExecutor(int, int, long, TimeUnit, java.util.concurrent.BlockingQueue)} + * @param maxPoolSize the max number of threads in the pool. + * @param keepAlive the keep-alive time for the thread pool. + * @param units the time units of keepAliveTime. + * @param poolName sets the name of threads spawn by this thread pool. If {@code null}, default + * thread naming will be used. + */ + public AbstractQueueVisitor(int corePoolSize, int maxPoolSize, long keepAlive, TimeUnit units, + String poolName) { + this(true, corePoolSize, maxPoolSize, keepAlive, units, poolName); + } + + protected BlockingQueue<Runnable> getWorkQueue() { + return new LinkedBlockingQueue<>(); + } + + /** + * Executes all tasks on the queue, and optionally shuts the pool down and deletes it. + * + * <p>Throws (the same) unchecked exception if any worker thread failed unexpectedly. If the pool + * is interrupted and a worker also throws an unchecked exception, the unchecked exception is + * rethrown, since it may indicate a programming bug. If callers handle the unchecked exception, + * they may check the interrupted bit to see if the pool was interrupted. + * + * @param interruptWorkers if true, interrupt worker threads when main thread gets an interrupt. + * If false, just wait for them to terminate normally. + */ + protected void work(boolean interruptWorkers) throws InterruptedException { + if (concurrent) { + awaitTermination(interruptWorkers); + } else { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException(); + } + } + } + + /** + * Schedules a call. + * Called in a worker thread if concurrent. + */ + protected void enqueue(Runnable runnable) { + if (concurrent) { + AtomicBoolean ranTask = new AtomicBoolean(false); + try { + pool.execute(wrapRunnable(runnable, ranTask)); + } catch (Throwable e) { + if (!ranTask.get()) { + // Note that keeping track of ranTask is necessary to disambiguate the case where + // execute() itself failed, vs. a caller-runs policy on pool exhaustion, where the + // runnable threw. To be extra cautious, we decrement the task count in a finally + // block, even though the CountDownLatch is unlikely to throw. + recordError(e); + } + } + } else { + runnable.run(); + } + } + + private void recordError(Throwable e) { + catastrophe = e; + try { + synchronized (this) { + if (unhandled == null) { // save only the first one. + unhandled = e; + exceptionLatch.countDown(); + } + } + } finally { + decrementRemainingTasks(); + } + } + + private Runnable wrapRunnable(final Runnable runnable, final AtomicBoolean ranTask) { + synchronized (zeroRemainingTasks) { + remainingTasks++; + } + return new Runnable() { + @Override + public void run() { + Thread thread = null; + boolean addedJob = false; + try { + ranTask.set(true); + thread = Thread.currentThread(); + addJob(thread); + addedJob = true; + if (blockNewActions()) { + // Make any newly enqueued tasks quickly die. We check after adding to the jobs map so + // that if another thread is racing to kill this thread and didn't make it before this + // conditional, it will be able to find and kill this thread anyway. + return; + } + runnable.run(); + } catch (Throwable e) { + synchronized (AbstractQueueVisitor.this) { + if (unhandled == null) { // save only the first one. + unhandled = e; + exceptionLatch.countDown(); + } + markToStopAllJobsIfNeeded(e); + } + } finally { + try { + if (thread != null && addedJob) { + removeJob(thread); + } + } finally { + decrementRemainingTasks(); + } + } + } + }; + } + + private final void addJob(Thread thread) { + // Note: this looks like a check-then-act race but it isn't, because each + // key implies thread-locality. + long count = jobs.containsKey(thread) ? jobs.get(thread) + 1 : 1; + jobs.put(thread, count); + } + + private final void removeJob(Thread thread) { + Long boxedCount = Preconditions.checkNotNull(jobs.get(thread), + "Can't retrieve job after successfully adding it"); + long count = boxedCount - 1; + if (count == 0) { + jobs.remove(thread); + } else { + jobs.put(thread, count); + } + } + + /** + * Set an internal flag to show that an interrupt was detected. + */ + private void setInterrupted() { + threadInterrupted = true; + setRejectedExecutionHandler(); + } + + private final void decrementRemainingTasks() { + synchronized (zeroRemainingTasks) { + if (--remainingTasks == 0) { + zeroRemainingTasks.notify(); + } + } + } + + /** + * If this returns true, don't enqueue new actions. + */ + protected boolean blockNewActions() { + return (failFastOnInterrupt && isInterrupted()) || (unhandled != null && failFastOnException); + } + + /** + * Await interruption. Used only in tests. + */ + @VisibleForTesting + public boolean awaitInterruptionForTestingOnly(long timeout, TimeUnit units) + throws InterruptedException { + return interruptedLatch.await(timeout, units); + } + + /** Get latch that is released when exception is received by visitor. Used only in tests. */ + @VisibleForTesting + public CountDownLatch getExceptionLatchForTestingOnly() { + return exceptionLatch; + } + + /** Get latch that is released when interruption is received by visitor. Used only in tests. */ + @VisibleForTesting + public CountDownLatch getInterruptionLatchForTestingOnly() { + return interruptedLatch; + } + + /** + * Get the value of the interrupted flag. + */ + @ThreadSafety.ThreadSafe + protected boolean isInterrupted() { + return threadInterrupted; + } + + /** + * Get number of jobs remaining. Note that this can increase in value + * if running tasks submit further jobs. + */ + @VisibleForTesting + protected long getTaskCount() { + synchronized (zeroRemainingTasks) { + return remainingTasks; + } + } + + /** + * Waits for the task queue to drain, then shuts down the thread pool and + * waits for it to terminate. Throws (the same) unchecked exception if any + * worker thread failed unexpectedly. + */ + private void awaitTermination(boolean interruptWorkers) throws InterruptedException { + Preconditions.checkState(failFastOnInterrupt || !interruptWorkers); + Throwables.propagateIfPossible(catastrophe); + try { + synchronized (zeroRemainingTasks) { + while (remainingTasks != 0 && !jobsMustBeStopped) { + zeroRemainingTasks.wait(); + } + } + } catch (InterruptedException e) { + // Mark the visitor, so that it's known to be interrupted, and + // then break out of here, stop the worker threads and return ASAP, + // sending the interruption to the parent thread. + setInterrupted(); + } + + reallyAwaitTermination(interruptWorkers); + + if (isInterrupted()) { + // Set interrupted bit on current thread so that callers can see that it was interrupted. Note + // that if the thread was interrupted while awaiting termination, we might not hit this + // codepath, but then the current thread's interrupt bit is already set, so we are fine. + Thread.currentThread().interrupt(); + } + // Throw the first unhandled (worker thread) exception in the main thread. We throw an unchecked + // exception instead of InterruptedException if both are present because an unchecked exception + // may indicate a catastrophic failure that should shut down the program. The caller can + // check the interrupted bit if they will handle the unchecked exception without crashing. + Throwables.propagateIfPossible(unhandled); + + if (Thread.interrupted()) { + throw new InterruptedException(); + } + } + + private void reallyAwaitTermination(boolean interruptWorkers) { + // TODO(bazel-team): verify that interrupt() is safe for every use of + // AbstractQueueVisitor and remove the interruptWorkers flag. + if (interruptWorkers && !jobs.isEmpty()) { + interruptInFlightTasks(); + } + + if (isInterrupted()) { + interruptedLatch.countDown(); + } + + Throwables.propagateIfPossible(catastrophe); + synchronized (zeroRemainingTasks) { + while (remainingTasks != 0) { + try { + zeroRemainingTasks.wait(); + } catch (InterruptedException e) { + setInterrupted(); + } + } + } + + if (ownThreadPool) { + pool.shutdown(); + for (;;) { + try { + Throwables.propagateIfPossible(catastrophe); + pool.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS); + break; + } catch (InterruptedException e) { + setInterrupted(); + } + } + } + } + + private void interruptInFlightTasks() { + Thread thisThread = Thread.currentThread(); + for (Thread thread : jobs.keySet()) { + if (thisThread != thread) { + thread.interrupt(); + } + } + } + + /** + * Makes the visitation terminate prematurely. + */ + public void interrupt() { + setInterrupted(); + reallyAwaitTermination(true); + } + + /** + * If this returns true, that means the exception {@code e} is critical + * and all running actions should be stopped. + * + * <p>Default value - always false. If different behavior is needed + * then we should override this method in subclasses. + * + * @param e the exception object to check + */ + protected boolean isCriticalError(Throwable e) { + return false; + } + + private void setRejectedExecutionHandler() { + if (ownThreadPool) { + pool.setRejectedExecutionHandler(new RejectedExecutionHandler() { + @Override + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + decrementRemainingTasks(); + } + }); + } + } + + /** + * If exception is critical then set a flag which signals + * to stop all jobs inside {@link #awaitTermination(boolean)}. + */ + private synchronized void markToStopAllJobsIfNeeded(Throwable e) { + if (isCriticalError(e) && !jobsMustBeStopped) { + jobsMustBeStopped = true; + synchronized (zeroRemainingTasks) { + zeroRemainingTasks.notify(); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/concurrent/ExecutorShutdownUtil.java b/src/main/java/com/google/devtools/build/lib/concurrent/ExecutorShutdownUtil.java new file mode 100644 index 0000000..95962b3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/concurrent/ExecutorShutdownUtil.java
@@ -0,0 +1,103 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.concurrent; + +import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Utilities for safely shutting down executors. + * TODO(bazel-team): Rename this class to something like "ExecutorUtil". + */ +public class ExecutorShutdownUtil { + + private ExecutorShutdownUtil() { + } + + /** + * Shutdown the executor. If an interrupt occurs, invoke shutdownNow(), but + * still block on the eventual termination of the pool. + * + * @param executor the executor service. + * @return true iff interrupted. + */ + public static boolean interruptibleShutdown(ExecutorService executor) { + return shutdownImpl(executor, /*interruptible=*/true); + } + + /** + * Shutdown the executor. If an interrupt occurs, ignore it and still block on the eventual + * termination of the pool. This way, all tasks are guaranteed to have completed normally. + * + * @param executor the executor service. + * @return true iff interrupted. + */ + public static boolean uninterruptibleShutdown(ExecutorService executor) { + return shutdownImpl(executor, /*interruptible=*/false); + } + + private static boolean shutdownImpl(ExecutorService executor, boolean interruptible) { + Preconditions.checkState(!executor.isShutdown()); + executor.shutdown(); + + // Common pattern: check for interrupt, but don't return until all threads + // are finished. + boolean interrupted = false; + while (true) { + try { + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS); + break; + } catch (InterruptedException e) { + if (interruptible) { + executor.shutdownNow(); + } + interrupted = true; + } + } + return interrupted; + } + + /** + * Create a "slack" thread pool which has the following properties: + * 1. the worker count shrinks as the threads go unused + * 2. the rejection policy is caller-runs + * + * @param threads maximum number of threads in the pool + * @param name name of the pool + * @return the new ThreadPoolExecutor + */ + public static ThreadPoolExecutor newSlackPool(int threads, String name) { + // Using a synchronous queue with a bounded thread pool means we'll reject + // tasks after the pool size. The CallerRunsPolicy, however, implies that + // saturation is handled in the calling thread. + ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 3L, TimeUnit.SECONDS, + new SynchronousQueue<Runnable>()); + // Do not consume threads when not in use. + pool.allowCoreThreadTimeOut(true); + pool.setThreadFactory(new ThreadFactoryBuilder().setNameFormat(name + " %d").build()); + pool.setRejectedExecutionHandler(new RejectedExecutionHandler() { + @Override + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + r.run(); + } + }); + return pool; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/concurrent/MoreFutures.java b/src/main/java/com/google/devtools/build/lib/concurrent/MoreFutures.java new file mode 100644 index 0000000..ab84f99 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/concurrent/MoreFutures.java
@@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.concurrent; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; + +import java.util.List; + +import javax.annotation.Nullable; + +/** + * Utility class for working with futures. + */ +public class MoreFutures { + + private MoreFutures() {} + + /** + * Creates a new {@code ListenableFuture} whose value is a list containing the + * values of all its input futures, if all succeed. If any input fails, the + * returned future fails. If any of the futures fails, it cancels all the other futures. + * + * <p> This method is similar to {@code Futures.allAsList} but additionally it cancels all the + * futures in case any of them fails. + */ + public static <V> ListenableFuture<List<V>> allAsListOrCancelAll( + final Iterable<? extends ListenableFuture<? extends V>> futures) { + ListenableFuture<List<V>> combinedFuture = Futures.allAsList(futures); + Futures.addCallback(combinedFuture, new FutureCallback<List<V>>() { + @Override + public void onSuccess(@Nullable List<V> vs) {} + + /** + * In case of a failure of any of the futures (that gets propagated to combinedFuture) we + * cancel all the futures in the list. + */ + @Override + public void onFailure(Throwable ignore) { + for (ListenableFuture<? extends V> future : futures) { + future.cancel(true); + } + } + }); + return combinedFuture; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/concurrent/Sharder.java b/src/main/java/com/google/devtools/build/lib/concurrent/Sharder.java new file mode 100644 index 0000000..67a63e0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/concurrent/Sharder.java
@@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.concurrent; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +/** + * A class to build shards (work queues) for a given task. + * + * <p>{@link #add}ed elements will be equally distributed among the shards. + * + * @param <T> the type of collection over which we're sharding + */ +public final class Sharder<T> implements Iterable<List<T>> { + private final List<List<T>> shards; + private int nextShard = 0; + + public Sharder(int maxNumShards, int expectedTotalSize) { + Preconditions.checkArgument(maxNumShards > 0); + Preconditions.checkArgument(expectedTotalSize >= 0); + this.shards = immutableListOfLists(maxNumShards, expectedTotalSize / maxNumShards); + } + + public void add(T item) { + shards.get(nextShard).add(item); + nextShard = (nextShard + 1) % shards.size(); + } + + /** + * Returns an immutable list of mutable lists. + * + * @param numLists the number of top-level lists. + * @param expectedSize the exepected size of each mutable list. + * @return a list of lists. + */ + private static <T> List<List<T>> immutableListOfLists(int numLists, int expectedSize) { + List<List<T>> list = Lists.newArrayListWithCapacity(numLists); + for (int i = 0; i < numLists; i++) { + list.add(Lists.<T>newArrayListWithExpectedSize(expectedSize)); + } + return Collections.unmodifiableList(list); + } + + @Override + public Iterator<List<T>> iterator() { + return Iterables.filter(shards, new Predicate<List<T>>() { + @Override + public boolean apply(List<T> list) { + return !list.isEmpty(); + } + }).iterator(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/concurrent/ThreadSafety.java b/src/main/java/com/google/devtools/build/lib/concurrent/ThreadSafety.java new file mode 100644 index 0000000..0c67fd9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/concurrent/ThreadSafety.java
@@ -0,0 +1,135 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.concurrent; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Define some standard attributes for documenting thread safety properties. + *<p> + * The names used here are adapted from those used in Joshua Bloch's book + * "Effective Java", which are also described at + * <http://www-128.ibm.com/developerworks/java/library/j-jtp09263.html>. + *<p> + * These attributes are just documentation. They don't have any run-time + * effect. The main aim is mainly just to standardize the terminology. + * (However, if this catches on, I can also imagine in the future having + * a presubmit check that checks that all new classes have thread safety + * annotations :) + *<p> + * See ThreadSafetyTest for examples of how these attributes should be used. + */ +public class ThreadSafety { + /** + * The Immutable attribute indicates that instances of the class are + * immutable, or at least appear that way are far as their external API + * is concerned. Immutable classes are usually also ThreadSafe, + * but can be ThreadHostile if they perform unsynchronized access to + * mutable static data. (We deviate from Bloch's nomenclature by + * not assuming that Immutable implies ThreadSafe; developers should + * explicitly annotate classes as both Immutable and ThreadSafe when + * appropriate.) + */ + @Documented + @Target(value = {ElementType.TYPE}) + @Retention(RetentionPolicy.RUNTIME) + public @interface Immutable {} + + /** + * The ThreadSafe attribute marks a class or method which can safely be used + * from multiple threads without any need for external synchronization. + * + * When applied to a class, this attribute indicates that instances + * of the class can safely be used concurrently from multiple threads + * without any need for external synchronization, i.e. that all non-static methods + * are thread-safe (except any private methods that are explicitly + * annotated with a different thread safety annotation). In addition it + * also indicates that all non-static nested classes are thread-safe (except any private + * nested classes that are explicitly annotated with a different thread + * safety annotation). Note that no guarantees are made about static class methods or static + * nested classes - they should be annotated separately. + * + * When applied to a method, this attribute indicates that the + * method can safely be called concurrently from multiple threads. + * The implementation must synchronize any accesses to mutable data. + */ + @Documented + @Target(value = {ElementType.CONSTRUCTOR, ElementType.METHOD, ElementType.TYPE}) + @Retention(RetentionPolicy.SOURCE) + public @interface ThreadSafe {} + + /** + * The ThreadCompatible attribute marks a class or method that + * is thread-safe provided that only one thread attempts to + * access each object at a time. + * + * The implementation of such a class must synchronize accesses + * to mutable static data, but can assume that each instance will + * only be accessed from one thread at a time. + * + * The client must obtain an appropriate lock before calling ThreadCompatible + * methods, or must otherwise ensure that only one thread calls such methods. + * Unless otherwise specified, an appropriate lock means synchronizing on the + * instance. + * + * A ThreadCompatible class may contain private methods or private nested + * classes that are not ThreadCompatible provided that they are explicitly + * annotated with a different thread safety annotation. + */ + @Documented + @Target(value = {ElementType.METHOD, ElementType.TYPE}) + @Retention(RetentionPolicy.SOURCE) + public @interface ThreadCompatible {} + + /** + * The ThreadHostile attribute marks a class or method that + * can't safely be used by multiple threads, for example because + * it performs unsynchronized access to mutable static objects. + */ + @Documented + @Target(value = {ElementType.CONSTRUCTOR, ElementType.METHOD, ElementType.TYPE}) + @Retention(RetentionPolicy.SOURCE) + public @interface ThreadHostile {} + + /** + * The ConditionallyThreadSafe attribute marks a class that contains + * some methods (or nested classes) which are ThreadSafe but others which are + * only ThreadCompatible or ThreadHostile. + * + * The methods (and nested classes) of a ConditionallyThreadSafe class should + * each have their thread safety marked. + */ + @Documented + @Target(value = {ElementType.METHOD, ElementType.TYPE}) + @Retention(RetentionPolicy.SOURCE) + public @interface ConditionallyThreadSafe {} + + /** + * The ConditionallyThreadCompatible attribute marks a class that contains + * some methods (or nested classes) which are ThreadCompatible but others + * which are ThreadHostile. + * + * The methods (and nested classes) of a ConditionallyThreadCompatible class + * should each have their thread safety marked. + */ + @Documented + @Target(value = {ElementType.METHOD, ElementType.TYPE}) + @Retention(RetentionPolicy.SOURCE) + public @interface ConditionallyThreadCompatible {} + +}
diff --git a/src/main/java/com/google/devtools/build/lib/concurrent/ThrowableRecordingRunnableWrapper.java b/src/main/java/com/google/devtools/build/lib/concurrent/ThrowableRecordingRunnableWrapper.java new file mode 100644 index 0000000..3f67d55 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/concurrent/ThrowableRecordingRunnableWrapper.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.concurrent; + +import com.google.common.base.Preconditions; + +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +/** + * A class that wraps Runnables and records the first Throwable thrown by the wrapped Runnables + * when they are run. + */ +public class ThrowableRecordingRunnableWrapper { + + private final String name; + private AtomicReference<Throwable> errorRef = new AtomicReference<>(); + + private static final Logger LOG = + Logger.getLogger(ThrowableRecordingRunnableWrapper.class.getName()); + + public ThrowableRecordingRunnableWrapper(String name) { + this.name = Preconditions.checkNotNull(name); + } + + @Nullable + public Throwable getFirstThrownError() { + return errorRef.get(); + } + + public Runnable wrap(final Runnable runnable) { + return new Runnable() { + @Override + public void run() { + try { + runnable.run(); + } catch (Throwable error) { + errorRef.compareAndSet(null, error); + LOG.log(Level.SEVERE, "Error thrown by runnable in " + name, error); + } + } + }; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/AbstractEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/AbstractEventHandler.java new file mode 100644 index 0000000..39faf14 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/AbstractEventHandler.java
@@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import java.util.Set; + +/** + * An abstract event handler that keeps track of the event mask. Events + * matching the mask will be handled. + */ +public abstract class AbstractEventHandler implements EventHandler { + + private final Set<EventKind> mask; + + /** + * Events matching the mask will be handled. + */ + public AbstractEventHandler(Set<EventKind> mask) { + this.mask = mask; + } + + public Set<EventKind> getEventMask() { + return mask; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/DelegatingEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/DelegatingEventHandler.java new file mode 100644 index 0000000..d26d70c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/DelegatingEventHandler.java
@@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.events; + +import com.google.common.base.Preconditions; + +/** + * An ErrorEventListener which delegates to another ErrorEventListener. + * Primarily useful as a base class for extending behavior. + */ +public class DelegatingEventHandler implements EventHandler { + protected final EventHandler delegate; + + public DelegatingEventHandler(EventHandler delegate) { + super(); + this.delegate = Preconditions.checkNotNull(delegate); + } + + @Override + public void handle(Event e) { + delegate.handle(e); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/DelegatingOnlyErrorsEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/DelegatingOnlyErrorsEventHandler.java new file mode 100644 index 0000000..dec220d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/DelegatingOnlyErrorsEventHandler.java
@@ -0,0 +1,32 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +/** + * An {@link EventHandler} implementation that only + * passes through error messages. + */ +public class DelegatingOnlyErrorsEventHandler extends DelegatingEventHandler { + + public DelegatingOnlyErrorsEventHandler(EventHandler eventHandler) { + super(eventHandler); + } + + @Override + public void handle(Event e) { + if (e.getKind() == EventKind.ERROR) { + super.handle(e); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/ErrorSensingEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/ErrorSensingEventHandler.java new file mode 100644 index 0000000..705f7f4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/ErrorSensingEventHandler.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.events; + +/** + * Passes through any events, and keeps a flag if any of them were errors. It is thread-safe as long + * as the target eventHandler is thread-safe. + */ +public final class ErrorSensingEventHandler extends DelegatingEventHandler { + + private volatile boolean hasErrors; + + public ErrorSensingEventHandler(EventHandler eventHandler) { + super(eventHandler); + } + + @Override + public void handle(Event e) { + hasErrors |= e.getKind() == EventKind.ERROR; + super.handle(e); + } + + /** + * Returns whether any of the events on this objects were errors. + */ + public boolean hasErrors() { + return hasErrors; + } + + /** + * Reset the error flag. Don't call this while other threads are accessing the same object. + */ + public void resetErrors() { + hasErrors = false; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/Event.java b/src/main/java/com/google/devtools/build/lib/events/Event.java new file mode 100644 index 0000000..db5bc5f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/Event.java
@@ -0,0 +1,183 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.base.Preconditions; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * An event is a situation encountered by the build system that's worth + * reporting: A 3-tuple of ({@link EventKind}, {@link Location}, message). + */ +@Immutable +public final class Event { + + private final EventKind kind; + private final Location location; + private final String message; + /** + * An alternative representation for message. + * Exactly one of message or messageBytes will be non-null. + * If messageBytes is non-null, then it contains the bytes + * of the message, encoded using the platform's default charset. + * We do this to avoid converting back and forth between Strings + * and bytes. + */ + private final byte[] messageBytes; + + @Nullable + private final String tag; + + public Event withTag(String tag) { + if (this.message != null) { + return new Event(this.kind, this.location, this.message, tag); + } else { + return new Event(this.kind, this.location, this.messageBytes, tag); + } + } + + public Event(EventKind kind, @Nullable Location location, String message) { + this(kind, location, message, null); + } + + public Event(EventKind kind, @Nullable Location location, String message, @Nullable String tag) { + this.kind = kind; + this.location = location; + this.message = Preconditions.checkNotNull(message); + this.messageBytes = null; + this.tag = tag; + } + + public Event(EventKind kind, @Nullable Location location, byte[] messageBytes) { + this(kind, location, messageBytes, null); + } + + public Event( + EventKind kind, @Nullable Location location, byte[] messageBytes, @Nullable String tag) { + this.kind = kind; + this.location = location; + this.message = null; + this.messageBytes = Preconditions.checkNotNull(messageBytes); + this.tag = tag; + } + + public String getMessage() { + return message != null ? message : new String(messageBytes); + } + + public byte[] getMessageBytes() { + return messageBytes != null ? messageBytes : message.getBytes(ISO_8859_1); + } + + public EventKind getKind() { + return kind; + } + + /** + * the tag is typically the action that generated the event. + */ + @Nullable + public String getTag() { + return tag; + } + + /** + * Returns the location of this event, if any. Returns null iff the event + * wasn't associated with any particular location, for example, a progress + * message. + */ + @Nullable public Location getLocation() { + return location; + } + + /** + * Returns <i>some</i> moderately sane representation of the event. Should never be used in + * user-visible places, only for debugging and testing. + */ + @Override + public String toString() { + return kind + " " + (location != null ? location.print() : "<no location>") + ": " + + getMessage(); + } + + /** + * Replay a sequence of events on an {@link EventHandler}. + */ + public static void replayEventsOn(EventHandler eventHandler, Iterable<Event> events) { + for (Event event : events) { + eventHandler.handle(event); + } + } + + /** + * Reports a warning. + */ + public static Event warn(Location location, String message) { + return new Event(EventKind.WARNING, location, message); + } + + /** + * Reports an error. + */ + public static Event error(Location location, String message){ + return new Event(EventKind.ERROR, location, message); + } + + /** + * Reports atemporal statements about the build, i.e. they're true for the duration of execution. + */ + public static Event info(Location location, String message) { + return new Event(EventKind.INFO, location, message); + } + + /** + * Reports a temporal statement about the build. + */ + public static Event progress(Location location, String message) { + return new Event(EventKind.PROGRESS, location, message); + } + + /** + * Reports a warning. + */ + public static Event warn(String message) { + return warn(null, message); + } + + /** + * Reports an error. + */ + public static Event error(String message){ + return error(null, message); + } + + /** + * Reports atemporal statements about the build, i.e. they're true for the duration of execution. + */ + public static Event info(String message) { + return info(null, message); + } + + /** + * Reports a temporal statement about the build. + */ + public static Event progress(String message) { + return progress(null, message); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/EventCollector.java b/src/main/java/com/google/devtools/build/lib/events/EventCollector.java new file mode 100644 index 0000000..774b323 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/EventCollector.java
@@ -0,0 +1,78 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.Set; + +/** + * An {@link EventHandler} that collects all events it encounters, and makes + * them available via the {@link Iterable} interface. The collected events + * contain not just the original event information but also the location + * context. + */ +public class EventCollector extends AbstractEventHandler implements Iterable<Event> { + + private final Collection<Event> collected; + + /** + * This collector will collect all events that match the event mask. + */ + public EventCollector(Set<EventKind> mask) { + this(mask, new ArrayList<Event>()); + } + + /** + * This collector will save the Event instances in the provided + * collection. + */ + public EventCollector(Set<EventKind> mask, Collection<Event> collected) { + super(mask); + this.collected = collected; + } + + /** + * Implements {@link EventHandler#handle(Event)}. + */ + @Override + public void handle(Event event) { + if (getEventMask().contains(event.getKind())) { + collected.add(event); + } + } + + /** + * Returns an iterator over the collected events. + */ + @Override + public Iterator<Event> iterator() { + return collected.iterator(); + } + + /** + * Returns the number of events collected. + */ + public int count() { + return collected.size(); + } + + /* + * Clears the collected events + */ + public void clear() { + collected.clear(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/EventHandler.java b/src/main/java/com/google/devtools/build/lib/events/EventHandler.java new file mode 100644 index 0000000..28b6265 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/EventHandler.java
@@ -0,0 +1,27 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.events; + +/** + * The ErrorEventListener is the primary means of reporting error and warning events. It is a subset + * of the functionality of the {@link Reporter}. In most cases, you should use this interface + * instead of the final {@code Reporter} class. + */ +public interface EventHandler { + /** + * Handles an event. + */ + public void handle(Event event); +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/EventKind.java b/src/main/java/com/google/devtools/build/lib/events/EventKind.java new file mode 100644 index 0000000..eb58873 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/EventKind.java
@@ -0,0 +1,146 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import java.util.EnumSet; +import java.util.Set; + +/** + * Indicates the kind of an {@link Event}. + */ +public enum EventKind { + + /** + * For errors that will prevent a successful, correct build. In general, the + * build tool will not attempt to start or continue a build if an error is + * encountered (though the behaviour specified by --keep-going flag is a + * counterexample). + * + * Errors of a more severe nature in the input, such as those which might + * cause later passes of the analysis to fail catastrophically, should be + * handled by throwing an exception. + */ + ERROR, + + /** + * For warnings of minor problems that do not affect the integrity of a + * build. + */ + WARNING, + + /** + * For atemporal information that is true throughout the entire duration + * of a build. (e.g. the number of targets found) + */ + INFO, + + /** + * For temporal information that changes during the duration of a build. + * (e.g. what action is executing now) + */ + PROGRESS, + + /** + * For progress messages (temporal information) relating to the start + * and end of particular tasks. + * (e.g. "Loading package foo", "Compiling bar", etc.) + */ + START, + FINISH, + + /** + * For command lines of subcommands executed by the build tool (like make-dbg + * "-v"). + */ + SUBCOMMAND, + + /** + * Output to stdout/stderr from subprocesses. + */ + STDOUT, + STDERR, + + /** + * Test result messages (similar to the INFO and ERROR, but test-specific). + */ + PASS, + FAIL, + TIMEOUT, + + /** + * For the reasoning of the dependency checker (like GNU Make "-d"). + */ + DEPCHECKER; + + // Convenient predefined EnumSets. Clients should not mutate them! + + public static final Set<EventKind> ALL_EVENTS = + EnumSet.allOf(EventKind.class); + + public static final Set<EventKind> OUTPUT = EnumSet.of( + EventKind.STDOUT, + EventKind.STDERR + ); + + public static final Set<EventKind> ERRORS = EnumSet.of( + EventKind.ERROR, + EventKind.FAIL, + EventKind.TIMEOUT + ); + + public static final Set<EventKind> ERRORS_AND_WARNINGS = EnumSet.of( + EventKind.ERROR, + EventKind.WARNING, + EventKind.FAIL, + EventKind.TIMEOUT + ); + + public static final Set<EventKind> ERRORS_WARNINGS_AND_INFO = EnumSet.of( + EventKind.ERROR, + EventKind.WARNING, + EventKind.PASS, + EventKind.FAIL, + EventKind.TIMEOUT, + EventKind.INFO + ); + + public static final Set<EventKind> ERRORS_AND_OUTPUT = EnumSet.of( + EventKind.ERROR, + EventKind.FAIL, + EventKind.TIMEOUT, + EventKind.STDOUT, + EventKind.STDERR + ); + + public static final Set<EventKind> ERRORS_AND_WARNINGS_AND_OUTPUT = EnumSet.of( + EventKind.ERROR, + EventKind.WARNING, + EventKind.FAIL, + EventKind.TIMEOUT, + EventKind.STDOUT, + EventKind.STDERR + ); + + public static final Set<EventKind> ERRORS_WARNINGS_AND_INFO_AND_OUTPUT = EnumSet.of( + EventKind.ERROR, + EventKind.WARNING, + EventKind.PASS, + EventKind.FAIL, + EventKind.TIMEOUT, + EventKind.INFO, + EventKind.STDOUT, + EventKind.STDERR + ); + +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/EventSensor.java b/src/main/java/com/google/devtools/build/lib/events/EventSensor.java new file mode 100644 index 0000000..5a31f13 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/EventSensor.java
@@ -0,0 +1,73 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import java.util.Set; + +/** + * A "latch" that just detects whether or not a particular type of event has happened, based on its + * kind. + * + * <p>Be careful when using this class to track errors reported during some operation. Namely, this + * pattern is not thread-safe: + * + * <pre><code> + * EventSensor sensor = new EventSensor(EventKind.ERRORS); + * reporter.addHandler(sensor); + * someActionThatMightCreateErrors(reporter) + * reporter.removeHandler(sensor); + * boolean containsErrors = sensor.wasTriggered(); + * </code></pre> + * + * <p>If other threads generate errors on the reporter, then containsErrors may be true even if + * someActionThatMightCreateErrors() did not cause any errors. + * + * <p>As a workaround, run someActionThatMightCreateErrors() with a local reporter, merging its + * events with those of the shared reporter. + */ +public class EventSensor extends AbstractEventHandler { + + private int triggerCount; + + /** + * Constructs a sensor that will register all events matching the mask. + */ + public EventSensor(Set<EventKind> mask) { + super(mask); + } + + /** + * Implements {@link EventHandler#handle(Event)}. + */ + @Override + public void handle(Event event) { + if (getEventMask().contains(event.getKind())) { + triggerCount++; + } + } + + /** + * Returns true iff a qualifying event was handled. + */ + public boolean wasTriggered() { + return triggerCount > 0; + } + + /** + * Returns the number of times the qualifying event was handled. + */ + public int getTriggerCount() { + return triggerCount; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/ExceptionListener.java b/src/main/java/com/google/devtools/build/lib/events/ExceptionListener.java new file mode 100644 index 0000000..174a5ca --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/ExceptionListener.java
@@ -0,0 +1,25 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +/** + * The ExceptionListener is the primary means of reporting exceptions. It is a subset of the + * functionality of the {@link Reporter}. + */ +public interface ExceptionListener { + /** + * Reports an error. + */ + void error(Location location, String message, Throwable error); +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/Location.java b/src/main/java/com/google/devtools/build/lib/events/Location.java new file mode 100644 index 0000000..39508d1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/Location.java
@@ -0,0 +1,215 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.events; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.Serializable; + +/** + * A Location is a range of characters within a file. + * + * The start and end locations may be the same, in which case the Location + * denotes a point in the file, not a range. The path may be null, indicating + * an unknown file. + * + * Implementations of Location should be optimised for speed of construction, + * not speed of attribute access, as far more Locations are created during + * parsing than are ever used to display error messages. + */ +public abstract class Location implements Serializable { + + @Immutable + private static final class LocationWithPathAndStartColumn extends Location { + private final PathFragment path; + private final LineAndColumn startLineAndColumn; + + private LocationWithPathAndStartColumn(Path path, int startOffSet, int endOffSet, + LineAndColumn startLineAndColumn) { + super(startOffSet, endOffSet); + this.path = path != null ? path.asFragment() : null; + this.startLineAndColumn = startLineAndColumn; + } + + @Override + public PathFragment getPath() { return path; } + + @Override + public LineAndColumn getStartLineAndColumn() { + return startLineAndColumn; + } + } + + protected final int startOffset; + protected final int endOffset; + + /** + * Returns a Location with a given Path, start and end offset and start line and column info. + */ + public static Location fromPathAndStartColumn(Path path, int startOffSet, int endOffSet, + LineAndColumn startLineAndColumn) { + return new LocationWithPathAndStartColumn(path, startOffSet, endOffSet, startLineAndColumn); + } + + /** + * Returns a Location relating to file 'path', but not to any specific part + * region within the file. Try to use a more specific location if possible. + */ + public static Location fromFile(Path path) { + return fromFileAndOffsets(path, 0, 0); + } + + /** + * Returns a Location relating to the subset of file 'path', starting at + * 'startOffset' and ending at 'endOffset'. + */ + public static Location fromFileAndOffsets(final Path path, + int startOffset, + int endOffset) { + return new LocationWithPathAndStartColumn(path, startOffset, endOffset, null); + } + + protected Location(int startOffset, int endOffset) { + this.startOffset = startOffset; + this.endOffset = endOffset; + } + + /** + * Returns the start offset relative to the beginning of the file the object + * resides in. + */ + public final int getStartOffset() { + return startOffset; + } + + /** + * Returns the end offset relative to the beginning of the file the object + * resides in. + * + * <p>The end offset is one position past the actual end position, making this method + * behave in a compatible fashion with {@link String#substring(int, int)}. + * + * <p>To compute the length of this location, use {@code getEndOffset() - getStartOffset()}. + */ + public final int getEndOffset() { + return endOffset; + } + + /** + * Returns the path of the file to which the start/end offsets refer. May be + * null if the file name information is not available. + * + * This method is intentionally abstract, as a space optimisation. Some + * subclass instances implement sharing of common data (e.g. tables for + * convering offsets into line numbers) and this enables them to share the + * Path value in the same way. + */ + public abstract PathFragment getPath(); + + /** + * Returns a (line, column) pair corresponding to the position denoted by + * getStartOffset. Returns null if this information is not available. + */ + public LineAndColumn getStartLineAndColumn() { + return null; + } + + /** + * Returns a (line, column) pair corresponding to the position denoted by + * getEndOffset. Returns null if this information is not available. + */ + public LineAndColumn getEndLineAndColumn() { + return null; + } + + /** + * A default implementation of toString() that formats the location in the + * following ways based on the amount of information available: + * <pre> + * "foo.cc:23:2" + * "23:2" + * "foo.cc:char offsets 123--456" + * "char offsets 123--456" + * </pre> + */ + public String print() { + StringBuilder buf = new StringBuilder(); + if (getPath() != null) { + buf.append(getPath()).append(':'); + } + LineAndColumn start = getStartLineAndColumn(); + if (start == null) { + if (getStartOffset() == 0 && getEndOffset() == 0) { + buf.append("1"); // i.e. line 1 (special case: no information at all) + } else { + buf.append("char offsets "). + append(getStartOffset()).append("--").append(getEndOffset()); + } + } else { + buf.append(start.getLine()).append(':').append(start.getColumn()); + } + return buf.toString(); + } + + /** + * Prints the object in a sort of reasonable way. This should never be used in user-visible + * places, only for debugging and testing. + */ + @Override + public String toString() { + return print(); + } + + /** + * A value class that describes the line and column of an offset in a file. + */ + @Immutable + public static final class LineAndColumn { + private final int line; + private final int column; + + public LineAndColumn(int line, int column) { + this.line = line; + this.column = column; + } + + public int getLine() { + return line; + } + + public int getColumn() { + return column; + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof LineAndColumn)) { + return false; + } + LineAndColumn lac = (LineAndColumn) o; + return lac.line == line && lac.column == column; + } + + @Override + public int hashCode() { + return line * 81 + column; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/NullEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/NullEventHandler.java new file mode 100644 index 0000000..8bee1eb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/NullEventHandler.java
@@ -0,0 +1,28 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.events; + +/** + * An ErrorEventListener which does nothing. + */ +public final class NullEventHandler implements EventHandler { + public static final EventHandler INSTANCE = new NullEventHandler(); + + private NullEventHandler() {} // Prevent instantiation + + @Override + public void handle(Event e) { + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/OutputFilter.java b/src/main/java/com/google/devtools/build/lib/events/OutputFilter.java new file mode 100644 index 0000000..b5ca34d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/OutputFilter.java
@@ -0,0 +1,75 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.events; + +import java.util.regex.Pattern; + +/** + * An output filter for warnings. + */ +public interface OutputFilter { + + /** An output filter that matches everything. */ + public static final OutputFilter OUTPUT_EVERYTHING = new OutputFilter() { + @Override + public boolean showOutput(String tag) { + return true; + } + }; + + /** An output filter that matches nothing. */ + public static final OutputFilter OUTPUT_NOTHING = new OutputFilter() { + @Override + public boolean showOutput(String tag) { + return false; + } + }; + + /** + * Returns true iff the given tag matches the output filter. + */ + boolean showOutput(String tag); + + /** + * An output filter using regular expression matching. + */ + public static final class RegexOutputFilter implements OutputFilter { + /** Returns an output filter for the given regex (by compiling it). */ + public static OutputFilter forRegex(String regex) { + return new RegexOutputFilter(Pattern.compile(regex)); + } + + /** Returns an output filter for the given pattern. */ + public static OutputFilter forPattern(Pattern pattern) { + return new RegexOutputFilter(pattern); + } + + private final Pattern pattern; + + private RegexOutputFilter(Pattern pattern) { + this.pattern = pattern; + } + + @Override + public boolean showOutput(String tag) { + return pattern.matcher(tag).find(); + } + + @Override + public String toString() { + return pattern.toString(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/PrintingEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/PrintingEventHandler.java new file mode 100644 index 0000000..fa94d95 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/PrintingEventHandler.java
@@ -0,0 +1,119 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import com.google.devtools.build.lib.util.io.OutErr; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Set; + +/** + * An event handler that prints to an OutErr stream pair in a + * canonical format, for example: + * <pre> + * ERROR: /home/jrluser/src/workspace/x/BUILD:23:1: syntax error. + * </pre> + * This syntax is parseable by Emacs's compile.el. + * + * <p> + * By default, the output will go to SYSTEM_OUT_ERR, + * but this can be changed by calling the setOut() method. + * + * <p> + * This class is used only for tests. + */ +public class PrintingEventHandler extends AbstractEventHandler + implements EventHandler { + + /** + * A convenient event-handler for terminal applications that prints all + * errors and warnings it encounters to the error stream. + * STDOUT and STDERR events pass their output directly + * through to the corresponding streams. + */ + public static final PrintingEventHandler ERRORS_AND_WARNINGS_TO_STDERR = + new PrintingEventHandler(EventKind.ERRORS_AND_WARNINGS_AND_OUTPUT); + + /** + * A convenient event-handler for terminal applications that prints all + * errors it encounters to the error stream. + * STDOUT and STDERR events pass their output directly + * through to the corresponding streams. + */ + public static final PrintingEventHandler ERRORS_TO_STDERR = + new PrintingEventHandler(EventKind.ERRORS_AND_OUTPUT); + + private OutErr outErr = OutErr.SYSTEM_OUT_ERR; + + /** + * Setup a printing event handler that will handle events matching the mask. + * Events will be printed to the original System.out and System.err + * unless/until redirected by a call to setOutErr(). + */ + public PrintingEventHandler(Set<EventKind> mask) { + super(mask); + } + + /** + * Redirect all output to the specified OutErr stream pair. + * Returns the previous OutErr. + */ + public OutErr setOutErr(OutErr outErr) { + OutErr prev = this.outErr; + this.outErr = outErr; + return prev; + } + + /** + * Print a description of the specified event to the appropriate + * output or error stream. + */ + @Override + public void handle(Event event) { + if (!getEventMask().contains(event.getKind())) { + return; + } + try { + switch (event.getKind()) { + case STDOUT: + outErr.getOutputStream().write(event.getMessageBytes()); + outErr.getOutputStream().flush(); + break; + case STDERR: + outErr.getErrorStream().write(event.getMessageBytes()); + outErr.getErrorStream().flush(); + break; + default: + PrintWriter err = new PrintWriter(outErr.getErrorStream()); + err.print(event.getKind()); + err.print(": "); + if (event.getLocation() != null) { + err.print(event.getLocation().print()); + err.print(": "); + } + err.println(event.getMessage()); + err.flush(); + } + } catch (IOException e) { + /* + * Note: we can't print to System.out or System.err here, + * because those will normally be set to streams which + * translate I/O to STDOUT and STDERR events, + * which would result in infinite recursion. + */ + outErr.printErrLn(e.getMessage()); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/Reporter.java b/src/main/java/com/google/devtools/build/lib/events/Reporter.java new file mode 100644 index 0000000..e0c3925 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/Reporter.java
@@ -0,0 +1,146 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import com.google.devtools.build.lib.util.io.OutErr; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; + +/** + * The reporter is the primary means of reporting events such as errors, + * warnings, progress information and diagnostic information to the user. It + * is not intended as a logging mechanism for developer-only messages; use a + * Logger for that. + * + * The reporter instance is consumed by the build system, and passes events to + * {@link EventHandler} instances. These handlers are registered via {@link + * #addHandler(EventHandler)}. + * + * <p>Thread-safe: calls to {@code #report} may be made on any thread. + * Handlers may be run in an arbitary thread (but right now, they will not be + * run concurrently). + */ +public final class Reporter implements EventHandler, ExceptionListener { + + private final List<EventHandler> handlers = new ArrayList<>(); + + /** An OutErr that sends all of its output to this Reporter. + * Each write will (when flushed) get mapped to an EventKind.STDOUT or EventKind.STDERR event. + */ + private final OutErr outErrToReporter = outErrForReporter(this); + private volatile OutputFilter outputFilter = OutputFilter.OUTPUT_EVERYTHING; + + public Reporter() {} + + public static OutErr outErrForReporter(EventHandler rep) { + return OutErr.create( + // We don't use BufferedOutputStream here, because in general the Blaze + // code base assumes that the output streams are not buffered. + new ReporterStream(rep, EventKind.STDOUT), + new ReporterStream(rep, EventKind.STDERR)); + } + + /** + * A copy constructor, to make it convenient to replicate a reporter + * config for temporary configuration changes. + */ + public Reporter(Reporter template) { + handlers.addAll(template.handlers); + } + + /** + * Constructor which configures a reporter with the specified handlers. + */ + public Reporter(EventHandler... handlers) { + for (EventHandler handler: handlers) { + addHandler(handler); + } + } + + /** + * Returns an OutErr that sends all of its output to this Reporter. + * Each write to the OutErr will cause an EventKind.STDOUT or EventKind.STDERR event. + */ + public OutErr getOutErr() { + return outErrToReporter; + } + + /** + * Adds a handler to this reporter. + */ + public synchronized void addHandler(EventHandler handler) { + handlers.add(handler); + } + + /** + * Removes handler from this reporter. + */ + public synchronized void removeHandler(EventHandler handler) { + handlers.remove(handler); + } + + /** + * This method is called by the build system to report an event. + */ + @Override + public synchronized void handle(Event e) { + if (e.getKind() != EventKind.ERROR && e.getTag() != null && !showOutput(e.getTag())) { + return; + } + for (EventHandler handler : handlers) { + handler.handle(e); + } + } + + /** + * Reports the start of a particular task. + * Is a wrapper around report() with event kind START. + * Should always be matched by a corresponding call to finishTask() + * with the same message, except that the leading percentage + * progress indicator (if any) in the message may differ. + */ + public void startTask(Location location, String message) { + handle(new Event(EventKind.START, location, message)); + } + + /** + * Reports the start of a particular task. + * Is a wrapper around report() with event kind FINISH. + * Should always be matched by a corresponding call to startTask() + * with the same message, except that the leading percentage + * progress indicator (if any) in the message may differ. + */ + public void finishTask(Location location, String message) { + handle(new Event(EventKind.FINISH, location, message)); + } + + @Override + public void error(Location location, String message, Throwable error) { + handle(new Event(EventKind.ERROR, location, message)); + error.printStackTrace(new PrintStream(getOutErr().getErrorStream())); + } + + /** + * Returns true iff the given tag matches the output filter. + */ + public boolean showOutput(String tag) { + return outputFilter.showOutput(tag); + } + + public void setOutputFilter(OutputFilter outputFilter) { + this.outputFilter = outputFilter; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/ReporterStream.java b/src/main/java/com/google/devtools/build/lib/events/ReporterStream.java new file mode 100644 index 0000000..9c625c8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/ReporterStream.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.events; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import java.io.OutputStream; + +/** + * An OutputStream that delegates all writes to a Reporter. + */ +public final class ReporterStream extends OutputStream { + + private final EventHandler reporter; + private final EventKind eventKind; + + public ReporterStream(EventHandler reporter, EventKind eventKind) { + this.reporter = reporter; + this.eventKind = eventKind; + } + + @Override + public void close() { + // NOP. + } + + @Override + public void flush() { + // NOP. + } + + @Override + public void write(int b) { + reporter.handle(new Event(eventKind, null, new byte[] { (byte) b })); + } + + @Override + public void write(byte[] bytes) { + reporter.handle(new Event(eventKind, null, bytes)); + } + + @Override + public void write(byte[] bytes, int offset, int len) { + reporter.handle(new Event(eventKind, null, new String(bytes, offset, len, ISO_8859_1))); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/StoredEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/StoredEventHandler.java new file mode 100644 index 0000000..be8a627 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/StoredEventHandler.java
@@ -0,0 +1,63 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.List; + +/** + * Stores error and warning events, and later replays them. Thread-safe. + */ +public class StoredEventHandler implements EventHandler { + + private final List<Event> events = new ArrayList<>(); + private boolean hasErrors; + + public synchronized ImmutableList<Event> getEvents() { + return ImmutableList.copyOf(events); + } + + /** Returns true if there are no stored events. */ + public synchronized boolean isEmpty() { + return events.isEmpty(); + } + + + @Override + public synchronized void handle(Event e) { + hasErrors |= e.getKind() == EventKind.ERROR; + events.add(e); + } + + /** + * Replay all events stored in this object on the given eventHandler, in the same order. + */ + public synchronized void replayOn(EventHandler eventHandler) { + Event.replayEventsOn(eventHandler, events); + } + + /** + * Returns whether any of the events on this objects were errors. + */ + public synchronized boolean hasErrors() { + return hasErrors; + } + + public synchronized void clear() { + events.clear(); + hasErrors = false; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/events/WarningsAsErrorsEventHandler.java b/src/main/java/com/google/devtools/build/lib/events/WarningsAsErrorsEventHandler.java new file mode 100644 index 0000000..91a2150 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/events/WarningsAsErrorsEventHandler.java
@@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.events; + +/** + * Passes through any events, and converts any warnings to errors. + */ +public final class WarningsAsErrorsEventHandler extends DelegatingEventHandler { + + boolean warningsEncountered = false; + + public WarningsAsErrorsEventHandler(EventHandler eventHandler) { + super(eventHandler); + } + + @Override + public synchronized void handle(Event e) { + if (e.getKind() == EventKind.WARNING) { + warningsEncountered = true; + super.handle(new Event(EventKind.ERROR, e.getLocation(), e.getMessage())); + } else { + super.handle(e); + } + } + + public boolean warningsEncountered() { + return warningsEncountered; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/AlwaysOutOfDateAction.java b/src/main/java/com/google/devtools/build/lib/exec/AlwaysOutOfDateAction.java new file mode 100644 index 0000000..0e484f7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/AlwaysOutOfDateAction.java
@@ -0,0 +1,21 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +/** + * Marker interface for actions that must be run unconditionally. + */ +public interface AlwaysOutOfDateAction { + +}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/CheckUpToDateFilter.java b/src/main/java/com/google/devtools/build/lib/exec/CheckUpToDateFilter.java new file mode 100644 index 0000000..84f3aef --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/CheckUpToDateFilter.java
@@ -0,0 +1,73 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.rules.test.TestRunnerAction; + +/** + * Class implements --check_???_up_to_date execution filter predicate + * that prevents certain actions from being executed (thus aborting + * the build if action is not up-to-date). + */ +public final class CheckUpToDateFilter implements Predicate<Action> { + + /** + * Determines an execution filter based on the --check_up_to_date and + * --check_tests_up_to_date options. Returns a singleton if possible. + */ + public static Predicate<Action> fromOptions(ExecutionOptions options) { + if (!options.testCheckUpToDate && !options.checkUpToDate) { + return Predicates.alwaysTrue(); + } + return new CheckUpToDateFilter(options); + } + + private final boolean allowBuildActionExecution; + private final boolean allowTestActionExecution; + + /** + * Creates new execution filter based on --check_up_to_date and + * --check_tests_up_to_date options. + */ + private CheckUpToDateFilter(ExecutionOptions options) { + // If we want to check whether test is up-to-date, we should disallow + // test execution. + this.allowTestActionExecution = !options.testCheckUpToDate; + + // Build action execution should be prohibited in two cases - if we are + // checking whether build is up-to-date or if we are checking that tests + // are up-to-date (and test execution is not allowed). + this.allowBuildActionExecution = allowTestActionExecution && !options.checkUpToDate; + } + + /** + * @return true if actions' execution is allowed, false - otherwise + */ + @Override + public boolean apply(Action action) { + if (action instanceof AlwaysOutOfDateAction) { + // Always allow fileset manifest action to execute because it identifies files included + // in the fileset during execution time. + return true; + } else if (action instanceof TestRunnerAction) { + return allowTestActionExecution; + } else { + return allowBuildActionExecution; + } + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/Digest.java b/src/main/java/com/google/devtools/build/lib/exec/Digest.java new file mode 100644 index 0000000..4262711 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/Digest.java
@@ -0,0 +1,182 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +import static java.nio.charset.StandardCharsets.US_ASCII; + +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.actions.cache.VirtualActionInput; +import com.google.devtools.build.lib.util.Pair; +import com.google.protobuf.ByteString; +import com.google.protobuf.MessageLite; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * A utility class for obtaining MD5 digests. + * Digests are represented as 32 characters in lowercase ASCII. + */ +public class Digest { + + public static final ByteString EMPTY_DIGEST = fromContent(new byte[]{}); + + private Digest() { + } + + /** + * Get the digest from the given byte array. + * @param bytes the byte array. + * @return a digest. + */ + public static ByteString fromContent(byte[] bytes) { + MessageDigest md = newBuilder(); + md.update(bytes, 0, bytes.length); + return toByteString(BaseEncoding.base16().lowerCase().encode(md.digest())); + } + + /** + * Get the digest from the given ByteBuffer. + * @param buffer the ByteBuffer. + * @return a digest. + */ + public static ByteString fromBuffer(ByteBuffer buffer) { + MessageDigest md = newBuilder(); + md.update(buffer); + return toByteString(BaseEncoding.base16().lowerCase().encode(md.digest())); + } + + /** + * Gets the digest of the given proto. + * + * @param proto a protocol buffer. + * @return the digest. + */ + public static ByteString fromProto(MessageLite proto) { + MD5OutputStream md5Stream = new MD5OutputStream(); + try { + proto.writeTo(md5Stream); + } catch (IOException e) { + throw new IllegalStateException("Unexpected IOException: ", e); + } + return toByteString(md5Stream.getDigest()); + } + + /** + * Gets the digest and size of a given VirtualActionInput. + * + * @param input the VirtualActionInput. + * @return the digest and size. + */ + public static Pair<ByteString, Long> fromVirtualActionInput(VirtualActionInput input) + throws IOException { + CountingMD5OutputStream md5Stream = new CountingMD5OutputStream(); + input.writeTo(md5Stream); + ByteString digest = toByteString(md5Stream.getDigest()); + return Pair.of(digest, md5Stream.getSize()); + } + + /** + * A Sink that does an online MD5 calculation, which avoids forcing us to keep the entire + * proto message in memory. + */ + private static class MD5OutputStream extends OutputStream { + private final MessageDigest md = newBuilder(); + + @Override + public void write(int b) { + md.update((byte) b); + } + + @Override + public void write(byte[] b, int off, int len) { + md.update(b, off, len); + } + + public String getDigest() { + return BaseEncoding.base16().lowerCase().encode(md.digest()); + } + } + + private static final class CountingMD5OutputStream extends MD5OutputStream { + private long size; + + @Override + public void write(int b) { + super.write(b); + size++; + } + + @Override + public void write(byte[] b, int off, int len) { + super.write(b, off, len); + size += len; + } + + public long getSize() { + return size; + } + } + + /** + * @param digest the digest to check. + * @return true iff digest is a syntactically legal digest. It must be 32 + * characters of hex with lowercase letters. + */ + public static boolean isDigest(ByteString digest) { + if (digest == null || digest.size() != 32) { + return false; + } + + for (byte b : digest) { + char c = (char) b; + if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) { + continue; + } + return false; + } + return true; + } + + /** + * @param digest the digest. + * @return true iff the digest is that of an empty file. + */ + public static boolean isEmpty(ByteString digest) { + return digest.equals(EMPTY_DIGEST); + } + + /** + * @return a new MD5 digest builder. + */ + public static MessageDigest newBuilder() { + try { + return MessageDigest.getInstance("md5"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("MD5 not available"); + } + } + + /** + * Convert a String digest into a ByteString using ascii. + * @param digest the digest in ascii. + * @return the digest as a ByteString. + */ + public static ByteString toByteString(String digest) { + return ByteString.copyFrom(digest.getBytes(US_ASCII)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java new file mode 100644 index 0000000..58e360b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
@@ -0,0 +1,195 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.packages.TestTimeout; +import com.google.devtools.build.lib.rules.test.TestStrategy; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestSummaryFormat; +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.Options; +import com.google.devtools.common.options.OptionsBase; + +import java.util.Map; + +/** + * Options affecting the execution phase of a build. + * + * These options are interpreted by the BuildTool to choose an Executor to + * be used for the build. + * + * Note: from the user's point of view, the characteristic function of this + * set of options is indistinguishable from that of the BuildRequestOptions: + * they are all per-request. The difference is only apparent in the + * implementation: these options are used only by the lib.exec machinery, which + * affects how C++ and Java compilation occur. (The BuildRequestOptions + * contain a mixture of "semantic" options affecting the choice of targets to + * build, and "non-semantic" options affecting the lib.actions machinery.) + * Ideally, the user would be unaware of the difference. For now, the usage + * strings are identical modulo "part 1", "part 2". + */ +public class ExecutionOptions extends OptionsBase { + + public static final ExecutionOptions DEFAULTS = Options.getDefaults(ExecutionOptions.class); + + @Option(name = "verbose_failures", + defaultValue = "false", + category = "verbosity", + help = "If a command fails, print out the full command line.") + public boolean verboseFailures; + + @Option(name = "subcommands", + abbrev = 's', + defaultValue = "false", + category = "verbosity", + help = "Display the subcommands executed during a build.") + public boolean showSubcommands; + + @Option(name = "check_up_to_date", + defaultValue = "false", + category = "what", + help = "Don't perform the build, just check if it is up-to-date. If all targets are " + + "up-to-date, the build completes successfully. If any step needs to be executed " + + "an error is reported and the build fails.") + public boolean checkUpToDate; + + @Option(name = "check_tests_up_to_date", + defaultValue = "false", + category = "testing", + implicitRequirements = { "--check_up_to_date" }, + help = "Don't run tests, just check if they are up-to-date. If all tests results are " + + "up-to-date, the testing completes successfully. If any test needs to be built or " + + "executed, an error is reported and the testing fails. This option implies " + + "--check_up_to_date behavior." + ) + public boolean testCheckUpToDate; + + @Option(name = "test_strategy", + defaultValue = "", + category = "testing", + help = "Specifies which strategy to use when running tests.") + public String testStrategy; + + @Option(name = "test_keep_going", + defaultValue = "true", + category = "testing", + help = "When disabled, any non-passing test will cause the entire build to stop. By default " + + "all tests are run, even if some do not pass.") + public boolean testKeepGoing; + + @Option(name = "runs_per_test_detects_flakes", + defaultValue = "false", + category = "testing", + help = "If true, any shard in which at least one run/attempt passes and at least one " + + "run/attempt fails gets a FLAKY status.") + public boolean runsPerTestDetectsFlakes; + + @Option(name = "flaky_test_attempts", + defaultValue = "default", + category = "testing", + converter = TestStrategy.TestAttemptsConverter.class, + help = "Each test will be retried up to the specified number of times in case of any test " + + "failure. Tests that required more than one attempt to pass would be marked as " + + "'FLAKY' in the test summary. If this option is set, it should specify an int N or the " + + "string 'default'. If it's an int, then all tests will be run up to N times. If it is " + + "not specified or its value is 'default', then only a single test attempt will be made " + + "for regular tests and three for tests marked explicitly as flaky by their rule " + + "(flaky=1 attribute).") + public int testAttempts; + + @Option(name = "test_tmpdir", + defaultValue = "null", + category = "testing", + converter = OptionsUtils.PathFragmentConverter.class, + help = "Specifies the base temporary directory for 'blaze test' to use.") + public PathFragment testTmpDir; + + @Option(name = "test_output", + defaultValue = "summary", + category = "testing", + converter = TestStrategy.TestOutputFormat.Converter.class, + help = "Specifies desired output mode. Valid values are 'summary' to " + + "output only test status summary, 'errors' to also print test logs " + + "for failed tests, 'all' to print logs for all tests and 'streamed' " + + "to output logs for all tests in real time (this will force tests " + + "to be executed locally one at a time regardless of --test_strategy " + + "value).") + public TestOutputFormat testOutput; + + @Option(name = "test_summary", + defaultValue = "short", + category = "testing", + converter = TestStrategy.TestSummaryFormat.Converter.class, + help = "Specifies the desired format ot the test summary. Valid values " + + "are 'short' to print information only about tests executed, " + + "'terse', to print information only about unsuccessful tests," + + "'detailed' to print detailed information about failed test cases, " + + "and 'none' to omit the summary.") + public TestSummaryFormat testSummary; + + @Option(name = "test_timeout", + defaultValue = "-1", + category = "testing", + converter = TestTimeout.TestTimeoutConverter.class, + help = "Override the default test timeout values for test timeouts (in secs). If a single " + + "positive integer value is specified it will override all categories. If 4 comma-" + + "separated integers are specified, they will override the timeouts for short, " + + "moderate, long and eternal (in that order). In either form, a value of -1 tells blaze " + + "to use its default timeouts for that category.") + public Map<TestTimeout, Integer> testTimeout; + + + @Option(name = "resource_autosense", + defaultValue = "false", + category = "strategy", + help = "Periodically (every 3 seconds) poll system CPU load and available memory " + + "and allow execution of build commands if system has sufficient idle CPU and " + + "free RAM resources. By default this option is disabled, and Blaze will rely on " + + "approximation algorithms based on the total amount of available memory and number " + + "of CPU cores.") + public boolean useResourceAutoSense; + + @Option(name = "ram_utilization_factor", + defaultValue = "67", + category = "strategy", + help = "Specify what percentage of the system's RAM Blaze should try to use for its " + + "subprocesses. " + + "This option affects how many processes Blaze will try to run in parallel. " + + "If you run several Blaze builds in parallel, using a lower value for " + + "this option may avoid thrashing and thus improve overall throughput. " + + "Using a value higher than the default is NOT recommended. " + + "Note that Blaze's estimates are very coarse, so the actual RAM usage may be much " + + "higher or much lower than specified. " + + "Note also that this option does not affect the amount of memory that the Blaze " + + "server itself will use. " + + "Also, this option has no effect if --resource_autosense is enabled." + ) + public int ramUtilizationPercentage; + + @Option(name = "local_resources", + defaultValue = "null", + category = "strategy", + help = "Explicitly set amount of local resources available to Blaze. " + + "By default, Blaze will query system configuration to estimate amount of RAM (in MB) " + + "and number of CPU cores available for the locally executed build actions. It would also " + + "assume default I/O capabilities of the local workstation (1.0). This options allows to " + + "explicitly set all 3 values. Note, that if this option is used, Blaze will ignore " + + "both --ram_utilization_factor and --resource_autosense values.", + converter = ResourceSet.ResourceSetConverter.class + ) + public ResourceSet availableResources; +}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/FileWriteStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/FileWriteStrategy.java new file mode 100644 index 0000000..5dc9914 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/FileWriteStrategy.java
@@ -0,0 +1,73 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.exec; + +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.analysis.actions.FileWriteActionContext; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * A strategy for executing an {@link AbstractFileWriteAction}. + */ +@ExecutionStrategy(name = { "local" }, contextType = FileWriteActionContext.class) +public final class FileWriteStrategy implements FileWriteActionContext { + + public static final Class<FileWriteStrategy> TYPE = FileWriteStrategy.class; + + public FileWriteStrategy() { + } + + @Override + public void exec(Executor executor, AbstractFileWriteAction action, FileOutErr outErr, + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException { + EventHandler reporter = executor == null ? null : executor.getEventHandler(); + try { + Path outputPath = Iterables.getOnlyElement(action.getOutputs()).getPath(); + try (OutputStream out = new BufferedOutputStream(outputPath.getOutputStream())) { + action.newDeterministicWriter(reporter, executor).writeOutputFile(out); + } + if (action.makeExecutable()) { + outputPath.setExecutable(true); + } + } catch (IOException e) { + throw new EnvironmentalExecException("failed to create file '" + + Iterables.getOnlyElement(action.getOutputs()).prettyPrint() + + "' due to I/O error: " + e.getMessage(), e); + } + } + + @Override + public ResourceSet estimateResourceConsumption(AbstractFileWriteAction action) { + return action.estimateResourceConsumptionLocal(); + } + + @Override + public String strategyLocality(AbstractFileWriteAction action) { + return "local"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/OutputService.java b/src/main/java/com/google/devtools/build/lib/exec/OutputService.java new file mode 100644 index 0000000..88d9b94 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/OutputService.java
@@ -0,0 +1,122 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.exec; + +import com.google.devtools.build.lib.actions.BuildFailedException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.vfs.BatchStat; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; + +/** + * An OutputService retains control over the Blaze output tree, and provides a higher level of + * abstraction compared to the VFS layer. + * + * <p>Higher-level facilities include batch statting, cleaning the output tree, creating symlink + * trees, and out-of-band insertion of metadata into the tree. + */ +public interface OutputService { + + /** + * @return the name of filesystem, akin to what you might see in /proc/mounts + */ + String getFilesSystemName(); + + /** + * @return true if the output service uses FUSE + */ + boolean usesFuse(); + + /** + * @return a human-readable, one word name for the service + */ + String getName(); + + /** + * Start the build. + * + * @throws BuildFailedException if build preparation failed + * @throws InterruptedException + */ + void startBuild() throws BuildFailedException, AbruptExitException, InterruptedException; + + /** + * Finish the build. + * + * @param buildSuccessful iff build was successful + * @throws BuildFailedException on failure + */ + void finalizeBuild(boolean buildSuccessful) throws BuildFailedException, AbruptExitException; + + /** + * Stages the given tool from the package path, possibly copying it to local disk. + * + * @param tool target representing the tool to stage + * @return a Path pointing to the staged target + */ + Path stageTool(Target tool) throws IOException; + + /** + * @return the name of the workspace this output service controls. + */ + String getWorkspace(); + + /** + * @return the BatchStat instance or null. + */ + BatchStat getBatchStatter(); + + /** + * @return true iff createSymlinkTree() is available. + */ + boolean canCreateSymlinkTree(); + + /** + * Creates the symlink tree + * + * @param inputPath the input manifest + * @param outputPath the output manifest + * @param filesetTree is true iff we're constructing a Fileset + * @param symlinkTreeRoot the symlink tree root, relative to the execRoot + * @throws ExecException on failure + * @throws InterruptedException + */ + void createSymlinkTree(Path inputPath, Path outputPath, boolean filesetTree, + PathFragment symlinkTreeRoot) throws ExecException, InterruptedException; + + /** + * Cleans the entire output tree. + * + * @throws ExecException on failure + * @throws InterruptedException + */ + void clean() throws ExecException, InterruptedException; + + /** + * @param file the File + * @return true iff the file actually lives on a remote server + */ + boolean isRemoteFile(Path file); + + /** + * @param path a fully-resolved path + * @return true iff path is under this output service's control + */ + boolean resolvedPathUnderTree(Path path); +}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java b/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java new file mode 100644 index 0000000..8ec1e51 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java
@@ -0,0 +1,143 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +import static java.nio.charset.StandardCharsets.US_ASCII; + +import com.google.common.base.Preconditions; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.Maps; +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.DigestOfDirectoryException; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.protobuf.ByteString; + +import java.io.File; +import java.io.IOException; +import java.util.Map; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +/** + * An in-memory cache to ensure we do I/O for source files only once during a single build. + * + * <p>Simply maintains a two-way cached mapping from digest <--> filename that may be populated + * only once. + */ +@ThreadSafe +public class SingleBuildFileCache implements ActionInputFileCache { + + private final String cwd; + private final FileSystem fs; + + public SingleBuildFileCache(String cwd, FileSystem fs) { + this.fs = Preconditions.checkNotNull(fs); + this.cwd = Preconditions.checkNotNull(cwd); + } + + // If we can't get the digest, we store the exception. This avoids extra file IO for files + // that are allowed to be missing, as we first check a likely non-existent content file + // first. Further we won't need to unwrap the exception in getDigest(). + private final LoadingCache<ActionInput, Pair<ByteString, IOException>> pathToDigest = + CacheBuilder.newBuilder() + // We default to 10 disk read threads, but we don't expect them all to edit the map + // simultaneously. + .concurrencyLevel(8) + // Even small-ish builds, as of 11/21/2011 typically have over 10k artifacts, so it's + // unlikely that this default will adversely affect memory in most cases. + .initialCapacity(10000) + .build(new CacheLoader<ActionInput, Pair<ByteString, IOException>>() { + @Override + public Pair<ByteString, IOException> load(ActionInput input) { + Path path = null; + try { + path = fs.getPath(fullPath(input)); + BaseEncoding hex = BaseEncoding.base16().lowerCase(); + ByteString digest = ByteString.copyFrom( + hex.encode(path.getMD5Digest()) + .getBytes(US_ASCII)); + pathToBytes.put(input, path.getFileSize()); + // Inject reverse mapping. Doing this unconditionally in getDigest() showed up + // as a hotspot in CPU profiling. + digestToPath.put(digest, input); + return Pair.of(digest, null); + } catch (IOException e) { + if (path != null && path.isDirectory()) { + pathToBytes.put(input, 0L); + return Pair.<ByteString, IOException>of(null, new DigestOfDirectoryException( + "Input is a directory: " + input.getExecPathString())); + } + + // Put value into size map to avoid trying to read file again later. + pathToBytes.put(input, 0L); + return Pair.of(null, e); + } + } + }); + + private final Map<ByteString, ActionInput> digestToPath = Maps.newConcurrentMap(); + + private final Map<ActionInput, Long> pathToBytes = Maps.newConcurrentMap(); + + @Nullable + @Override + public File getFileFromDigest(ByteString digest) { + ActionInput relPath = digestToPath.get(digest); + return relPath == null ? null : new File(fullPath(relPath)); + } + + @Override + public long getSizeInBytes(ActionInput input) throws IOException { + // TODO(bazel-team): this only works if pathToDigest has already been called. + Long sz = pathToBytes.get(input); + if (sz != null) { + return sz; + } + Path path = fs.getPath(fullPath(input)); + sz = path.getFileSize(); + pathToBytes.put(input, sz); + return sz; + } + + @Override + public ByteString getDigest(ActionInput input) throws IOException { + Pair<ByteString, IOException> result = pathToDigest.getUnchecked(input); + if (result.second != null) { + throw result.second; + } + return result.first; + } + + @Override + public boolean contentsAvailableLocally(ByteString digest) { + return digestToPath.containsKey(digest); + } + + /** + * Creates a File object that refers to fileName, if fileName is an absolute path. Otherwise, + * returns a File object that refers to the fileName appended to the (absolute) current working + * directory. + */ + private String fullPath(ActionInput input) { + String relPath = input.getExecPathString(); + return relPath.startsWith("/") ? relPath : new File(cwd, relPath).getPath(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SourceManifestActionContextImpl.java b/src/main/java/com/google/devtools/build/lib/exec/SourceManifestActionContextImpl.java new file mode 100644 index 0000000..40fed77 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/SourceManifestActionContextImpl.java
@@ -0,0 +1,37 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.exec; + +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.analysis.SourceManifestAction; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * A context for {@link SourceManifestAction} that uses the runtime to determine + * the workspace suffix. + */ +@ExecutionStrategy(contextType = SourceManifestAction.Context.class) +public class SourceManifestActionContextImpl implements SourceManifestAction.Context { + private final PathFragment runfilesPrefix; + + public SourceManifestActionContextImpl(PathFragment runfilesPrefix) { + this.runfilesPrefix = runfilesPrefix; + } + + @Override + public PathFragment getRunfilesPrefix() { + return runfilesPrefix; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java new file mode 100644 index 0000000..6127cee --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java
@@ -0,0 +1,137 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ResourceManager; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.util.CommandBuilder; +import com.google.devtools.build.lib.util.OsUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.List; + +/** + * Helper class responsible for the symlink tree creation. + * Used to generate runfiles and fileset symlink farms. + */ +public final class SymlinkTreeHelper { + + private static final String BUILD_RUNFILES = "build-runfiles" + OsUtils.executableExtension(); + + /** + * These actions run faster overall when serialized, because most of their + * cost is in the ext2 block allocator, and there's less seeking required if + * their directory creations get non-interleaved allocations. So we give them + * a huge resource cost. + */ + public static final ResourceSet RESOURCE_SET = new ResourceSet(1000, 0.5, 0.75); + + private final PathFragment inputManifest; + private final PathFragment symlinkTreeRoot; + private final boolean filesetTree; + + /** + * Creates SymlinkTreeHelper instance. Can be used independently of + * SymlinkTreeAction. + * + * @param inputManifest exec path to the input runfiles manifest + * @param symlinkTreeRoot exec path to the symlink tree location + * @param filesetTree true if this is fileset symlink tree, + * false if this is a runfiles symlink tree. + */ + public SymlinkTreeHelper(PathFragment inputManifest, PathFragment symlinkTreeRoot, + boolean filesetTree) { + this.inputManifest = inputManifest; + this.symlinkTreeRoot = symlinkTreeRoot; + this.filesetTree = filesetTree; + } + + public PathFragment getSymlinkTreeRoot() { return symlinkTreeRoot; } + + /** + * Creates a symlink tree using a CommandBuilder. This means that the symlink + * tree will always be present on the developer's workstation. Useful when + * running commands locally. + * + * Warning: this method REALLY executes the command on the box Blaze was + * run on, without any kind of synchronization, locking, or anything else. + * + * @param config the configuration that is used for creating the symlink tree. + * @throws CommandException + */ + public void createSymlinksUsingCommand(Path execRoot, + BuildConfiguration config, BinTools binTools) throws CommandException { + List<String> argv = getSpawnArgumentList(execRoot, binTools); + + CommandBuilder builder = new CommandBuilder(); + builder.addArgs(argv); + builder.setWorkingDir(execRoot); + builder.build().execute(); + } + + /** + * Creates symlink tree using appropriate method. At this time tree + * always created using build-runfiles helper application. + * + * Note: method may try to acquire resources - meaning that it would + * block for undetermined period of time. If it is interrupted during + * that wait, ExecException will be thrown but interrupted bit will be + * preserved. + * + * @param action action instance that requested symlink tree creation + * @param actionExecutionContext Services that are in the scope of the action. + */ + public void createSymlinks(AbstractAction action, ActionExecutionContext actionExecutionContext, + BinTools binTools) throws ExecException, InterruptedException { + List<String> args = getSpawnArgumentList( + actionExecutionContext.getExecutor().getExecRoot(), binTools); + try { + ResourceManager.instance().acquireResources(action, RESOURCE_SET); + actionExecutionContext.getExecutor().getSpawnActionContext(action.getMnemonic()).exec( + new BaseSpawn.Local(args, ImmutableMap.<String, String>of(), action), + actionExecutionContext); + } finally { + ResourceManager.instance().releaseResources(action, RESOURCE_SET); + } + } + + /** + * Returns the complete argument list build-runfiles has to be called with. + */ + private List<String> getSpawnArgumentList(Path execRoot, BinTools binTools) { + List<String> args = Lists.newArrayList( + execRoot.getRelative(binTools.getExecPath(BUILD_RUNFILES)) + .getPathString()); + + if (filesetTree) { + args.add("--allow_relative"); + args.add("--use_metadata"); + } + + args.add(inputManifest.getPathString()); + args.add(symlinkTreeRoot.getPathString()); + + return args; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java new file mode 100644 index 0000000..d9470e4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
@@ -0,0 +1,60 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.analysis.SymlinkTreeAction; +import com.google.devtools.build.lib.analysis.SymlinkTreeActionContext; +import com.google.devtools.build.lib.analysis.config.BinTools; + +/** + * Implements SymlinkTreeAction by using the output service or by running an embedded script to + * create the symlink tree. + */ +@ExecutionStrategy(contextType = SymlinkTreeActionContext.class) +public final class SymlinkTreeStrategy implements SymlinkTreeActionContext { + private final OutputService outputService; + private final BinTools binTools; + + public SymlinkTreeStrategy(OutputService outputService, BinTools binTools) { + this.outputService = outputService; + this.binTools = binTools; + } + + @Override + public void createSymlinks(SymlinkTreeAction action, + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + try { + SymlinkTreeHelper helper = new SymlinkTreeHelper( + action.getInputManifest().getExecPath(), + action.getOutputManifest().getExecPath().getParentDirectory(), action.isFilesetTree()); + if (outputService != null && outputService.canCreateSymlinkTree()) { + outputService.createSymlinkTree(action.getInputManifest().getPath(), + action.getOutputManifest().getPath(), + action.isFilesetTree(), helper.getSymlinkTreeRoot()); + } else { + helper.createSymlinks(action, actionExecutionContext, binTools); + } + } catch (ExecException e) { + throw e.toActionExecutionException( + action.getProgressMessage(), executor.getVerboseFailures(), action); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/AbstractGraphVisitor.java b/src/main/java/com/google/devtools/build/lib/graph/AbstractGraphVisitor.java new file mode 100644 index 0000000..a08ed42 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/graph/AbstractGraphVisitor.java
@@ -0,0 +1,31 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// All Rights Reserved. + +package com.google.devtools.build.lib.graph; + +/** + * <p> A stub implementation of GraphVisitor providing default behaviour (do + * nothing) for all its methods. </p> + */ +public class AbstractGraphVisitor<T> implements GraphVisitor<T> { + @Override + public void beginVisit() {} + @Override + public void endVisit() {} + @Override + public void visitEdge(Node<T> lhs, Node<T> rhs) {} + @Override + public void visitNode(Node<T> node) {} +}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/CollectingVisitor.java b/src/main/java/com/google/devtools/build/lib/graph/CollectingVisitor.java new file mode 100644 index 0000000..caeb07b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/graph/CollectingVisitor.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.graph; + +import java.util.ArrayList; +import java.util.List; + +/** + * A graph visitor that collects the visited nodes in the order in which + * they were visited, and allows them to be accessed as a list. + */ +public class CollectingVisitor<T> extends AbstractGraphVisitor<T> { + + private final List<Node<T>> order = new ArrayList<Node<T>>(); + + @Override + public void visitNode(Node<T> node) { + order.add(node); + } + + /** + * Returns a reference to (not a copy of) the list of visited nodes in the + * order they were visited. + */ + public List<Node<T>> getVisitedNodes() { + return order; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/DFS.java b/src/main/java/com/google/devtools/build/lib/graph/DFS.java new file mode 100644 index 0000000..37cd30e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/graph/DFS.java
@@ -0,0 +1,118 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.graph; + +import com.google.common.collect.Lists; + +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * <p> The DFS class encapsulates a depth-first search visitation, including + * the order in which nodes are to be visited relative to their successors + * (PREORDER/POSTORDER), whether the forward or transposed graph is to be + * used, and which nodes have been seen already. </p> + * + * <p> A variety of common uses of DFS are offered through methods of + * Digraph; however clients can use this class directly for maximum + * flexibility. See the implementation of + * Digraph.getStronglyConnectedComponents() for an example. </p> + * + * <p> Clients should not modify the enclosing Digraph instance of a DFS + * while a traversal is in progress. </p> + */ +public class DFS<T> { + + // (Preferred over a boolean to avoid parameter confusion.) + public enum Order { + PREORDER, + POSTORDER + } + + private final Order order; // = (PREORDER|POSTORDER) + + private final Comparator<Node<T>> edgeOrder; + + private final boolean transpose; + + private final Set<Node<T>> marked = new HashSet<Node<T>>(); + + /** + * Constructs a DFS instance for searching over the enclosing Digraph + * instance, using the specified visitation parameters. + * + * @param order PREORDER or POSTORDER, determines node visitation order + * @param edgeOrder an ordering in which the edges originating from the same + * node should be visited (if null, the order is unspecified) + * @param transpose iff true, the graph is implicitly transposed during + * visitation. + */ + public DFS(Order order, final Comparator<T> edgeOrder, boolean transpose) { + this.order = order; + this.transpose = transpose; + + if (edgeOrder == null) { + this.edgeOrder = null; + } else { + this.edgeOrder = new Comparator<Node<T>>() { + @Override + public int compare(Node<T> o1, Node<T> o2) { + return edgeOrder.compare(o1.getLabel(), o2.getLabel()); + } + }; + } + } + + public DFS(Order order, boolean transpose) { + this(order, null, transpose); + } + + /** + * Returns the (immutable) set of nodes visited so far. + */ + public Set<Node<T>> getMarked() { + return Collections.unmodifiableSet(marked); + } + + public void visit(Node<T> node, GraphVisitor<T> visitor) { + if (!marked.add(node)) { + return; + } + + if (order == Order.PREORDER) { + visitor.visitNode(node); + } + + Collection<Node<T>> edgeTargets = transpose + ? node.getPredecessors() : node.getSuccessors(); + if (edgeOrder != null) { + List<Node<T>> mutableNodeList = Lists.newArrayList(edgeTargets); + Collections.sort(mutableNodeList, edgeOrder); + edgeTargets = mutableNodeList; + } + + for (Node<T> v: edgeTargets) { + visit(v, visitor); + } + + if (order == Order.POSTORDER) { + visitor.visitNode(node); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/Digraph.java b/src/main/java/com/google/devtools/build/lib/graph/Digraph.java new file mode 100644 index 0000000..bea2f5a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/graph/Digraph.java
@@ -0,0 +1,1063 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.graph; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.common.collect.Ordering; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.Set; + +/** + * <p> {@code Digraph} a generic directed graph or "digraph", suitable for + * modeling asymmetric binary relations. </p> + * + * <p> An instance <code>G = <V,E></code> consists of a set of nodes or + * vertices <code>V</code>, and a set of directed edges <code>E</code>, which + * is a subset of <code>V × V</code>. This permits self-edges but does + * not represent multiple edges between the same pair of nodes. </p> + * + * <p> Nodes may be labeled with values of any type (type parameter + * T). All nodes within a graph have distinct labels. The null + * pointer is not a valid label.</p> + * + * <p> The package supports various operations for modeling partial order + * relations, and supports input/output in AT&T's 'dot' format. See + * http://www.research.att.com/sw/tools/graphviz/. </p> + * + * <p> Some invariants: </p> + * <ul> + * + * <li> Each graph instances "owns" the nodes is creates. The behaviour of + * operations on nodes a graph does not own is undefined. + * + * <li> {@code Digraph} assumes immutability of node labels, much like {@link + * HashMap} assumes it for keys. + * + * <li> Mutating the underlying graph invalidates any sets and iterators backed + * by it. + * + * </ul> + * + * <p>Each node stores successor and predecessor adjacency sets using a + * representation that dynamically changes with size: small sets are stored as + * arrays, large sets using hash tables. This representation provides + * significant space and time performance improvements upon two prior versions: + * the earliest used only HashSets; a later version used linked lists, as + * described in Cormen, Leiserson & Rivest. + */ +public final class Digraph<T> implements Cloneable { + + /** + * Maps labels to nodes, which are in strict 1:1 correspondence. + */ + private final HashMap<T, Node<T>> nodes = Maps.newHashMap(); + + /** + * A source of unique, deterministic hashCodes for {@link Node} instances. + */ + private int nextHashCode = 0; + + /** + * Construct an empty Digraph. + */ + public Digraph() {} + + /** + * Sanity-check: assert that a node is indeed a member of this graph and not + * another one. Perform this check whenever a function is supplied a node by + * the user. + */ + private final void checkNode(Node<T> node) { + if (getNode(node.getLabel()) != node) { + throw new IllegalArgumentException("node " + node + + " is not a member of this graph"); + } + } + + /** + * Adds a directed edge between the nodes labelled 'from' and 'to', creating + * them if necessary. + * + * @return true iff the edge was not already present. + */ + public boolean addEdge(T from, T to) { + Node<T> fromNode = createNode(from); + Node<T> toNode = createNode(to); + return addEdge(fromNode, toNode); + } + + /** + * Adds a directed edge between the specified nodes, which must exist and + * belong to this graph. + * + * @return true iff the edge was not already present. + * + * Note: multi-edges are ignored. Self-edges are permitted. + */ + public boolean addEdge(Node<T> fromNode, Node<T> toNode) { + checkNode(fromNode); + checkNode(toNode); + boolean isNewSuccessor = fromNode.addSuccessor(toNode); + boolean isNewPredecessor = toNode.addPredecessor(fromNode); + if (isNewPredecessor != isNewSuccessor) { + throw new IllegalStateException(); + } + return isNewSuccessor; + } + + /** + * Returns true iff the graph contains an edge between the + * specified nodes, which must exist and belong to this graph. + */ + public boolean containsEdge(Node<T> fromNode, Node<T> toNode) { + checkNode(fromNode); + checkNode(toNode); + // TODO(bazel-team): (2009) iterate only over the shorter of from.succs, to.preds. + return fromNode.getSuccessors().contains(toNode); + } + + /** + * Removes the edge between the specified nodes. Idempotent: attempts to + * remove non-existent edges have no effect. + * + * @return true iff graph changed. + */ + public boolean removeEdge(Node<T> fromNode, Node<T> toNode) { + checkNode(fromNode); + checkNode(toNode); + boolean changed = fromNode.removeSuccessor(toNode); + if (changed) { + toNode.removePredecessor(fromNode); + } + return changed; + } + + /** + * Remove all nodes and edges. + */ + public void clear() { + nodes.clear(); + } + + @Override + public String toString() { + return "Digraph[" + getNodeCount() + " nodes]"; + } + + @Override + public int hashCode() { + throw new UnsupportedOperationException(); // avoid nondeterminism + } + + /** + * Returns true iff the two graphs are equivalent, i.e. have the same set + * of node labels, with the same connectivity relation. + * + * O(n^2) in the worst case, i.e. equivalence. The algorithm could be speed up by + * close to a factor 2 in the worst case by a more direct implementation instead + * of using isSubgraph twice. + */ + @Override + public boolean equals(Object thatObject) { + /* If this graph is a subgraph of thatObject, then we know that thatObject is of + * type Digraph<?> and thatObject can be cast to this type. + */ + return isSubgraph(thatObject) && ((Digraph<?>) thatObject).isSubgraph(this); + } + + /** + * Returns true iff this graph is a subgraph of the argument. This means that this graph's nodes + * are a subset of those of the argument; moreover, for each node of this graph the set of + * successors is a subset of those of the corresponding node in the argument graph. + * + * This algorithm is O(n^2), but linear in the total sizes of the graphs. + */ + public boolean isSubgraph(Object thatObject) { + if (this == thatObject) { + return true; + } + if (!(thatObject instanceof Digraph)) { + return false; + } + + @SuppressWarnings("unchecked") + Digraph<T> that = (Digraph<T>) thatObject; + if (this.getNodeCount() > that.getNodeCount()) { + return false; + } + for (Node<T> n1: nodes.values()) { + Node<T> n2 = that.getNodeMaybe(n1.getLabel()); + if (n2 == null) { + return false; // 'that' is missing a node + } + + // Now compare the successor relations. + // Careful: + // - We can't do simple equality on the succs-sets because the + // nodes belong to two different graphs! + // - There's no need to check both predecessor and successor + // relations, either one is sufficient. + Collection<Node<T>> n1succs = n1.getSuccessors(); + Collection<Node<T>> n2succs = n2.getSuccessors(); + if (n1succs.size() > n2succs.size()) { + return false; + } + // foreach successor of n1, ensure n2 has a similarly-labeled succ. + for (Node<T> succ1: n1succs) { + Node<T> succ2 = that.getNodeMaybe(succ1.getLabel()); + if (succ2 == null) { + return false; + } + if (!n2succs.contains(succ2)) { + return false; + } + } + } + return true; + } + + /** + * Returns a duplicate graph with the same set of node labels and the same + * connectivity relation. The labels themselves are not cloned. + */ + @Override + public Digraph<T> clone() { + final Digraph<T> that = new Digraph<T>(); + visitNodesBeforeEdges(new AbstractGraphVisitor<T>() { + @Override + public void visitEdge(Node<T> lhs, Node<T> rhs) { + that.addEdge(lhs.getLabel(), rhs.getLabel()); + } + @Override + public void visitNode(Node<T> node) { + that.createNode(node.getLabel()); + } + }); + return that; + } + + /** + * Returns a deterministic immutable view of the nodes of this graph. + */ + public Collection<Node<T>> getNodes(final Comparator<T> comparator) { + Ordering<Node<T>> ordering = new Ordering<Node<T>>() { + @Override + public int compare(Node<T> o1, Node<T> o2) { + return comparator.compare(o1.getLabel(), o2.getLabel()); + } + }; + return ordering.immutableSortedCopy(nodes.values()); + } + + /** + * Returns an immutable view of the nodes of this graph. + * + * Note: we have to return Collection and not Set because values() returns + * one: the 'nodes' HashMap doesn't know that it is injective. :-( + */ + public Collection<Node<T>> getNodes() { + return Collections.unmodifiableCollection(nodes.values()); + } + + /** + * @return the set of root nodes: those with no predecessors. + * + * NOTE: in a cyclic graph, there may be nodes that are not reachable from + * any "root". + */ + public Set<Node<T>> getRoots() { + Set<Node<T>> roots = new HashSet<Node<T>>(); + for (Node<T> node: nodes.values()) { + if (!node.hasPredecessors()) { + roots.add(node); + } + } + return roots; + } + + /** + * @return the set of leaf nodes: those with no successors. + */ + public Set<Node<T>> getLeaves() { + Set<Node<T>> leaves = new HashSet<Node<T>>(); + for (Node<T> node: nodes.values()) { + if (!node.hasSuccessors()) { + leaves.add(node); + } + } + return leaves; + } + + /** + * @return an immutable view of the set of labels of this graph's nodes. + */ + public Set<T> getLabels() { + return Collections.unmodifiableSet(nodes.keySet()); + } + + /** + * Finds and returns the node with the specified label. If there is no such + * node, an exception is thrown. The null pointer is not a valid label. + * + * @return the node whose label is "label". + * @throws IllegalArgumentException if no node was found with the specified + * label. + */ + public Node<T> getNode(T label) { + if (label == null) { + throw new NullPointerException(); + } + Node<T> node = nodes.get(label); + if (node == null) { + throw new IllegalArgumentException("No such node label: " + label); + } + return node; + } + + /** + * Find the node with the specified label. Returns null if it doesn't exist. + * The null pointer is not a valid label. + * + * @return the node whose label is "label", or null if it was not found. + */ + public Node<T> getNodeMaybe(T label) { + if (label == null) { + throw new NullPointerException(); + } + return nodes.get(label); + } + + /** + * @return the number of nodes in the graph. + */ + public int getNodeCount() { + return nodes.size(); + } + + /** + * @return the number of edges in the graph. + * + * Note: expensive! Useful when asserting against mutations though. + */ + public int getEdgeCount() { + int edges = 0; + for (Node<T> node: nodes.values()) { + edges += node.getSuccessors().size(); + } + return edges; + } + + /** + * Find or create a node with the specified label. This is the <i>only</i> + * factory of Nodes. The null pointer is not a valid label. + */ + public Node<T> createNode(T label) { + if (label == null) { + throw new NullPointerException(); + } + Node<T> n = nodes.get(label); + if (n == null) { + nodes.put(label, n = new Node<T>(label, nextHashCode++)); + } + return n; + } + + /****************************************************************** + * * + * Graph Algorithms * + * * + ******************************************************************/ + + /** + * These only manipulate the graph through methods defined above. + */ + + /** + * Returns true iff the graph is cyclic. Time: O(n). + */ + public boolean isCyclic() { + + // To detect cycles, we use a colored depth-first search. All nodes are + // initially marked white. When a node is encountered, it is marked grey, + // and when its descendants are completely visited, it is marked black. + // If a grey node is ever encountered, then there is a cycle. + final Object WHITE = null; // i.e. not present in nodeToColor, the default. + final Object GREY = new Object(); + final Object BLACK = new Object(); + final Map<Node<T>, Object> nodeToColor = + new HashMap<Node<T>, Object>(); // empty => all white + + class CycleDetector { /* defining a class gives us lexical scope */ + boolean visit(Node<T> node) { + nodeToColor.put(node, GREY); + for (Node<T> succ: node.getSuccessors()) { + if (nodeToColor.get(succ) == GREY) { + return true; + } else if (nodeToColor.get(succ) == WHITE) { + if (visit(succ)) { + return true; + } + } + } + nodeToColor.put(node, BLACK); + return false; + } + } + + CycleDetector detector = new CycleDetector(); + for (Node<T> node: nodes.values()) { + if (nodeToColor.get(node) == WHITE) { + if (detector.visit(node)) { + return true; + } + } + } + return false; + } + + /** + * Returns the strong component graph of "this". That is, returns a new + * acyclic graph in which all strongly-connected components in the original + * graph have been "fused" into a single node. + * + * @return a new graph, whose node labels are sets of nodes of the + * original graph. (Do not get confused as to which graph each + * set of Nodes belongs!) + */ + public Digraph<Set<Node<T>>> getStrongComponentGraph() { + Collection<Set<Node<T>>> sccs = getStronglyConnectedComponents(); + Digraph<Set<Node<T>>> scGraph = createImageUnderPartition(sccs); + scGraph.removeSelfEdges(); // scGraph should be acyclic: no self-edges + return scGraph; + } + + /** + * Returns a partition of the nodes of this graph into sets, each set being + * one strongly-connected component of the graph. + */ + public Collection<Set<Node<T>>> getStronglyConnectedComponents() { + final List<Set<Node<T>>> sccs = new ArrayList<Set<Node<T>>>(); + NodeSetReceiver<T> r = new NodeSetReceiver<T>() { + @Override + public void accept(Set<Node<T>> scc) { + sccs.add(scc); + } + }; + SccVisitor<T> v = new SccVisitor<T>(); + for (Node<T> node : nodes.values()) { + v.visit(r, node); + } + return sccs; + } + + /** + * <p> Given a partition of the graph into sets of nodes, returns the image + * of this graph under the function which maps each node to the + * partition-set in which it appears. The labels of the new graph are the + * (immutable) sets of the partition, and the edges of the new graph are the + * edges of the original graph, mapped via the same function. </p> + * + * <p> Note: the resulting graph may contain self-edges. If these are not + * wanted, call <code>removeSelfEdges()</code>> on the result. </p> + * + * <p> Interesting special case: if the partition is the set of + * strongly-connected components, the result of this function is the + * strong-component graph. </p> + */ + public Digraph<Set<Node<T>>> + createImageUnderPartition(Collection<Set<Node<T>>> partition) { + + // Build mapping function: each node label is mapped to its equiv class: + Map<T, Set<Node<T>>> labelToImage = + new HashMap<T, Set<Node<T>>>(); + for (Set<Node<T>> set: partition) { + // It's important to use immutable sets of node labels when sets are keys + // in a map; see ImmutableSet class for explanation. + Set<Node<T>> imageSet = ImmutableSet.copyOf(set); + for (Node<T> node: imageSet) { + labelToImage.put(node.getLabel(), imageSet); + } + } + + if (labelToImage.size() != getNodeCount()) { + throw new IllegalArgumentException( + "createImageUnderPartition(): argument is not a partition"); + } + + return createImageUnderMapping(labelToImage); + } + + /** + * Returns the image of this graph in a given function, expressed as a + * mapping from labels to some other domain. + */ + public <IMAGE> Digraph<IMAGE> + createImageUnderMapping(Map<T, IMAGE> map) { + + Digraph<IMAGE> imageGraph = new Digraph<IMAGE>(); + + for (Node<T> fromNode: nodes.values()) { + T fromLabel = fromNode.getLabel(); + + IMAGE fromImage = map.get(fromLabel); + if (fromImage == null) { + throw new IllegalArgumentException( + "Incomplete function: undefined for " + fromLabel); + } + imageGraph.createNode(fromImage); + + for (Node<T> toNode: fromNode.getSuccessors()) { + T toLabel = toNode.getLabel(); + + IMAGE toImage = map.get(toLabel); + if (toImage == null) { + throw new IllegalArgumentException( + "Incomplete function: undefined for " + toLabel); + } + imageGraph.addEdge(fromImage, toImage); + } + } + + return imageGraph; + } + + /** + * Removes any self-edges (x,x) in this graph. + */ + public void removeSelfEdges() { + for (Node<T> node: nodes.values()) { + removeEdge(node, node); + } + } + + /** + * Finds the shortest directed path from "fromNode" to "toNode". The path is + * returned as an ordered list of nodes, including both endpoints. Returns + * null if there is no path. Uses breadth-first search. Running time is + * O(n). + */ + public List<Node<T>> getShortestPath(Node<T> fromNode, + Node<T> toNode) { + checkNode(fromNode); + checkNode(toNode); + + if (fromNode == toNode) { + return Collections.singletonList(fromNode); + } + + Map<Node<T>, Node<T>> pathPredecessor = + new HashMap<Node<T>, Node<T>>(); + + Set<Node<T>> marked = new HashSet<Node<T>>(); + + LinkedList<Node<T>> queue = new LinkedList<Node<T>>(); + queue.addLast(fromNode); + marked.add(fromNode); + + while (queue.size() > 0) { + Node<T> u = queue.removeFirst(); + for (Node<T> v: u.getSuccessors()) { + if (marked.add(v)) { + pathPredecessor.put(v, u); + if (v == toNode) { + return getPathToTreeNode(pathPredecessor, v); // found a path + } + queue.addLast(v); + } + } + } + return null; // no path + } + + /** + * Given a tree (expressed as a map from each node to its parent), and a + * starting node, returns the path from the root of the tree to 'node' as a + * list. + */ + private static <X> List<X> getPathToTreeNode(Map<X, X> tree, X node) { + List<X> path = new ArrayList<X>(); + while (node != null) { + path.add(node); + node = tree.get(node); // get parent + } + Collections.reverse(path); + return path; + } + + /** + * Returns the nodes of an acyclic graph in topological order + * [a.k.a "reverse post-order" of depth-first search.] + * + * A topological order is one such that, if (u, v) is a path in + * acyclic graph G, then u is before v in the topological order. + * In other words "tails before heads" or "roots before leaves". + * + * @return The nodes of the graph, in a topological order + */ + public List<Node<T>> getTopologicalOrder() { + List<Node<T>> order = getPostorder(); + Collections.reverse(order); + return order; + } + + /** + * Returns the nodes of an acyclic graph in topological order + * [a.k.a "reverse post-order" of depth-first search.] + * + * A topological order is one such that, if (u, v) is a path in + * acyclic graph G, then u is before v in the topological order. + * In other words "tails before heads" or "roots before leaves". + * + * If an ordering is given, returns a specific topological order from the set + * of all topological orders; if no ordering given, returns an arbitrary + * (nondeterministic) one, but is a bit faster because no sorting needs to be + * done for each node. + * + * @param edgeOrder the ordering in which edges originating from the same node + * are visited. + * @return The nodes of the graph, in a topological order + */ + public List<Node<T>> getTopologicalOrder( + Comparator<T> edgeOrder) { + CollectingVisitor<T> visitor = new CollectingVisitor<T>(); + DFS<T> visitation = new DFS<T>(DFS.Order.POSTORDER, edgeOrder, false); + visitor.beginVisit(); + for (Node<T> node : getNodes(edgeOrder)) { + visitation.visit(node, visitor); + } + visitor.endVisit(); + + List<Node<T>> order = visitor.getVisitedNodes(); + Collections.reverse(order); + return order; + } + + /** + * Returns the nodes of an acyclic graph in post-order. + */ + public List<Node<T>> getPostorder() { + CollectingVisitor<T> collectingVisitor = new CollectingVisitor<T>(); + visitPostorder(collectingVisitor); + return collectingVisitor.getVisitedNodes(); + } + + /** + * Returns the (immutable) set of nodes reachable from node 'n' (reflexive + * transitive closure). + */ + public Set<Node<T>> getFwdReachable(Node<T> n) { + return getFwdReachable(Collections.singleton(n)); + } + + /** + * Returns the (immutable) set of nodes reachable from any node in {@code + * startNodes} (reflexive transitive closure). + */ + public Set<Node<T>> getFwdReachable(Collection<Node<T>> startNodes) { + // This method is intentionally not static, to permit future expansion. + DFS<T> dfs = new DFS<T>(DFS.Order.PREORDER, false); + for (Node<T> n : startNodes) { + dfs.visit(n, new AbstractGraphVisitor<T>()); + } + return dfs.getMarked(); + } + + /** + * Returns the (immutable) set of nodes that reach node 'n' (reflexive + * transitive closure). + */ + public Set<Node<T>> getBackReachable(Node<T> n) { + return getBackReachable(Collections.singleton(n)); + } + + /** + * Returns the (immutable) set of nodes that reach some node in {@code + * startNodes} (reflexive transitive closure). + */ + public Set<Node<T>> getBackReachable(Collection<Node<T>> startNodes) { + // This method is intentionally not static, to permit future expansion. + DFS<T> dfs = new DFS<T>(DFS.Order.PREORDER, true); + for (Node<T> n : startNodes) { + dfs.visit(n, new AbstractGraphVisitor<T>()); + } + return dfs.getMarked(); + } + + /** + * Removes the node in the graph specified by the given label. Optionally, + * preserves the graph order (by connecting up the broken edges) or drop them + * all. If the specified label is not the label of any node in the graph, + * does nothing. + * + * @param label the label of the node to remove. + * @param preserveOrder if true, adds edges between the neighbours + * of the removed node so as to maintain the graph ordering + * relation between all pairs of such nodes. If false, simply + * discards all edges from the deleted node to its neighbours. + * @return true iff 'label' identifies a node (i.e. the graph was changed). + */ + public boolean removeNode(T label, boolean preserveOrder) { + Node<T> node = getNodeMaybe(label); + if (node != null) { + removeNode(node, preserveOrder); + return true; + } else { + return false; + } + } + + /** + * Removes the specified node in the graph. + * + * @param n the node to remove (must be in the graph). + * @param preserveOrder see removeNode(T, boolean). + */ + public void removeNode(Node<T> n, boolean preserveOrder) { + checkNode(n); + for (Node<T> b: n.getSuccessors()) { // edges from n + // exists: n -> b + if (preserveOrder) { + for (Node<T> a: n.getPredecessors()) { // edges to n + // exists: a -> n + // beware self edges: they prevent n's deletion! + if (a != n && b != n) { + addEdge(a, b); // concurrent mod? + } + } + } + b.removePredecessor(n); // remove edge n->b in b + } + for (Node<T> a: n.getPredecessors()) { // edges to n + a.removeSuccessor(n); // remove edge a->n in a + } + + n.removeAllEdges(); // remove edges b->n and a->n in n + Object del = nodes.remove(n.getLabel()); + if (del != n) { + throw new IllegalStateException(del + " " + n); + } + } + + /** + * Extracts the subgraph G' of this graph G, containing exactly the nodes + * specified by the labels in V', and preserving the original + * <i>transitive</i> graph relation among those nodes. </p> + * + * @param subset a subset of the labels of this graph; the resulting graph + * will have only the nodes with these labels. + */ + public Digraph<T> extractSubgraph(final Set<T> subset) { + Digraph<T> subgraph = this.clone(); + subgraph.subgraph(subset); + return subgraph; + } + + /** + * Removes all nodes from this graph except those whose label is an element of {@code keepLabels}. + * Edges are added so as to preserve the <i>transitive</i> closure relation. + * + * @param keepLabels a subset of the labels of this graph; the resulting graph + * will have only the nodes with these labels. + */ + public void subgraph(final Set<T> keepLabels) { + // This algorithm does the following: + // Let keep = nodes that have labels in keepLabels. + // Let toRemove = nodes \ keep. reachables = successors and predecessors of keep in nodes. + // reachables is the subset of nodes of remove that are an immediate neighbor of some node in + // keep. + // + // Removes all nodes of reachables from keepLabels. + // Until reachables is empty: + // Takes n from reachables + // for all s in succ(n) + // for all p in pred(n) + // add the edge (p, s) + // add s to reachables + // for all p in pred(n) + // add p to reachables + // Remove n and its edges + // + // A few adjustments are needed to do the whole computation. + + final Set<Node<T>> toRemove = new HashSet<>(); + final Set<Node<T>> keepNeighbors = new HashSet<>(); + + // Look for all nodes if they are to be kept or removed + for (Node<T> node : nodes.values()) { + if (keepLabels.contains(node.getLabel())) { + // Node is to be kept + keepNeighbors.addAll(node.getPredecessors()); + keepNeighbors.addAll(node.getSuccessors()); + } else { + // node is to be removed. + toRemove.add(node); + } + } + + if (toRemove.isEmpty()) { + // This premature return is needed to avoid 0-size priority queue creation. + return; + } + + // We use a priority queue to look for low-order nodes first so we don't propagate the high + // number of paths of high-order nodes making the time consumption explode. + // For perfect results we should reorder the set each time we add a new edge but this would + // be too expensive, so this is a good enough approximation. + final PriorityQueue<Node<T>> reachables = new PriorityQueue<>(toRemove.size(), + new Comparator<Node<T>>() { + @Override + public int compare(Node<T> o1, Node<T> o2) { + return Long.compare((long) o1.numPredecessors() * (long) o1.numSuccessors(), + (long) o2.numPredecessors() * (long) o2.numSuccessors()); + } + }); + + // Construct the reachables queue with the list of successors and predecessors of keep in + // toRemove. + keepNeighbors.retainAll(toRemove); + reachables.addAll(keepNeighbors); + toRemove.removeAll(reachables); + + // Remove nodes, least connected first, preserving reachability. + while (!reachables.isEmpty()) { + Node<T> node = reachables.poll(); + for (Node<T> s : node.getSuccessors()) { + if (s == node) { continue; } // ignore self-edge + + for (Node<T> p : node.getPredecessors()) { + if (p == node) { continue; } // ignore self-edge + addEdge(p, s); + } + + // removes n -> s + s.removePredecessor(node); + if (toRemove.remove(s)) { + reachables.add(s); + } + } + + for (Node<T> p : node.getPredecessors()) { + if (p == node) { continue; } // ignore self-edge + p.removeSuccessor(node); + if (toRemove.remove(p)) { + reachables.add(p); + } + } + + // After the node deletion, the graph is again well-formed and the original topological order + // is preserved. + nodes.remove(node.getLabel()); + } + + // Final cleanup for non-reachable nodes. + for (Node<T> node : toRemove) { + removeNode(node, false); + } + } + + private interface NodeSetReceiver<T> { + void accept(Set<Node<T>> nodes); + } + + /** + * Find strongly connected components using path-based strong component + * algorithm. This has the advantage over the default method of returning + * the components in postorder. + * + * We visit nodes depth-first, keeping track of the order that + * we visit them in (preorder). Our goal is to find the smallest node (in + * this preorder of visitation) reachable from a given node. We keep track of the + * smallest node pointed to so far at the top of a stack. If we ever find an + * already-visited node, then if it is not already part of a component, we + * pop nodes from that stack until we reach this already-visited node's number + * or an even smaller one. + * + * Once the depth-first visitation of a node is complete, if this node's + * number is at the top of the stack, then it is the "first" element visited + * in its strongly connected component. Hence we pop all elements that were + * pushed onto the visitation stack and put them in a strongly connected + * component with this one, then send a passed-in {@link Digraph.NodeSetReceiver} this component. + */ + private class SccVisitor<T> { + // Nodes already assigned to a strongly connected component. + private final Set<Node<T>> assigned = new HashSet<Node<T>>(); + // The order each node was visited in. + private final Map<Node<T>, Integer> preorder = new HashMap<Node<T>, Integer>(); + // Stack of all nodes visited whose SCC has not yet been determined. When an SCC is found, + // that SCC is an initial segment of this stack, and is popped off. Every time a new node is + // visited, it is put on this stack. + private final List<Node<T>> stack = new ArrayList<Node<T>>(); + // Stack of visited indices for the first-visited nodes in each of their known-so-far + // strongly connected components. A node pushes its index on when it is visited. If any of + // its successors have already been visited and are not in an already-found strongly connected + // component, then, since the successor was already visited, it and this node must be part of a + // cycle. So every node visited since the successor is actually in the same strongly connected + // component. In this case, preorderStack is popped until the top is at most the successor's + // index. + // + // After all descendants of a node have been visited, if the top element of preorderStack is + // still the current node's index, then it was the first element visited of the current strongly + // connected component. So all nodes on {@code stack} down to the current node are in its + // strongly connected component. And the node's index is popped from preorderStack. + private final List<Integer> preorderStack = new ArrayList<Integer>(); + // Index of node being visited. + private int counter = 0; + + private void visit(NodeSetReceiver<T> visitor, Node<T> node) { + if (preorder.containsKey(node)) { + // This can only happen if this was a non-recursive call, and a previous + // visit call had already visited node. + return; + } + preorder.put(node, counter); + stack.add(node); + preorderStack.add(counter++); + int preorderLength = preorderStack.size(); + for (Node<T> succ : node.getSuccessors()) { + Integer succPreorder = preorder.get(succ); + if (succPreorder == null) { + visit(visitor, succ); + } else { + // Does succ not already belong to an SCC? If it doesn't, then it + // must be in the same SCC as node. The "starting node" of this SCC + // must have been visited before succ (or is succ itself). + if (!assigned.contains(succ)) { + while (preorderStack.get(preorderStack.size() - 1) > succPreorder) { + preorderStack.remove(preorderStack.size() - 1); + } + } + } + } + if (preorderLength == preorderStack.size()) { + // If the length of the preorderStack is unchanged, we did not find any earlier-visited + // nodes that were part of a cycle with this node. So this node is the first-visited + // element in its strongly connected component, and we collect the component. + preorderStack.remove(preorderStack.size() - 1); + Set<Node<T>> scc = new HashSet<Node<T>>(); + Node<T> compNode; + do { + compNode = stack.remove(stack.size() - 1); + assigned.add(compNode); + scc.add(compNode); + } while (!node.equals(compNode)); + visitor.accept(scc); + } + } + } + + /******************************************************************** + * * + * Orders, traversals and visitors * + * * + ********************************************************************/ + + /** + * A visitation over all the nodes in the graph that invokes + * <code>visitor.visitNode()</code> for each node in a depth-first + * post-order: each node is visited <i>after</i> each of its successors; the + * order in which edges are traversed is the order in which they were added + * to the graph. <code>visitor.visitEdge()</code> is not called. + * + * @param startNodes the set of nodes from which to begin the visitation. + */ + public void visitPostorder(GraphVisitor<T> visitor, + Iterable<Node<T>> startNodes) { + visitDepthFirst(visitor, DFS.Order.POSTORDER, false, startNodes); + } + + /** + * Equivalent to {@code visitPostorder(visitor, getNodes())}. + */ + public void visitPostorder(GraphVisitor<T> visitor) { + visitPostorder(visitor, nodes.values()); + } + + /** + * A visitation over all the nodes in the graph that invokes + * <code>visitor.visitNode()</code> for each node in a depth-first + * pre-order: each node is visited <i>before</i> each of its successors; the + * order in which edges are traversed is the order in which they were added + * to the graph. <code>visitor.visitEdge()</code> is not called. + * + * @param startNodes the set of nodes from which to begin the visitation. + */ + public void visitPreorder(GraphVisitor<T> visitor, + Iterable<Node<T>> startNodes) { + visitDepthFirst(visitor, DFS.Order.PREORDER, false, startNodes); + } + + /** + * Equivalent to {@code visitPreorder(visitor, getNodes())}. + */ + public void visitPreorder(GraphVisitor<T> visitor) { + visitPreorder(visitor, nodes.values()); + } + + /** + * A visitation over all the nodes in the graph in depth-first order. See + * DFS constructor for meaning of 'order' and 'transpose' parameters. + * + * @param startNodes the set of nodes from which to begin the visitation. + */ + public void visitDepthFirst(GraphVisitor<T> visitor, + DFS.Order order, + boolean transpose, + Iterable<Node<T>> startNodes) { + DFS<T> visitation = new DFS<T>(order, transpose); + visitor.beginVisit(); + for (Node<T> node: startNodes) { + visitation.visit(node, visitor); + } + visitor.endVisit(); + } + + /** + * A visitation over the graph that visits all nodes and edges in some order + * such that each node is visited before any edge coming out of that node; + * the order is otherwise unspecified. + * + * @param startNodes the set of nodes from which to begin the visitation. + */ + public void visitNodesBeforeEdges(GraphVisitor<T> visitor, + Iterable<Node<T>> startNodes) { + visitor.beginVisit(); + for (Node<T> fromNode: startNodes) { + visitor.visitNode(fromNode); + for (Node<T> toNode: fromNode.getSuccessors()) { + visitor.visitEdge(fromNode, toNode); + } + } + visitor.endVisit(); + } + + /** + * Equivalent to {@code visitNodesBeforeEdges(visitor, getNodes())}. + */ + public void visitNodesBeforeEdges(GraphVisitor<T> visitor) { + visitNodesBeforeEdges(visitor, nodes.values()); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/DotOutputVisitor.java b/src/main/java/com/google/devtools/build/lib/graph/DotOutputVisitor.java new file mode 100644 index 0000000..2d18ac2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/graph/DotOutputVisitor.java
@@ -0,0 +1,93 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// All Rights Reserved. + +package com.google.devtools.build.lib.graph; + +import java.io.PrintWriter; + +/** + * <p> An implementation of GraphVisitor for displaying graphs in dot + * format. </p> + */ +public class DotOutputVisitor<T> implements GraphVisitor<T> { + + /** + * Constructs a dot output visitor. + * + * The visitor writes to writer 'out', and rendering node labels as + * strings using the specified displayer, 'disp'. + */ + public DotOutputVisitor(PrintWriter out, LabelSerializer<T> disp) { + // assert disp != null; + // assert out != null; + this.out = out; + this.disp = disp; + } + + private final LabelSerializer<T> disp; + protected final PrintWriter out; + private boolean closeAtEnd = false; + + @Override + public void beginVisit() { + out.println("digraph mygraph {"); + } + + @Override + public void endVisit() { + out.println("}"); + out.flush(); + if (closeAtEnd) { + out.close(); + } + } + + @Override + public void visitEdge(Node<T> lhs, Node<T> rhs) { + String s_lhs = disp.serialize(lhs); + String s_rhs = disp.serialize(rhs); + out.println("\"" + s_lhs + "\" -> \"" + s_rhs + "\""); + } + + @Override + public void visitNode(Node<T> node) { + out.println("\"" + disp.serialize(node) + "\""); + } + + /****************************************************************** + * * + * Factories * + * * + ******************************************************************/ + + /** + * Create a DotOutputVisitor for output to a writer; uses default + * LabelSerializer. + */ + public static <U> DotOutputVisitor<U> create(PrintWriter writer) { + return new DotOutputVisitor<U>(writer, new DefaultLabelSerializer<U>()); + } + + /** + * The default implementation of LabelSerializer simply serializes + * each node using its toString method. + */ + private static class DefaultLabelSerializer<T> implements LabelSerializer<T> { + @Override + public String serialize(Node<T> node) { + return node.getLabel().toString(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/DotSyntaxException.java b/src/main/java/com/google/devtools/build/lib/graph/DotSyntaxException.java new file mode 100644 index 0000000..adf70aa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/graph/DotSyntaxException.java
@@ -0,0 +1,34 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// All Rights Reserved. + +package com.google.devtools.build.lib.graph; + +import java.io.File; + +/** + * <p> A DotSyntaxException represents a syntax error encountered while + * parsing a dot-format fule. Thrown by createFromDotFile if syntax errors + * are encountered. May also be thrown by implementations of + * LabelDeserializer. </p> + * + * <p> The 'file' and 'lineNumber' fields indicate location of syntax error, + * and are populated externally by Digraph.createFromDotFile(). </p> + */ +public class DotSyntaxException extends Exception { + + public DotSyntaxException(String message) { + super(message); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/GraphVisitor.java b/src/main/java/com/google/devtools/build/lib/graph/GraphVisitor.java new file mode 100644 index 0000000..f7e0f62 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/graph/GraphVisitor.java
@@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// All Rights Reserved. + +package com.google.devtools.build.lib.graph; + +/** + * <p> An graph visitor interface; particularly useful for allowing subclasses + * to specify how to output a graph. The order in which node and edge + * callbacks are made (DFS, BFS, etc) is defined by the choice of Digraph + * visitation method used. </p> + */ +public interface GraphVisitor<T> { + + /** + * Called before visitation commences. + */ + void beginVisit(); + + /** + * Called after visitation is complete. + */ + void endVisit(); + + /** + * <p> Called for each edge. </p> + * + * TODO(bazel-team): This method is not essential, and in all known cases so + * far, the visitEdge code can always be placed within visitNode. Perhaps + * we should remove it, and the begin/end methods, and make this just a + * NodeVisitor? Are there any algorithms for which edge-visitation order is + * important? + */ + void visitEdge(Node<T> lhs, Node<T> rhs); + /** + * Called for each node. + */ + void visitNode(Node<T> node); +}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/LabelDeserializer.java b/src/main/java/com/google/devtools/build/lib/graph/LabelDeserializer.java new file mode 100644 index 0000000..2ae9f96 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/graph/LabelDeserializer.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// All Rights Reserved. + +package com.google.devtools.build.lib.graph; + +/** + * <p> An interface for specifying a graph label de-serialization + * function. </p> + * + * <p> Implementations should provide a means of mapping from the string + * representation to an instance of the graph label type T. </p> + * + * <p> e.g. to construct Digraph{Integer} from a String representation, the + * LabelDeserializer{Integer} implementation would return + * Integer.parseInt(rep). </p> + */ +public interface LabelDeserializer<T> { + + /** + * Returns an instance of the label object (of type T) + * corresponding to serialized representation 'rep'. + * + * @throws DotSyntaxException if 'rep' is invalid. + */ + T deserialize(String rep) throws DotSyntaxException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/LabelSerializer.java b/src/main/java/com/google/devtools/build/lib/graph/LabelSerializer.java new file mode 100644 index 0000000..7386583 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/graph/LabelSerializer.java
@@ -0,0 +1,28 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// All Rights Reserved. + +package com.google.devtools.build.lib.graph; + +/** + * <p> An interface for specifying a user-defined serialization of graph node + * labels as strings. </p> + */ +public interface LabelSerializer<T> { + + /** + * Returns the serialized form of the label of the specified node. + */ + String serialize(Node<T> node); +}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/Matrix.java b/src/main/java/com/google/devtools/build/lib/graph/Matrix.java new file mode 100644 index 0000000..225d3d2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/graph/Matrix.java
@@ -0,0 +1,105 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.graph; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * <p>A simple and inefficient directed graph with the adjacency + * relation represented as a 2-D bit-matrix. </p> + * + * <p> Used as an adjunct to Digraph for performing certain algorithms + * which are more naturally implemented on this representation, + * e.g. transitive closure and reduction. </p> + * + * <p> Not many operations are supported. </p> + */ +final class Matrix<T> { + + /** + * Constructs a square bit-matrix, initially empty, with the ith row/column + * corresponding to the ith element of 'labels', in iteration order. + * + * Does not retain a references to 'labels'. + */ + public Matrix(Set<T> labels) { + this.N = labels.size(); + this.values = new ArrayList<T>(N); + this.indices = new HashMap<T, Integer>(); + this.m = new boolean[N][N]; + + for (T label: labels) { + int idx = values.size(); + values.add(label); + indices.put(label, idx); + } + } + + /** + * Constructs a matrix from the set of logical values specified. There is + * one row/column for each node in the graph, and the entry matrix[i,j] is + * set iff there is an edge in 'graph' from the node labelled values[i] to + * the node labelled values[j]. + */ + public Matrix(Digraph<T> graph) { + this(graph.getLabels()); + + for (Node<T> nfrom: graph.getNodes()) { + Integer ifrom = indices.get(nfrom.getLabel()); + for (Node<T> nto: nfrom.getSuccessors()) { + Integer ito = indices.get(nto.getLabel()); + m[ifrom][ito] = true; + } + } + } + + /** + * The size of one side of the matrix. + */ + private final int N; + + /** + * The logical values associated with each row/column. + */ + private final List<T> values; + + /** + * The mapping from logical values to row/column index. + */ + private final Map<T, Integer> indices; + + /** + * The bit-matrix itself. + * m[from][to] indicates an edge from-->to. + */ + private final boolean[][] m; + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (int ii = 0; ii < N; ++ii) { + for (int jj = 0; jj < N; ++jj) { + sb.append(m[ii][jj] ? '1' : '0'); + } + sb.append(' ').append(values.get(ii)).append('\n'); + } + return sb.toString(); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/graph/Node.java b/src/main/java/com/google/devtools/build/lib/graph/Node.java new file mode 100644 index 0000000..9db2a4c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/graph/Node.java
@@ -0,0 +1,294 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.graph; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +/** + * <p>A generic directed-graph Node class. Type parameter T is the type + * of the node's label. + * + * <p>Each node is identified by a label, which is unique within the graph + * owning the node. + * + * <p>Nodes are immutable, that is, their labels cannot be changed. However, + * their predecessor/successor lists are mutable. + * + * <p>Nodes cannot be created directly by clients. + * + * <p>Clients should not confuse nodes belonging to two different graphs! (Use + * Digraph.checkNode() to catch such errors.) There is no way to find the + * graph to which a node belongs; it is intentionally not represented, to save + * space. + */ +public final class Node<T> { + + private static final int ARRAYLIST_THRESHOLD = 6; + private static final int INITIAL_HASHSET_CAPACITY = 12; + + // The succs and preds set representation changes depending on its size. + // It is implemented using the following collections: + // - null for size = 0. + // - Collections$SingletonList for size = 1. + // - ArrayList(6) for size = [2..6]. + // - HashSet(12) for size > 6. + // These numbers were chosen based on profiling. + + private final T label; + + /** + * A duplicate-free collection of edges from this node. May be null, + * indicating the empty set. + */ + private Collection<Node<T>> succs = null; + + /** + * A duplicate-free collection of edges to this node. May be null, + * indicating the empty set. + */ + private Collection<Node<T>> preds = null; + + private final int hashCode; + + /** + * Only Digraph.createNode() can call this! + */ + Node(T label, int hashCode) { + if (label == null) { throw new NullPointerException("label"); } + this.label = label; + this.hashCode = hashCode; + } + + /** + * Returns the label for this node. + */ + public T getLabel() { + return label; + } + + /** + * Returns a duplicate-free collection of the nodes that this node links to. + */ + public Collection<Node<T>> getSuccessors() { + if (succs == null) { + return Collections.emptyList(); + } else { + return Collections.unmodifiableCollection(succs); + } + } + + /** + * Equivalent to {@code !getSuccessors().isEmpty()} but possibly more + * efficient. + */ + public boolean hasSuccessors() { + return succs != null; + } + + /** + * Equivalent to {@code getSuccessors().size()} but possibly more efficient. + */ + public int numSuccessors() { + return succs == null ? 0 : succs.size(); + } + + /** + * Removes all edges to/from this node. + * Private: breaks graph invariant! + */ + void removeAllEdges() { + this.succs = null; + this.preds = null; + } + + /** + * Returns an (unordered, possibly immutable) set of the nodes that link to + * this node. + */ + public Collection<Node<T>> getPredecessors() { + if (preds == null) { + return Collections.emptyList(); + } else { + return Collections.unmodifiableCollection(preds); + } + } + + /** + * Equivalent to {@code getPredecessors().size()} but possibly more + * efficient. + */ + public int numPredecessors() { + return preds == null ? 0 : preds.size(); + } + + /** + * Equivalent to {@code !getPredecessors().isEmpty()} but possibly more + * efficient. + */ + public boolean hasPredecessors() { + return preds != null; + } + + /** + * Adds 'value' to either the predecessor or successor set, updating the + * appropriate field as necessary. + * @return {@code true} if the set was modified; {@code false} if the set + * was not modified + */ + private boolean add(boolean predecessorSet, Node<T> value) { + final Collection<Node<T>> set = predecessorSet ? preds : succs; + if (set == null) { + // null -> SingletonList + return updateField(predecessorSet, Collections.singletonList(value)); + } + if (set.contains(value)) { + // already exists in this set + return false; + } + int previousSize = set.size(); + if (previousSize == 1) { + // SingletonList -> ArrayList + Collection<Node<T>> newSet = + new ArrayList<Node<T>>(ARRAYLIST_THRESHOLD); + newSet.addAll(set); + newSet.add(value); + return updateField(predecessorSet, newSet); + } else if (previousSize < ARRAYLIST_THRESHOLD) { + // ArrayList + set.add(value); + return true; + } else if (previousSize == ARRAYLIST_THRESHOLD) { + // ArrayList -> HashSet + Collection<Node<T>> newSet = + new HashSet<Node<T>>(INITIAL_HASHSET_CAPACITY); + newSet.addAll(set); + newSet.add(value); + return updateField(predecessorSet, newSet); + } else { + // HashSet + set.add(value); + return true; + } + } + + /** + * Removes 'value' from either 'preds' or 'succs', updating the appropriate + * field as necessary. + * @return {@code true} if the set was modified; {@code false} if the set + * was not modified + */ + private boolean remove(boolean predecessorSet, Node<T> value) { + final Collection<Node<T>> set = predecessorSet ? preds : succs; + if (set == null) { + // null + return false; + } + + int previousSize = set.size(); + if (previousSize == 1) { + if (set.contains(value)) { + // -> null + return updateField(predecessorSet, null); + } else { + return false; + } + } + // now remove the value + if (set.remove(value)) { + // may need to change representation + if (previousSize == 2) { + // -> SingletonList + List<Node<T>> list = + Collections.singletonList(set.iterator().next()); + return updateField(predecessorSet, list); + + } else if (previousSize == 1 + ARRAYLIST_THRESHOLD) { + // -> ArrayList + Collection<Node<T>> newSet = + new ArrayList<Node<T>>(ARRAYLIST_THRESHOLD); + newSet.addAll(set); + return updateField(predecessorSet, newSet); + } + return true; + } + return false; + } + + /** + * Update either the {@link #preds} or {@link #succs} field to point to the + * new set. + * @return {@code true}, because the set must have been updated + */ + private boolean updateField(boolean predecessorSet, + Collection<Node<T>> newSet) { + if (predecessorSet) { + preds = newSet; + } else { + succs = newSet; + } + return true; + } + + + /** + * Add 'to' as a successor of 'this' node. Returns true iff + * the graph changed. Private: breaks graph invariant! + */ + boolean addSuccessor(Node<T> to) { + return add(false, to); + } + + /** + * Add 'from' as a predecessor of 'this' node. Returns true iff + * the graph changed. Private: breaks graph invariant! + */ + boolean addPredecessor(Node<T> from) { + return add(true, from); + } + + /** + * Remove edge: fromNode.succs = {n | n in fromNode.succs && n != toNode} + * Private: breaks graph invariant! + */ + boolean removeSuccessor(Node<T> to) { + return remove(false, to); + } + + /** + * Remove edge: toNode.preds = {n | n in toNode.preds && n != fromNode} + * Private: breaks graph invariant! + */ + boolean removePredecessor(Node<T> from) { + return remove(true, from); + } + + @Override + public String toString() { + return "node:" + label; + } + + @Override + public int hashCode() { + return hashCode; // Fast, deterministic. + } + + @Override + public boolean equals(Object that) { + return this == that; // Nodes are unique for a given label + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AbstractAttributeMapper.java b/src/main/java/com/google/devtools/build/lib/packages/AbstractAttributeMapper.java new file mode 100644 index 0000000..20b9304 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/AbstractAttributeMapper.java
@@ -0,0 +1,212 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.syntax.Label; + +import javax.annotation.Nullable; + +/** + * Base {@link AttributeMap} implementation providing direct, unmanipulated access to + * underlying attribute data as stored within the Rule. + * + * <p>Any instantiable subclass should define a clear policy of what it does with this + * data before exposing it to consumers. + */ +public abstract class AbstractAttributeMapper implements AttributeMap { + + private final Package pkg; + private final RuleClass ruleClass; + private final Label ruleLabel; + private final AttributeContainer attributes; + + public AbstractAttributeMapper(Package pkg, RuleClass ruleClass, Label ruleLabel, + AttributeContainer attributes) { + this.pkg = pkg; + this.ruleClass = ruleClass; + this.ruleLabel = ruleLabel; + this.attributes = attributes; + } + + @Override + public String getName() { + return ruleLabel.getName(); + } + + @Override + public Label getLabel() { + return ruleLabel; + } + + @Nullable + @Override + public <T> T get(String attributeName, Type<T> type) { + int index = getIndexWithTypeCheck(attributeName, type); + Object value = attributes.getAttributeValue(index); + if (value instanceof Attribute.ComputedDefault) { + value = ((Attribute.ComputedDefault) value).getDefault(this); + } + return type.cast(value); + } + + /** + * Returns the given attribute if it's a computed default, null otherwise. + * + * @throws IllegalArgumentException if the given attribute doesn't exist with the specified + * type. This happens whether or not it's a computed default. + */ + protected <T> Attribute.ComputedDefault getComputedDefault(String attributeName, Type<T> type) { + int index = getIndexWithTypeCheck(attributeName, type); + Object value = attributes.getAttributeValue(index); + if (value instanceof Attribute.ComputedDefault) { + return (Attribute.ComputedDefault) value; + } else { + return null; + } + } + + @Override + public Iterable<String> getAttributeNames() { + ImmutableList.Builder<String> names = ImmutableList.builder(); + for (Attribute a : ruleClass.getAttributes()) { + names.add(a.getName()); + } + return names.build(); + } + + @Nullable + @Override + public Type<?> getAttributeType(String attrName) { + Attribute attr = getAttributeDefinition(attrName); + return attr == null ? null : attr.getType(); + } + + @Nullable + @Override + public Attribute getAttributeDefinition(String attrName) { + return ruleClass.getAttributeByNameMaybe(attrName); + } + + @Override + public boolean isAttributeValueExplicitlySpecified(String attributeName) { + return attributes.isAttributeValueExplicitlySpecified(attributeName); + } + + @Override + public String getPackageDefaultHdrsCheck() { + return pkg.getDefaultHdrsCheck(); + } + + @Override + public Boolean getPackageDefaultObsolete() { + return pkg.getDefaultObsolete(); + } + + @Override + public Boolean getPackageDefaultTestOnly() { + return pkg.getDefaultTestOnly(); + } + + @Override + public String getPackageDefaultDeprecation() { + return pkg.getDefaultDeprecation(); + } + + @Override + public ImmutableList<String> getPackageDefaultCopts() { + return pkg.getDefaultCopts(); + } + + @Override + public void visitLabels(AcceptsLabelAttribute observer) { + for (Attribute attribute : ruleClass.getAttributes()) { + Type<?> type = attribute.getType(); + // TODO(bazel-team): This is incoherent: we shouldn't have to special-case these types + // for our visitation policy (e.g., why is Type.NODEP_LABEL_LIST excluded but not + // Type.NODEP_LABEL?). But this is the semantics the calling code requires. Audit + // exactly which calling code expects what and clean up this interface. + if (type == Type.OUTPUT || type == Type.OUTPUT_LIST || type == Type.NODEP_LABEL_LIST) { + continue; + } + for (Object value : visitAttribute(attribute.getName(), type)) { + if (value == null) { + // This is particularly possible for computed defaults. + continue; + } + for (Label label : type.getLabels(value)) { + observer.acceptLabelAttribute(label, attribute); + } + } + } + } + + /** + * Implementations should provide policy-appropriate mappings when an attribute is requested in + * the context of a rule visitation. + */ + protected abstract <T> Iterable<T> visitAttribute(String attributeName, Type<T> type); + + /** + * Returns a {@link Type.Selector} for the given attribute if the attribute is configurable + * for this rule, null otherwise. + * + * @return a {@link Type.Selector} if the attribute takes the form + * "attrName = { 'a': value1_of_type_T, 'b': value2_of_type_T }") for this rule, null + * if it takes the form "attrName = value_of_type_T", null if it doesn't exist + * @throws IllegalArgumentException if the attribute is configurable but of the wrong type + */ + @Nullable + protected <T> Type.Selector<T> getSelector(String attributeName, Type<T> type) { + Integer index = ruleClass.getAttributeIndex(attributeName); + if (index == null) { + return null; + } + Object attrValue = attributes.getAttributeValue(index); + if (!(attrValue instanceof Type.Selector<?>)) { + return null; + } + if (((Type.Selector<?>) attrValue).getOriginalType() != type) { + throw new IllegalArgumentException("Attribute " + attributeName + + " is not of type " + type + " in rule " + ruleLabel.getName()); + } + return (Type.Selector<T>) attrValue; + } + + /** + * Returns the index of the specified attribute, if its type is 'type'. Throws + * an exception otherwise. + */ + private int getIndexWithTypeCheck(String attrName, Type<?> type) { + Integer index = ruleClass.getAttributeIndex(attrName); + if (index == null) { + throw new IllegalArgumentException("No such attribute " + attrName + + " in rule " + ruleLabel.getName()); + } + Attribute attr = ruleClass.getAttribute(index); + if (attr.getType() != type) { + throw new IllegalArgumentException("Attribute " + attrName + + " is not of type " + type + " in rule " + ruleLabel.getName()); + } + return index; + } + + /** + * Helper routine that just checks the given attribute has the given type for this rule and + * throws an IllegalException if not. + */ + protected void checkType(String attrName, Type<?> type) { + getIndexWithTypeCheck(attrName, type); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AggregatingAttributeMapper.java b/src/main/java/com/google/devtools/build/lib/packages/AggregatingAttributeMapper.java new file mode 100644 index 0000000..2aef224 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/AggregatingAttributeMapper.java
@@ -0,0 +1,218 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * {@link AttributeMap} implementation that provides the ability to retrieve *all possible* + * values an attribute might take. + */ +public class AggregatingAttributeMapper extends AbstractAttributeMapper { + + /** + * Store for all of this rule's attributes that are non-configurable. These are + * unconditionally available to computed defaults no matter what dependencies + * they've declared. + */ + private final List<String> nonconfigurableAttributes; + + private AggregatingAttributeMapper(Rule rule) { + super(rule.getPackage(), rule.getRuleClassObject(), rule.getLabel(), + rule.getAttributeContainer()); + + ImmutableList.Builder<String> nonconfigurableAttributesBuilder = ImmutableList.builder(); + for (Attribute attr : rule.getAttributes()) { + if (!attr.isConfigurable()) { + nonconfigurableAttributesBuilder.add(attr.getName()); + } + } + nonconfigurableAttributes = nonconfigurableAttributesBuilder.build(); + } + + public static AggregatingAttributeMapper of(Rule rule) { + return new AggregatingAttributeMapper(rule); + } + + /** + * Override that also visits the rule's configurable attribute keys (which are + * themselves labels). + */ + @Override + public void visitLabels(AcceptsLabelAttribute observer) { + super.visitLabels(observer); + for (String attrName : getAttributeNames()) { + Attribute attribute = getAttributeDefinition(attrName); + Type.Selector<?> selector = getSelector(attrName, attribute.getType()); + if (selector != null) { + for (Label configLabel : selector.getEntries().keySet()) { + if (!Type.Selector.isReservedLabel(configLabel)) { + observer.acceptLabelAttribute(configLabel, attribute); + } + } + } + } + } + + /** + * Returns a list of all possible values an attribute can take for this rule. + */ + @Override + public <T> Iterable<T> visitAttribute(String attributeName, Type<T> type) { + // If this attribute value is configurable, visit all possible values. + Type.Selector<T> selector = getSelector(attributeName, type); + if (selector != null) { + ImmutableList.Builder<T> builder = ImmutableList.builder(); + for (Map.Entry<Label, T> entry : selector.getEntries().entrySet()) { + builder.add(entry.getValue()); + } + return builder.build(); + } + + // If this attribute is a computed default, feed it all possible value combinations of + // its declared dependencies and return all computed results. For example, if this default + // uses attributes x and y, x can configurably be x1 or x2, and y can configurably be y1 + // or y1, then compute default values for the (x1,y1), (x1,y2), (x2,y1), and (x2,y2) cases. + Attribute.ComputedDefault computedDefault = getComputedDefault(attributeName, type); + if (computedDefault != null) { + // This will hold every (value1, value2, ..) combination of the declared dependencies. + List<Map<String, Object>> depMaps = new LinkedList<>(); + // Collect those combinations. + mapDepsForComputedDefault(computedDefault.dependencies(), depMaps, + ImmutableMap.<String, Object>of()); + List<T> possibleValues = new ArrayList<>(); // Not ImmutableList.Builder: values may be null. + // For each combination, call getDefault on a specialized AttributeMap providing those values. + for (Map<String, Object> depMap : depMaps) { + possibleValues.add(type.cast(computedDefault.getDefault(mapBackedAttributeMap(depMap)))); + } + return possibleValues; + } + + // For any other attribute, just return its direct value. + T value = get(attributeName, type); + return value == null ? ImmutableList.<T>of() : ImmutableList.of(value); + } + + /** + * Given (possibly configurable) attributes that a computed default depends on, creates an + * {attrName -> attrValue} map for every possible combination of those attribute values and + * returns a list of all the maps. This defines the complete dependency space that can affect + * the computed default's values. + * + * <p>For example, given dependencies x and y, which might respectively have values x1, x2 and + * y1, y2, this returns: + * <pre> + * [ + * {x: x1, y: y1}, + * {x: x1, y: y2}, + * {x: x2, y: y1}, + * {x: x2, y: y2} + * ] + * </pre> + * + * @param depAttributes the names of the attributes this computed default depends on + * @param mappings the list of {attrName --> attrValue} maps defining the computed default's + * dependency space. This is where this method's results are written. + * @param currentMap a (possibly non-empty) map to add {attrName --> attrValue} + * entries to. Outside callers can just pass in an empty map. + */ + private void mapDepsForComputedDefault(List<String> depAttributes, + List<Map<String, Object>> mappings, Map<String, Object> currentMap) { + // Because this method uses exponential time/space on the number of inputs, keep the + // maximum number of inputs conservatively small. + Preconditions.checkState(depAttributes.size() <= 2); + + if (depAttributes.isEmpty()) { + // Recursive base case: store whatever's already been populated in currentMap. + mappings.add(currentMap); + return; + } + + // Take the first attribute in the dependency list and iterate over all its values. For each + // value x, copy currentMap with the additional entry { firstAttrName: x }, then feed + // this recursively into a subcall over all remaining dependencies. This recursively + // continues until we run out of values. + String firstAttribute = depAttributes.get(0); + for (Object value : visitAttribute(firstAttribute, getAttributeType(firstAttribute))) { + Map<String, Object> newMap = new HashMap<>(); + newMap.putAll(currentMap); + newMap.put(firstAttribute, value); + mapDepsForComputedDefault(depAttributes.subList(1, depAttributes.size()), mappings, newMap); + } + } + + /** + * A custom {@link AttributeMap} that reads attribute values from the given Map. All + * non-configurable attributes are also readable. Any attempt to read an attribute + * that's not in one of these two cases triggers an IllegalArgumentException. + */ + private AttributeMap mapBackedAttributeMap(final Map<String, Object> directMap) { + final AggregatingAttributeMapper owner = AggregatingAttributeMapper.this; + return new AttributeMap() { + + @Override + public <T> T get(String attributeName, Type<T> type) { + owner.checkType(attributeName, type); + if (nonconfigurableAttributes.contains(attributeName)) { + return owner.get(attributeName, type); + } + if (!directMap.containsKey(attributeName)) { + throw new IllegalArgumentException("attribute \"" + attributeName + + "\" isn't available in this computed default context"); + } + return type.cast(directMap.get(attributeName)); + } + + @Override public String getName() { return owner.getName(); } + @Override public Label getLabel() { return owner.getLabel(); } + @Override public Iterable<String> getAttributeNames() { + return ImmutableList.<String>builder() + .addAll(directMap.keySet()).addAll(nonconfigurableAttributes).build(); + } + @Override + public void visitLabels(AcceptsLabelAttribute observer) { owner.visitLabels(observer); } + @Override + public String getPackageDefaultHdrsCheck() { return owner.getPackageDefaultHdrsCheck(); } + @Override + public Boolean getPackageDefaultObsolete() { return owner.getPackageDefaultObsolete(); } + @Override + public Boolean getPackageDefaultTestOnly() { return owner.getPackageDefaultTestOnly(); } + @Override + public String getPackageDefaultDeprecation() { return owner.getPackageDefaultDeprecation(); } + @Override + public ImmutableList<String> getPackageDefaultCopts() { + return owner.getPackageDefaultCopts(); + } + @Nullable @Override + public Type<?> getAttributeType(String attrName) { return owner.getAttributeType(attrName); } + @Nullable @Override public Attribute getAttributeDefinition(String attrName) { + return owner.getAttributeDefinition(attrName); + } + @Override public boolean isAttributeValueExplicitlySpecified(String attributeName) { + return owner.isAttributeValueExplicitlySpecified(attributeName); + } + }; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AnalysisIssues.java b/src/main/java/com/google/devtools/build/lib/packages/AnalysisIssues.java new file mode 100644 index 0000000..22c02b5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/AnalysisIssues.java
@@ -0,0 +1,109 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import javax.annotation.Nullable; + +/** + * Checked exception for analysis-time errors, which can store the errors for later reporting. + * + * <p>It's more robust for a method to throw this exception than expecting a + * {@link RuleErrorConsumer} object (which may be null). + */ +public final class AnalysisIssues extends Exception { + + /** + * An error entry. + * + * <p>{@link AnalysisIssues} can accumulate multiple of these, and report all of them at once. + */ + public static final class Entry { + private final String attribute; + private final String messageTemplate; + private final Object[] arguments; + + private Entry(@Nullable String attribute, String messageTemplate, Object... arguments) { + this.attribute = attribute; + this.messageTemplate = messageTemplate; + this.arguments = arguments; + } + + private void reportTo(RuleErrorConsumer errors) { + String msg = String.format(messageTemplate, arguments); + if (attribute == null) { + errors.ruleError(msg); + } else { + errors.attributeError(attribute, msg); + } + } + + @Override + public String toString() { + if (attribute == null) { + return String.format("ERROR: " + messageTemplate, arguments); + } else { + List<Object> args = new ArrayList<>(); + args.add(attribute); + args.addAll(Arrays.asList(arguments)); + return String.format("ERROR in '%s': " + messageTemplate, args.toArray()); + } + } + } + + private final ImmutableList<Entry> entries; + + public AnalysisIssues(Entry entry) { + this.entries = ImmutableList.of(Preconditions.checkNotNull(entry)); + } + + public AnalysisIssues(Collection<Entry> entries) { + this.entries = ImmutableList.copyOf(Preconditions.checkNotNull(entries)); + } + + /** + * Creates a attribute error entry that will be added to a {@link AnalysisIssues} later. + */ + public static Entry attributeError(String attribute, String messageTemplate, + Object... arguments) { + return new Entry(attribute, messageTemplate, arguments); + } + + public static Entry ruleError(String messageTemplate, Object... arguments) { + return new Entry(null, messageTemplate, arguments); + } + + /** + * Report all accumulated errors and warnings to the given consumer object. + */ + public void reportTo(RuleErrorConsumer errors) { + Preconditions.checkNotNull(errors); + for (Entry e : entries) { + e.reportTo(errors); + } + } + + @Override + public String toString() { + return "Errors during analysis:\n" + Joiner.on("\n").join(entries); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AspectDefinition.java b/src/main/java/com/google/devtools/build/lib/packages/AspectDefinition.java new file mode 100644 index 0000000..e1b5f05 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/AspectDefinition.java
@@ -0,0 +1,167 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +/** + * The definition of an aspect (see {@link com.google.devtools.build.lib.analysis.Aspect} for more + * information.) + * + * <p>Contains enough information to build up the configured target graph except for the actual way + * to build the Skyframe node (that is the territory of + * {@link com.google.devtools.build.lib.view AspectFactory}). In particular: + * <ul> + * <li>The condition that must be fulfilled for an aspect to be able to operate on a configured + * target + * <li>The (implicit or late-bound) attributes of the aspect that denote dependencies the aspect + * itself needs (e.g. runtime libraries for a new language for protocol buffers) + * <li>The aspects this aspect requires from its direct dependencies + * </ul> + * + * <p>The way to build the Skyframe node is not here because this data needs to be accessible from + * the {@code .packages} package and that one requires references to the {@code .view} package. + */ +@Immutable +public final class AspectDefinition { + + private final String name; + private final ImmutableSet<Class<?>> requiredProviders; + private final ImmutableMap<String, Attribute> attributes; + private final ImmutableMultimap<String, Class<? extends AspectFactory<?, ?, ?>>> attributeAspects; + + private AspectDefinition( + String name, + ImmutableSet<Class<?>> requiredProviders, + ImmutableMap<String, Attribute> attributes, + ImmutableMultimap<String, Class<? extends AspectFactory<?, ?, ?>>> attributeAspects) { + this.name = name; + this.requiredProviders = requiredProviders; + this.attributes = attributes; + this.attributeAspects = attributeAspects; + } + + public String getName() { + return name; + } + + /** + * Returns the attributes of the aspect in the form of a String -> {@link Attribute} map. + * + * <p>All attributes are either implicit or late-bound. + */ + public ImmutableMap<String, Attribute> getAttributes() { + return attributes; + } + + /** + * Returns the set of {@link com.google.devtools.build.lib.analysis.TransitiveInfoProvider} instances + * that must be present on a configured target so that this aspect can be applied to it. + * + * <p>We cannot refer to that class here due to our dependency structure, so this returns a set + * of unconstrained class objects. + * + * <p>If a configured target does not have a required provider, the aspect is silently not created + * for it. + */ + public ImmutableSet<Class<?>> getRequiredProviders() { + return requiredProviders; + } + + /** + * Returns the attribute -> set of required aspects map. + * + * <p>Note that the map actually contains {@link AspectFactory} + * instances, except that we cannot reference that class here. + */ + public ImmutableMultimap<String, Class<? extends AspectFactory<?, ?, ?>>> getAttributeAspects() { + return attributeAspects; + } + + /** + * Builder class for {@link AspectDefinition}. + */ + public static final class Builder { + private final String name; + private final Map<String, Attribute> attributes = new LinkedHashMap<>(); + private final Set<Class<?>> requiredProviders = new LinkedHashSet<>(); + private final Multimap<String, Class<? extends AspectFactory<?, ?, ?>>> attributeAspects = + LinkedHashMultimap.create(); + + public Builder(String name) { + this.name = name; + } + + /** + * Asserts that this aspect can only be evaluated for rules that supply the specified provider. + */ + public Builder requireProvider(Class<?> requiredProvider) { + this.requiredProviders.add(requiredProvider); + return this; + } + + /** + * Tells that in order for this aspect to work, the given aspect must be computed for the + * direct dependencies in the attribute with the specified name on the associated configured + * target. + * + * <p>Note that {@code AspectFactory} instances are expected in the second argument, but we + * cannot reference that interface here. + */ + public Builder attributeAspect( + String attribute, Class<? extends AspectFactory<?, ?, ?>> aspectFactory) { + this.attributeAspects.put( + Preconditions.checkNotNull(attribute), Preconditions.checkNotNull(aspectFactory)); + return this; + } + + /** + * Adds an attribute to the aspect. + * + * <p>Since aspects do not appear in BUILD files, the attribute must be either implicit + * (not available in the BUILD file, starting with '$') or late-bound (determined after the + * configuration is available, starting with ':') + */ + public <TYPE> Builder add(Attribute.Builder<TYPE> attr) { + Attribute attribute = attr.build(); + Preconditions.checkState(attribute.isImplicit() || attribute.isLateBound()); + Preconditions.checkState(!attributes.containsKey(attribute.getName()), + "An attribute with the name '%s' already exists.", attribute.getName()); + attributes.put(attribute.getName(), attribute); + return this; + } + + /** + * Builds the aspect definition. + * + * <p>The builder object is reusable afterwards. + */ + public AspectDefinition build() { + return new AspectDefinition(name, ImmutableSet.copyOf(requiredProviders), + ImmutableMap.copyOf(attributes), ImmutableMultimap.copyOf(attributeAspects)); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AspectFactory.java b/src/main/java/com/google/devtools/build/lib/packages/AspectFactory.java new file mode 100644 index 0000000..2283f1b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/AspectFactory.java
@@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +/** + * Creates the Skyframe node of an aspect. + * + * <p>Also has a reference to the definition of the aspect. + */ +public interface AspectFactory<TConfiguredTarget, TRuleContext, TAspect> { + /** + * Creates the aspect based on the configured target of the associated rule. + * + * @param base the configured target of the associated rule + * @param context the context of the associated configured target plus all the attributes the + * aspect itself has defined + */ + TAspect create(TConfiguredTarget base, TRuleContext context); + + /** + * Returns the definition of the aspect. + */ + AspectDefinition getDefinition(); + + /** + * Dummy wrapper class for utility methods because interfaces cannot even have static ones. + */ + public static final class Util { + private Util() { + // Should never be instantiated + } + + public static AspectFactory create(Class<? extends AspectFactory<?, ?, ?>> clazz) { + // TODO(bazel-team): This should be cached somehow, because this method is invoked quite often + try { + return clazz.newInstance(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/Attribute.java b/src/main/java/com/google/devtools/build/lib/packages/Attribute.java new file mode 100644 index 0000000..9a8ae61 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/Attribute.java
@@ -0,0 +1,1343 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.packages.Type.ConversionException; +import com.google.devtools.build.lib.syntax.ClassObject; +import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkCallbackFunction; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.util.StringUtil; + +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.concurrent.Immutable; + +/** + * Metadata of a rule attribute. Contains the attribute name and type, and an + * default value to be used if none is provided in a rule declaration in a BUILD + * file. Attributes are immutable, and may be shared by more than one rule (for + * example, <code>foo_binary</code> and <code>foo_library</code> may share many + * attributes in common). + */ +@Immutable +public final class Attribute implements Comparable<Attribute> { + + public static final Predicate<RuleClass> ANY_RULE = Predicates.alwaysTrue(); + + public static final Predicate<RuleClass> NO_RULE = Predicates.alwaysFalse(); + + /** + * A configuration transition. + */ + public interface Transition { + /** + * Usually, a non-existent entry in the configuration transition table indicates an error. + * Unfortunately, that means that we need to always build the full table. This method allows a + * transition to indicate that a non-existent entry indicates a self transition, i.e., that the + * resulting configuration is the same as the current configuration. This can simplify the code + * needed to set up the transition table. + */ + boolean defaultsToSelf(); + } + + /** + * A configuration split transition; this should be used to transition to multiple configurations + * simultaneously. Note that the corresponding rule implementations must have special support to + * handle this. + */ + // TODO(bazel-team): Serializability constraints? + public interface SplitTransition<T> extends Transition { + /** + * Return the list of {@code BuildOptions} after splitting; empty if not applicable. + */ + List<T> split(T buildOptions); + } + + /** + * Declaration how the configuration should change when following a label or + * label list attribute. + */ + public enum ConfigurationTransition implements Transition { + /** No transition, i.e., the same configuration as the current. */ + NONE, + + /** Transition to the host configuration. */ + HOST, + + /** Transition from the target configuration to the data configuration. */ + // TODO(bazel-team): Move this elsewhere. + DATA; + + @Override + public boolean defaultsToSelf() { + return false; + } + } + + private enum PropertyFlag { + MANDATORY, + EXECUTABLE, + UNDOCUMENTED, + TAGGABLE, + + /** + * Whether the list attribute is order-independent and can be sorted. + */ + ORDER_INDEPENDENT, + + /** + * Whether the allowedRuleClassesForLabels or allowedFileTypesForLabels are + * set to custom values. If so, and the attribute is called "deps", the + * legacy deps checking is skipped, and the new stricter checks are used + * instead. For non-"deps" attributes, this allows skipping the check if it + * would pass anyway, as the default setting allows any rule classes and + * file types. + */ + STRICT_LABEL_CHECKING, + + /** + * Set for things that would cause the a compile or lint-like action to + * be executed when the input changes. Used by compile_one_dependency. + * Set for attributes like hdrs and srcs on cc_ rules or srcs on java_ + * or py_rules. Generally not set on data/resource attributes. + */ + DIRECT_COMPILE_TIME_INPUT, + + /** + * Whether the value of the list type attribute must not be an empty list. + */ + NON_EMPTY, + + /** + * Verifies that the referenced rule produces a single artifact. Note that this check happens + * on a per label basis, i.e. the check happens separately for every label in a label list. + */ + SINGLE_ARTIFACT, + + /** + * Whether we perform silent ruleclass filtering of the dependencies of the label type + * attribute according to their rule classes. I.e. elements of the list which don't match the + * allowedRuleClasses predicate or not rules will be filtered out without throwing any errors. + * This flag is introduced to handle plugins, do not use it in other cases. + */ + SILENT_RULECLASS_FILTER, + + // TODO(bazel-team): This is a hack introduced because of the bad design of the original rules. + // Depot cleanup would be too expensive, but don't migrate this to Skylark. + /** + * Whether to perform analysis time filetype check on this label-type attribute or not. + * If the flag is set, we skip the check that applies the allowedFileTypes filter + * to generated files. Do not use this if avoidable. + */ + SKIP_ANALYSIS_TIME_FILETYPE_CHECK, + + /** + * Whether the value of the attribute should come from a given set of values. + */ + CHECK_ALLOWED_VALUES, + + /** + * Whether this attribute is opted out of "configurability", i.e. the ability to determine + * its value based on properties of the build configuration. + */ + NONCONFIGURABLE, + } + + // TODO(bazel-team): modify this interface to extend Predicate and have an extra error + // message function like AllowedValues does + /** + * A predicate-like class that determines whether an edge between two rules is valid or not. + */ + public interface ValidityPredicate { + /** + * This method should return null if the edge is valid, or a suitable error message + * if it is not. Note that warnings are not supported. + */ + String checkValid(Rule from, Rule to); + } + + public static final ValidityPredicate ANY_EDGE = + new ValidityPredicate() { + @Override + public String checkValid(Rule from, Rule to) { + return null; + } + }; + + /** + * Using this callback function, rules can set the configuration of their dependencies during the + * analysis phase. + */ + public interface Configurator<TConfig, TRule> { + TConfig apply(TRule fromRule, TConfig fromConfiguration, Attribute attribute, Target toTarget); + } + + /** + * A predicate class to check if the value of the attribute comes from a predefined set. + */ + public static class AllowedValueSet implements PredicateWithMessage<Object> { + + private final Set<Object> allowedValues; + + public AllowedValueSet(Iterable<?> values) { + Preconditions.checkNotNull(values); + Preconditions.checkArgument(!Iterables.isEmpty(values)); + allowedValues = ImmutableSet.copyOf(values); + } + + @Override + public boolean apply(Object input) { + return allowedValues.contains(input); + } + + @Override + public String getErrorReason(Object value) { + return String.format("has to be one of %s instead of '%s'", + StringUtil.joinEnglishList(allowedValues, "or", "'"), value); + } + + @VisibleForTesting + public Collection<Object> getAllowedValues() { + return allowedValues; + } + } + + /** + * Creates a new attribute builder. + * + * @param name attribute name + * @param type attribute type + * @return attribute builder + * + * @param <TYPE> attribute type class + */ + public static <TYPE> Attribute.Builder<TYPE> attr(String name, Type<TYPE> type) { + return new Builder<>(name, type); + } + + /** + * A fluent builder for the {@code Attribute} instances. + * + * <p>All methods could be called only once per builder. The attribute + * already undocumented based on its name cannot be marked as undocumented. + */ + public static class Builder <TYPE> { + private String name; + private final Type<TYPE> type; + private Transition configTransition = ConfigurationTransition.NONE; + private Predicate<RuleClass> allowedRuleClassesForLabels = Predicates.alwaysTrue(); + private Predicate<RuleClass> allowedRuleClassesForLabelsWarning = Predicates.alwaysFalse(); + private Configurator<?, ?> configurator = null; + private boolean allowedFileTypesForLabelsSet; + private FileTypeSet allowedFileTypesForLabels = FileTypeSet.ANY_FILE; + private ValidityPredicate validityPredicate = ANY_EDGE; + private Object value; + private boolean valueSet; + private Predicate<AttributeMap> condition; + private Set<PropertyFlag> propertyFlags = EnumSet.noneOf(PropertyFlag.class); + private PredicateWithMessage<Object> allowedValues = null; + private ImmutableSet<String> mandatoryProviders = ImmutableSet.<String>of(); + private Set<Class<? extends AspectFactory<?, ?, ?>>> aspects = new LinkedHashSet<>(); + + /** + * Creates an attribute builder with given name and type. This attribute is optional, uses + * target configuration and has a default value the same as its type default value. This + * attribute will be marked as undocumented if its name starts with the dollar sign ({@code $}) + * or colon ({@code :}). + * + * @param name attribute name + * @param type attribute type + */ + public Builder(String name, Type<TYPE> type) { + this.name = Preconditions.checkNotNull(name); + this.type = Preconditions.checkNotNull(type); + if (isImplicit(name) || isLateBound(name)) { + setPropertyFlag(PropertyFlag.UNDOCUMENTED, "undocumented"); + } + } + + private Builder<TYPE> setPropertyFlag(PropertyFlag flag, String propertyName) { + Preconditions.checkState(!propertyFlags.contains(flag), + propertyName + " flag is already set"); + propertyFlags.add(flag); + return this; + } + + /** + * Sets the property flag of the corresponding name if exists, otherwise throws an Exception. + * Only meant to use from Skylark, do not use from Java. + */ + public Builder<TYPE> setPropertyFlag(String propertyName) { + PropertyFlag flag = null; + try { + flag = PropertyFlag.valueOf(propertyName); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("unknown attribute flag " + propertyName); + } + setPropertyFlag(flag, propertyName); + return this; + } + + /** + * Makes the built attribute mandatory. + */ + public Builder<TYPE> mandatory() { + return setPropertyFlag(PropertyFlag.MANDATORY, "mandatory"); + } + + /** + * Makes the built attribute non empty, meaning the attribute cannot have an empty list value. + * Only applicable for list type attributes. + */ + public Builder<TYPE> nonEmpty() { + Preconditions.checkNotNull(type.getListElementType(), + "attribute '" + name + "' must be a list"); + return setPropertyFlag(PropertyFlag.NON_EMPTY, "non_empty"); + } + + /** + * Makes the built attribute producing a single artifact. + */ + public Builder<TYPE> singleArtifact() { + Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST), + "attribute '" + name + "' must be a label-valued type"); + return setPropertyFlag(PropertyFlag.SINGLE_ARTIFACT, "single_artifact"); + } + + /** + * Forces silent ruleclass filtering on the label type attribute. + * This flag is introduced to handle plugins, do not use it in other cases. + */ + public Builder<TYPE> silentRuleClassFilter() { + Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST), + "must be a label-valued type"); + return setPropertyFlag(PropertyFlag.SILENT_RULECLASS_FILTER, "silent_ruleclass_filter"); + } + + /** + * Skip analysis time filetype check. Don't use it if avoidable. + */ + public Builder<TYPE> skipAnalysisTimeFileTypeCheck() { + Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST), + "must be a label-valued type"); + return setPropertyFlag(PropertyFlag.SKIP_ANALYSIS_TIME_FILETYPE_CHECK, + "skip_analysis_time_filetype_check"); + } + + /** + * Mark the built attribute as order-independent. + */ + public Builder<TYPE> orderIndependent() { + Preconditions.checkNotNull(type.getListElementType(), + "attribute '" + name + "' must be a list"); + return setPropertyFlag(PropertyFlag.ORDER_INDEPENDENT, "order-independent"); + } + + /** + * Defines the configuration transition for this attribute. Defaults to + * {@code NONE}. + */ + public Builder<TYPE> cfg(Transition configTransition) { + Preconditions.checkState(this.configTransition == ConfigurationTransition.NONE, + "the configuration transition is already set"); + this.configTransition = configTransition; + return this; + } + + public Builder<TYPE> cfg(Configurator<?, ?> configurator) { + this.configurator = configurator; + return this; + } + + /** + * Requires the attribute target to be executable; only for label or label + * list attributes. Defaults to {@code false}. + */ + public Builder<TYPE> exec() { + return setPropertyFlag(PropertyFlag.EXECUTABLE, "executable"); + } + + /** + * Indicates that the attribute (like srcs or hdrs) should be used as an input when calculating + * compile_one_dependency. + */ + public Builder<TYPE> direct_compile_time_input() { + return setPropertyFlag(PropertyFlag.DIRECT_COMPILE_TIME_INPUT, + "direct_compile_time_input"); + } + + /** + * Makes the built attribute undocumented. + * + * @param reason explanation why the attribute is undocumented. This is not + * used but required for documentation + */ + public Builder<TYPE> undocumented(String reason) { + return setPropertyFlag(PropertyFlag.UNDOCUMENTED, "undocumented"); + } + + /** + * Sets the attribute default value. The type of the default value must + * match the type parameter. (e.g. list=[], integer=0, string="", + * label=null). The {@code defaultValue} must be immutable. + * + * <p>If defaultValue is of type Label and is a target, that target will + * become an implicit dependency of the Rule; we will load the target + * (and its dependencies) if it encounters the Rule and build the target + * if needs to apply the Rule. + */ + public Builder<TYPE> value(TYPE defaultValue) { + Preconditions.checkState(!valueSet, "the default value is already set"); + value = defaultValue; + valueSet = true; + return this; + } + + /** + * See value(TYPE) above. This method is only meant for Skylark usage. + */ + public Builder<TYPE> defaultValue(Object defaultValue) throws ConversionException { + Preconditions.checkState(!valueSet, "the default value is already set"); + value = type.convert(defaultValue, "attribute " + name); + valueSet = true; + return this; + } + + /** + * Sets the attribute default value to a computed default value - use + * this when the default value is a function of other attributes of the + * Rule. The type of the computed default value for a mandatory attribute + * must match the type parameter: (e.g. list=[], integer=0, string="", + * label=null). The {@code defaultValue} implementation must be immutable. + * + * <p>If computedDefault returns a Label that is a target, that target will + * become an implicit dependency of this Rule; we will load the target + * (and its dependencies) if it encounters the Rule and build the target if + * needs to apply the Rule. + */ + public Builder<TYPE> value(ComputedDefault defaultValue) { + Preconditions.checkState(!valueSet, "the default value is already set"); + value = defaultValue; + valueSet = true; + return this; + } + + /** + * Sets the attribute default value to be late-bound, i.e., it is derived from the build + * configuration. + */ + public Builder<TYPE> value(LateBoundDefault<?> defaultValue) { + Preconditions.checkState(!valueSet, "the default value is already set"); + Preconditions.checkState(name.isEmpty() || isLateBound(name)); + value = defaultValue; + valueSet = true; + return this; + } + + /** + * Returns true if a late-bound value has been set. Useful only for Skylark. + */ + public boolean hasLateBoundValue() { + return value != null && value instanceof LateBoundDefault; + } + + /** + * Sets a condition predicate. The default value of the attribute only applies if the condition + * evaluates to true. If the value is explicitly provided, then this condition is ignored. + * + * <p>The condition is only evaluated if the attribute is not explicitly set, and after all + * explicit attributes have been set. It can generally not access default values of other + * attributes. + */ + public Builder<TYPE> condition(Predicate<AttributeMap> condition) { + Preconditions.checkState(this.condition == null, "the condition is already set"); + this.condition = condition; + return this; + } + + /** + * Switches on the capability of an attribute to be published to the rule's + * tag set. + */ + public Builder<TYPE> taggable() { + return setPropertyFlag(PropertyFlag.TAGGABLE, "taggable"); + } + + /** + * If this is a label or label-list attribute, then this sets the allowed + * rule types for the labels occurring in the attribute. If the attribute + * contains Labels of any other rule type, then an error is produced during + * the analysis phase. Defaults to allow any types. + * + * <p>This only works on a per-target basis, not on a per-file basis; with + * other words, it works for 'deps' attributes, but not 'srcs' attributes. + */ + public Builder<TYPE> allowedRuleClasses(Iterable<String> allowedRuleClasses) { + return allowedRuleClasses( + new RuleClass.Builder.RuleClassNamePredicate(allowedRuleClasses)); + } + + /** + * If this is a label or label-list attribute, then this sets the allowed + * rule types for the labels occurring in the attribute. If the attribute + * contains Labels of any other rule type, then an error is produced during + * the analysis phase. Defaults to allow any types. + * + * <p>This only works on a per-target basis, not on a per-file basis; with + * other words, it works for 'deps' attributes, but not 'srcs' attributes. + */ + public Builder<TYPE> allowedRuleClasses(Predicate<RuleClass> allowedRuleClasses) { + Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST), + "must be a label-valued type"); + propertyFlags.add(PropertyFlag.STRICT_LABEL_CHECKING); + allowedRuleClassesForLabels = allowedRuleClasses; + return this; + } + + /** + * If this is a label or label-list attribute, then this sets the allowed + * rule types for the labels occurring in the attribute. If the attribute + * contains Labels of any other rule type, then an error is produced during + * the analysis phase. Defaults to allow any types. + * + * <p>This only works on a per-target basis, not on a per-file basis; with + * other words, it works for 'deps' attributes, but not 'srcs' attributes. + */ + public Builder<TYPE> allowedRuleClasses(String... allowedRuleClasses) { + return allowedRuleClasses(ImmutableSet.copyOf(allowedRuleClasses)); + } + + /** + * If this is a label or label-list attribute, then this sets the allowed + * file types for file labels occurring in the attribute. If the attribute + * contains labels that correspond to files of any other type, then an error + * is produced during the analysis phase. + * + * <p>This only works on a per-target basis, not on a per-file basis; with + * other words, it works for 'deps' attributes, but not 'srcs' attributes. + */ + public Builder<TYPE> allowedFileTypes(FileTypeSet allowedFileTypes) { + Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST), + "must be a label-valued type"); + propertyFlags.add(PropertyFlag.STRICT_LABEL_CHECKING); + allowedFileTypesForLabelsSet = true; + allowedFileTypesForLabels = allowedFileTypes; + return this; + } + + /** + * Allow all files for legacy compatibility. All uses of this method should be audited and then + * removed. In some cases, it's correct to allow any file, but mostly the set of files should be + * restricted to a reasonable set. + */ + public Builder<TYPE> legacyAllowAnyFileType() { + return allowedFileTypes(FileTypeSet.ANY_FILE); + } + + /** + * If this is a label or label-list attribute, then this sets the allowed + * file types for file labels occurring in the attribute. If the attribute + * contains labels that correspond to files of any other type, then an error + * is produced during the analysis phase. + * + * <p>This only works on a per-target basis, not on a per-file basis; with + * other words, it works for 'deps' attributes, but not 'srcs' attributes. + */ + public Builder<TYPE> allowedFileTypes(FileType... allowedFileTypes) { + return allowedFileTypes(FileTypeSet.of(allowedFileTypes)); + } + + /** + * If this is a label or label-list attribute, then this sets the allowed + * rule types with warning for the labels occurring in the attribute. If the attribute + * contains Labels of any other rule type (other than this or those set in + * allowedRuleClasses()), then a warning is produced during + * the analysis phase. Defaults to deny any types. + * + * <p>This only works on a per-target basis, not on a per-file basis; with + * other words, it works for 'deps' attributes, but not 'srcs' attributes. + */ + public Builder<TYPE> allowedRuleClassesWithWarning(Collection<String> allowedRuleClasses) { + return allowedRuleClassesWithWarning( + new RuleClass.Builder.RuleClassNamePredicate(allowedRuleClasses)); + } + + /** + * If this is a label or label-list attribute, then this sets the allowed + * rule types for the labels occurring in the attribute. If the attribute + * contains Labels of any other rule type (other than this or those set in + * allowedRuleClasses()), then a warning is produced during + * the analysis phase. Defaults to deny any types. + * + * <p>This only works on a per-target basis, not on a per-file basis; with + * other words, it works for 'deps' attributes, but not 'srcs' attributes. + */ + public Builder<TYPE> allowedRuleClassesWithWarning(Predicate<RuleClass> allowedRuleClasses) { + Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST), + "must be a label-valued type"); + propertyFlags.add(PropertyFlag.STRICT_LABEL_CHECKING); + allowedRuleClassesForLabelsWarning = allowedRuleClasses; + return this; + } + + /** + * If this is a label or label-list attribute, then this sets the allowed + * rule types for the labels occurring in the attribute. If the attribute + * contains Labels of any other rule type (other than this or those set in + * allowedRuleClasses()), then a warning is produced during + * the analysis phase. Defaults to deny any types. + * + * <p>This only works on a per-target basis, not on a per-file basis; with + * other words, it works for 'deps' attributes, but not 'srcs' attributes. + */ + public Builder<TYPE> allowedRuleClassesWithWarning(String... allowedRuleClasses) { + return allowedRuleClassesWithWarning(ImmutableSet.copyOf(allowedRuleClasses)); + } + + /** + * Sets a set of mandatory Skylark providers. Every configured target occurring in + * this label type attribute has to provide all of these providers, otherwise an + * error is produces during the analysis phase for every missing provider. + */ + public Builder<TYPE> mandatoryProviders(Iterable<String> providers) { + Preconditions.checkState((type == Type.LABEL) || (type == Type.LABEL_LIST), + "must be a label-valued type"); + this.mandatoryProviders = ImmutableSet.copyOf(providers); + return this; + } + + /** + * Asserts that a particular aspect needs to be computed for all direct dependencies through + * this attribute. + */ + public Builder<TYPE> aspect(Class<? extends AspectFactory<?, ?, ?>> aspect) { + this.aspects.add(aspect); + return this; + } + /** + * Sets the predicate-like edge validity checker. + */ + public Builder<TYPE> validityPredicate(ValidityPredicate validityPredicate) { + propertyFlags.add(PropertyFlag.STRICT_LABEL_CHECKING); + this.validityPredicate = validityPredicate; + return this; + } + + /** + * The value of the attribute must be one of allowedValues. + */ + public Builder<TYPE> allowedValues(PredicateWithMessage<Object> allowedValues) { + this.allowedValues = allowedValues; + propertyFlags.add(PropertyFlag.CHECK_ALLOWED_VALUES); + return this; + } + + /** + * Makes the built attribute "non-configurable", i.e. its value cannot be influenced by + * the build configuration. Attributes are "configurable" unless explicitly opted out here. + * + * <p>Non-configurability indicates an exceptional state: there exists Blaze logic that needs + * the attribute's value, has no access to configurations, and can't apply a workaround + * through an appropriate {@link AbstractAttributeMapper} implementation. Scenarios like + * this should be as uncommon as possible, so it's important we maintain clear documentation + * on what causes them and why users consequently can't configure certain attributes. + * + * @param reason why this attribute can't be configurable. This isn't used by Blaze - it's + * solely a documentation mechanism. + */ + public Builder<TYPE> nonconfigurable(String reason) { + Preconditions.checkState(!reason.isEmpty()); + return setPropertyFlag(PropertyFlag.NONCONFIGURABLE, "nonconfigurable"); + } + + /** + * Creates the attribute. Uses name, type, optionality, configuration type + * and the default value configured by the builder. + */ + public Attribute build() { + return build(this.name); + } + + /** + * Creates the attribute. Uses type, optionality, configuration type + * and the default value configured by the builder. Use the name + * passed as an argument. This function is used by Skylark where the + * name is provided only when we build. We don't want to modify the + * builder, as it is shared in a multithreaded environment. + */ + public Attribute build(String name) { + Preconditions.checkState(!name.isEmpty(), "name has not been set"); + // TODO(bazel-team): Remove this check again, and remove all allowedFileTypes() calls. + if ((type == Type.LABEL) || (type == Type.LABEL_LIST)) { + if ((name.startsWith("$") || name.startsWith(":")) && !allowedFileTypesForLabelsSet) { + allowedFileTypesForLabelsSet = true; + allowedFileTypesForLabels = FileTypeSet.ANY_FILE; + } + if (!allowedFileTypesForLabelsSet) { + throw new IllegalStateException(name); + } + } + return new Attribute(name, type, Sets.immutableEnumSet(propertyFlags), + valueSet ? value : type.getDefaultValue(), configTransition, configurator, + allowedRuleClassesForLabels, allowedRuleClassesForLabelsWarning, + allowedFileTypesForLabels, allowedFileTypesForLabelsSet, validityPredicate, condition, + allowedValues, mandatoryProviders, ImmutableSet.copyOf(aspects)); + } + } + + /** + * A computed default is a default value for a Rule attribute that is a + * function of other attributes of the rule. + * + * <p>Attributes whose defaults are computed are first initialized to the default + * for their type, and then the computed defaults are evaluated after all + * non-computed defaults have been initialized. There is no defined order + * among computed defaults, so they must not depend on each other. + * + * <p>If a computed default reads the value of another attribute, at least one of + * the following must be true: + * + * <ol> + * <li>The other attribute must be declared in the computed default's constructor</li> + * <li>The other attribute must be non-configurable ({@link Builder#nonconfigurable()}</li> + * </ol> + * + * <p>The reason for enforced declarations is that, since attribute values might be + * configurable, a computed default that depends on them may itself take multiple + * values. Since we have no access to a target's configuration at the time these values + * are computed, we need the ability to probe the default's *complete* dependency space. + * Declared dependencies allow us to do so sanely. Non-configurable attributes don't have + * this problem because their value is fixed and known even without configuration information. + * + * <p>Implementations of this interface must be immutable. + */ + public abstract static class ComputedDefault { + private final List<String> dependencies; + List<String> dependencies() { return dependencies; } + + /** + * Create a computed default that can read all non-configurable attribute values and no + * configurable attribute values. + */ + public ComputedDefault() { + dependencies = ImmutableList.of(); + } + + /** + * Create a computed default that can read all non-configurable attributes values and one + * explicitly specified configurable attribute value + */ + public ComputedDefault(String depAttribute) { + dependencies = ImmutableList.of(depAttribute); + } + + /** + * Create a computed default that can read all non-configurable attributes values and two + * explicitly specified configurable attribute values. + */ + public ComputedDefault(String depAttribute1, String depAttribute2) { + dependencies = ImmutableList.of(depAttribute1, depAttribute2); + } + + public abstract Object getDefault(AttributeMap rule); + } + + /** + * Marker interface for late-bound values. Unfortunately, we can't refer to BuildConfiguration + * right now, since that is in a separate compilation unit. + * + * <p>Implementations of this interface must be immutable. + * + * <p>Use sparingly - having different values for attributes during loading and analysis can + * confuse users. + */ + public interface LateBoundDefault<T> { + /** + * Whether to look up the label in the host configuration. This is only here for the host JDK - + * we usually need to look up labels in the target configuration. + */ + boolean useHostConfiguration(); + + /** + * Returns the set of required configuration fragments, i.e., fragments that will be accessed by + * the code. + */ + Set<Class<?>> getRequiredConfigurationFragments(); + + /** + * The default value for the attribute that is set during the loading phase. + */ + Object getDefault(); + + /** + * The actual value for the attribute for the analysis phase, which depends on the build + * configuration. Note that configurations transitions are applied after the late-bound + * attribute was evaluated. + */ + Object getDefault(Rule rule, T o) throws EvalException; + } + + /** + * Abstract super class for label-typed {@link LateBoundDefault} implementations that simplifies + * the client code a little and makes it a bit more type-safe. + */ + public abstract static class LateBoundLabel<T> implements LateBoundDefault<T> { + private final Label label; + private final ImmutableSet<Class<?>> requiredConfigurationFragments; + + public LateBoundLabel() { + this((Label) null); + } + + public LateBoundLabel(Label label) { + this.label = label; + this.requiredConfigurationFragments = ImmutableSet.of(); + } + + public LateBoundLabel(Label label, Class<?>... requiredConfigurationFragments) { + this.label = label; + this.requiredConfigurationFragments = ImmutableSet.copyOf(requiredConfigurationFragments); + } + + public LateBoundLabel(String label) { + this(Label.parseAbsoluteUnchecked(label)); + } + + public LateBoundLabel(String label, Class<?>... requiredConfigurationFragments) { + this(Label.parseAbsoluteUnchecked(label), requiredConfigurationFragments); + } + + @Override + public boolean useHostConfiguration() { + return false; + } + + @Override + public ImmutableSet<Class<?>> getRequiredConfigurationFragments() { + return requiredConfigurationFragments; + } + + @Override + public final Label getDefault() { + return label; + } + + @Override + public abstract Label getDefault(Rule rule, T configuration); + } + + /** + * Abstract super class for label-list-typed {@link LateBoundDefault} implementations that + * simplifies the client code a little and makes it a bit more type-safe. + */ + public abstract static class LateBoundLabelList<T> implements LateBoundDefault<T> { + private final ImmutableList<Label> labels; + + public LateBoundLabelList() { + this.labels = ImmutableList.of(); + } + + public LateBoundLabelList(List<Label> labels) { + this.labels = ImmutableList.copyOf(labels); + } + + @Override + public boolean useHostConfiguration() { + return false; + } + + @Override + public ImmutableSet<Class<?>> getRequiredConfigurationFragments() { + return ImmutableSet.of(); + } + + @Override + public final List<Label> getDefault() { + return labels; + } + + @Override + public abstract List<Label> getDefault(Rule rule, T configuration); + } + + /** + * A class for late bound attributes defined in Skylark. + */ + public static final class SkylarkLateBound implements LateBoundDefault<Object> { + + private final SkylarkCallbackFunction callback; + + public SkylarkLateBound(SkylarkCallbackFunction callback) { + this.callback = callback; + } + + @Override + public boolean useHostConfiguration() { + return false; + } + + @Override + public ImmutableSet<Class<?>> getRequiredConfigurationFragments() { + return ImmutableSet.of(); + } + + @Override + public Object getDefault() { + return null; + } + + @Override + public Object getDefault(Rule rule, Object o) throws EvalException { + Map<String, Object> attrValues = new HashMap<>(); + // TODO(bazel-team): support configurable attributes here. RawAttributeMapper will throw + // an exception on any instance of configurable attributes. + AttributeMap attributes = RawAttributeMapper.of(rule); + for (Attribute attr : rule.getAttributes()) { + if (!attr.isLateBound()) { + Object value = attributes.get(attr.getName(), attr.getType()); + if (value != null) { + attrValues.put(attr.getName(), value); + } + } + } + ClassObject attrs = new SkylarkClassObject(attrValues, + "No such regular (non late-bound) attribute '%s'."); + return callback.call(attrs, o); + } + } + + private final String name; + + private final Type<?> type; + + private final Set<PropertyFlag> propertyFlags; + + // Exactly one of these conditions is true: + // 1. defaultValue == null. + // 2. defaultValue instanceof ComputedDefault && + // type.isValid(defaultValue.getDefault()) + // 3. type.isValid(defaultValue). + // 4. defaultValue instanceof LateBoundDefault && + // type.isValid(defaultValue.getDefault(configuration)) + // (We assume a hypothetical Type.isValid(Object) predicate.) + private final Object defaultValue; + + private final Transition configTransition; + + private final Configurator<?, ?> configurator; + + /** + * For label or label-list attributes, this predicate returns which rule + * classes are allowed for the targets in the attribute. + */ + private final Predicate<RuleClass> allowedRuleClassesForLabels; + + /** + * For label or label-list attributes, this predicate returns which rule + * classes are allowed for the targets in the attribute with warning. + */ + private final Predicate<RuleClass> allowedRuleClassesForLabelsWarning; + + /** + * For label or label-list attributes, this predicate returns which file + * types are allowed for targets in the attribute that happen to be file + * targets (rather than rules). + */ + private final FileTypeSet allowedFileTypesForLabels; + private final boolean allowedFileTypesForLabelsSet; + + /** + * This predicate-like object checks + * if the edge between two rules using this attribute is valid + * in the dependency graph. Returns null if valid, otherwise an error message. + */ + private final ValidityPredicate validityPredicate; + + private final Predicate<AttributeMap> condition; + + private final PredicateWithMessage<Object> allowedValues; + + private final ImmutableSet<String> mandatoryProviders; + + private final ImmutableSet<Class<? extends AspectFactory<?, ?, ?>>> aspects; + + /** + * Constructs a rule attribute with the specified name, type and default + * value. + * + * @param name the name of the attribute + * @param type the type of the attribute + * @param defaultValue the default value to use for this attribute if none is + * specified in rule declaration in the BUILD file. Must be null, or of + * type "type". May be an instance of ComputedDefault, in which case + * its getDefault() method must return an instance of "type", or null. + * Must be immutable. + * @param configTransition the configuration transition for this attribute + * (which must be of type LABEL, LABEL_LIST, NODEP_LABEL or + * NODEP_LABEL_LIST). + */ + private Attribute(String name, Type<?> type, Set<PropertyFlag> propertyFlags, + Object defaultValue, Transition configTransition, + Configurator<?, ?> configurator, + Predicate<RuleClass> allowedRuleClassesForLabels, + Predicate<RuleClass> allowedRuleClassesForLabelsWarning, + FileTypeSet allowedFileTypesForLabels, + boolean allowedFileTypesForLabelsSet, + ValidityPredicate validityPredicate, + Predicate<AttributeMap> condition, + PredicateWithMessage<Object> allowedValues, + ImmutableSet<String> mandatoryProviders, + ImmutableSet<Class<? extends AspectFactory<?, ?, ?>>> aspects) { + Preconditions.checkNotNull(configTransition); + Preconditions.checkArgument( + (configTransition == ConfigurationTransition.NONE && configurator == null) + || type == Type.LABEL || type == Type.LABEL_LIST + || type == Type.NODEP_LABEL || type == Type.NODEP_LABEL_LIST, + "Configuration transitions can only be specified for label or label list attributes"); + Preconditions.checkArgument(isLateBound(name) == (defaultValue instanceof LateBoundDefault), + "late bound attributes require a default value that is late bound (and vice versa): " + + name); + if (isLateBound(name)) { + LateBoundDefault<?> lateBoundDefault = (LateBoundDefault<?>) defaultValue; + Preconditions.checkArgument((configurator == null), + "a late bound attribute cannot specify a configurator"); + Preconditions.checkArgument(!lateBoundDefault.useHostConfiguration() + || (configTransition == ConfigurationTransition.HOST), + "a late bound default value using the host configuration must use the host transition"); + } + + this.name = name; + this.type = type; + this.propertyFlags = propertyFlags; + this.defaultValue = defaultValue; + this.configTransition = configTransition; + this.configurator = configurator; + this.allowedRuleClassesForLabels = allowedRuleClassesForLabels; + this.allowedRuleClassesForLabelsWarning = allowedRuleClassesForLabelsWarning; + this.allowedFileTypesForLabels = allowedFileTypesForLabels; + this.allowedFileTypesForLabelsSet = allowedFileTypesForLabelsSet; + this.validityPredicate = validityPredicate; + this.condition = condition; + this.allowedValues = allowedValues; + this.mandatoryProviders = mandatoryProviders; + this.aspects = aspects; + } + + /** + * Returns the name of this attribute. + */ + public String getName() { + return name; + } + + /** + * Returns the logical type of this attribute. (May differ from the actual + * representation as a value in the build interpreter; for example, an + * attribute may logically be a list of labels, but be represented as a list + * of strings.) + */ + public Type<?> getType() { + return type; + } + + private boolean getPropertyFlag(PropertyFlag flag) { + return propertyFlags.contains(flag); + } + + /** + * Returns true if this parameter is mandatory. + */ + public boolean isMandatory() { + return getPropertyFlag(PropertyFlag.MANDATORY); + } + + /** + * Returns true if this list parameter cannot have an empty list as a value. + */ + public boolean isNonEmpty() { + return getPropertyFlag(PropertyFlag.NON_EMPTY); + } + + /** + * Returns true if this label parameter must produce a single artifact. + */ + public boolean isSingleArtifact() { + return getPropertyFlag(PropertyFlag.SINGLE_ARTIFACT); + } + + /** + * Returns true if this label type parameter is checked by silent ruleclass filtering. + */ + public boolean isSilentRuleClassFilter() { + return getPropertyFlag(PropertyFlag.SILENT_RULECLASS_FILTER); + } + + /** + * Returns true if this label type parameter skips the analysis time filetype check. + */ + public boolean isSkipAnalysisTimeFileTypeCheck() { + return getPropertyFlag(PropertyFlag.SKIP_ANALYSIS_TIME_FILETYPE_CHECK); + } + + /** + * Returns true if this parameter is order-independent. + */ + public boolean isOrderIndependent() { + return getPropertyFlag(PropertyFlag.ORDER_INDEPENDENT); + } + + /** + * Returns the configuration transition for this attribute for label or label + * list attributes. For other attributes it will always return {@code NONE}. + */ + public Transition getConfigurationTransition() { + return configTransition; + } + + /** + * Returns the configurator instance for this attribute for label or label list attributes. + * For other attributes it will always return {@code null}. + */ + public Configurator<?, ?> getConfigurator() { + return configurator; + } + + /** + * Returns whether the target is required to be executable for label or label + * list attributes. For other attributes it always returns {@code false}. + */ + public boolean isExecutable() { + return getPropertyFlag(PropertyFlag.EXECUTABLE); + } + + /** + * Returns {@code true} iff the rule is a direct input for an action. + */ + public boolean isDirectCompileTimeInput() { + return getPropertyFlag(PropertyFlag.DIRECT_COMPILE_TIME_INPUT); + } + + /** + * Returns {@code true} iff this attribute requires documentation. + */ + public boolean isDocumented() { + return !getPropertyFlag(PropertyFlag.UNDOCUMENTED); + } + + /** + * Returns {@code true} iff this attribute should be published to the rule's + * tag set. Note that not all Type classes support tag conversion. + */ + public boolean isTaggable() { + return getPropertyFlag(PropertyFlag.TAGGABLE); + } + + public boolean isStrictLabelCheckingEnabled() { + return getPropertyFlag(PropertyFlag.STRICT_LABEL_CHECKING); + } + + /** + * Returns true if the value of this attribute should be a part of a given set. + */ + public boolean checkAllowedValues() { + return getPropertyFlag(PropertyFlag.CHECK_ALLOWED_VALUES); + } + + /** + * Returns true if this attribute's value can be influenced by the build configuration. + */ + public boolean isConfigurable() { + return !(type == Type.OUTPUT // Excluded because of Rule#populateExplicitOutputFiles. + || type == Type.OUTPUT_LIST + || getPropertyFlag(PropertyFlag.NONCONFIGURABLE)); + } + + /** + * Returns a predicate that evaluates to true for rule classes that are + * allowed labels in this attribute. If this is not a label or label-list + * attribute, the returned predicate always evaluates to true. + */ + public Predicate<RuleClass> getAllowedRuleClassesPredicate() { + return allowedRuleClassesForLabels; + } + + /** + * Returns a predicate that evaluates to true for rule classes that are + * allowed labels in this attribute with warning. If this is not a label or label-list + * attribute, the returned predicate always evaluates to true. + */ + public Predicate<RuleClass> getAllowedRuleClassesWarningPredicate() { + return allowedRuleClassesForLabelsWarning; + } + + /** + * Returns the set of mandatory Skylark providers. + */ + public ImmutableSet<String> getMandatoryProviders() { + return mandatoryProviders; + } + + public FileTypeSet getAllowedFileTypesPredicate() { + return allowedFileTypesForLabels; + } + + public ValidityPredicate getValidityPredicate() { + return validityPredicate; + } + + public Predicate<AttributeMap> getCondition() { + return condition == null ? Predicates.<AttributeMap>alwaysTrue() : condition; + } + + public PredicateWithMessage<Object> getAllowedValues() { + return allowedValues; + } + + /** + * Returns the set of aspects required for dependencies through this attribute. + */ + public ImmutableSet<Class<? extends AspectFactory<?, ?, ?>>> getAspects() { + return aspects; + } + + /** + * Returns the default value of this attribute in the context of the + * specified Rule. For attributes with a computed default, i.e. {@code + * hasComputedDefault()}, {@code rule} must be non-null since the result may + * depend on the values of its other attributes. + * + * <p>The result may be null (although this is not a value in the build + * language). + * + * <p>During population of the rule's attribute dictionary, all non-computed + * defaults must be set before all computed ones. + * + * @param rule the rule to which this attribute belongs; non-null if + * {@code hasComputedDefault()}; ignored otherwise. + */ + public Object getDefaultValue(Rule rule) { + if (!getCondition().apply(rule == null ? null : NonconfigurableAttributeMapper.of(rule))) { + return null; + } else if (defaultValue instanceof LateBoundDefault<?>) { + return ((LateBoundDefault<?>) defaultValue).getDefault(); + } else { + return defaultValue; + } + } + + /** + * Returns the default value of this attribute, even if it has a condition, is a computed default, + * or a late-bound default. + */ + @VisibleForTesting + public Object getDefaultValueForTesting() { + return defaultValue; + } + + public LateBoundDefault<?> getLateBoundDefault() { + Preconditions.checkState(isLateBound()); + return (LateBoundDefault<?>) defaultValue; + } + + /** + * Returns true iff this attribute has a computed default or a condition. + * + * @see #getDefaultValue(Rule) + */ + boolean hasComputedDefault() { + return (defaultValue instanceof ComputedDefault) || (condition != null); + } + + /** + * Returns if this attribute is an implicit dependency according to the naming policy that + * designates implicit attributes. + */ + public boolean isImplicit() { + return isImplicit(getName()); + } + + /** + * Returns if an attribute with the given name is an implicit dependency according to the + * naming policy that designates implicit attributes. + */ + public static boolean isImplicit(String name) { + return name.startsWith("$"); + } + + /** + * Returns if this attribute is late-bound according to the naming policy that designates + * late-bound attributes. + */ + public boolean isLateBound() { + return isLateBound(getName()); + } + + /** + * Returns if an attribute with the given name is late-bound according to the naming policy + * that designates late-bound attributes. + */ + public static boolean isLateBound(String name) { + return name.startsWith(":"); + } + + @Override + public String toString() { + return "Attribute(" + name + ", " + type + ")"; + } + + @Override + public int compareTo(Attribute other) { + return name.compareTo(other.name); + } + + /** + * Returns a replica builder of this Attribute. + */ + public Attribute.Builder<?> cloneBuilder() { + Builder<?> builder = new Builder<>(name, this.type); + builder.allowedFileTypesForLabels = allowedFileTypesForLabels; + builder.allowedFileTypesForLabelsSet = allowedFileTypesForLabelsSet; + builder.allowedRuleClassesForLabels = allowedRuleClassesForLabels; + builder.allowedRuleClassesForLabelsWarning = allowedRuleClassesForLabelsWarning; + builder.validityPredicate = validityPredicate; + builder.condition = condition; + builder.configTransition = configTransition; + builder.propertyFlags = propertyFlags.isEmpty() ? + EnumSet.noneOf(PropertyFlag.class) : EnumSet.copyOf(propertyFlags); + builder.value = defaultValue; + builder.valueSet = false; + builder.allowedValues = allowedValues; + + return builder; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java b/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java new file mode 100644 index 0000000..be7584a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java
@@ -0,0 +1,116 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.devtools.build.lib.events.Location; + +import java.util.BitSet; + +/** + * Provides attribute setting and retrieval for a Rule. Encapsulating attribute access + * here means it can be passed around independently of the Rule itself. In particular, + * it can be consumed by independent {@link AttributeMap} instances that can apply + * varying kinds of logic for determining the "value" of an attribute. For example, + * a configurable attribute's "value" may be a { config --> value } dictionary + * or a configuration-bound lookup on that dictionary, depending on the context in + * which it's requested. + * + * <p>This class provides the lowest-level access to attribute information. It is *not* + * intended to be a robust public interface, but rather just an input to {@link AttributeMap} + * instances. Use those instances for all domain-level attribute access. + */ +public class AttributeContainer { + + private final RuleClass ruleClass; + + // Attribute values, keyed by attribute index: + private final Object[] attributeValues; + + // Whether an attribute value has been set explicitly in the BUILD file, keyed by attribute index. + private final BitSet attributeValueExplicitlySpecified; + + // Attribute locations, keyed by attribute index: + private final Location[] attributeLocations; + + /** + * Create a container for a rule of the given rule class. + */ + AttributeContainer(RuleClass ruleClass) { + this.ruleClass = ruleClass; + this.attributeValues = new Object[ruleClass.getAttributeCount()]; + this.attributeValueExplicitlySpecified = new BitSet(ruleClass.getAttributeCount()); + this.attributeLocations = new Location[ruleClass.getAttributeCount()]; + } + + /** + * Returns an attribute value by instance, or null on no match. + */ + public Object getAttr(Attribute attribute) { + return getAttr(attribute.getName()); + } + + /** + * Returns an attribute value by name, or null on no match. + */ + public Object getAttr(String attrName) { + Integer idx = ruleClass.getAttributeIndex(attrName); + return idx != null ? attributeValues[idx] : null; + } + + /** + * Returns true iff the given attribute exists for this rule and its value + * is explicitly set in the BUILD file (as opposed to its default value). + */ + public boolean isAttributeValueExplicitlySpecified(Attribute attribute) { + return isAttributeValueExplicitlySpecified(attribute.getName()); + } + + public boolean isAttributeValueExplicitlySpecified(String attributeName) { + Integer idx = ruleClass.getAttributeIndex(attributeName); + return idx != null ? attributeValueExplicitlySpecified.get(idx) : false; + } + + /** + * Returns the location of the attribute definition for this rule, or null if not found. + */ + public Location getAttributeLocation(String attrName) { + Integer idx = ruleClass.getAttributeIndex(attrName); + return idx != null ? attributeLocations[idx] : null; + } + + Object getAttributeValue(int index) { + return attributeValues[index]; + } + + void setAttributeValue(Attribute attribute, Object value, boolean explicit) { + Integer index = ruleClass.getAttributeIndex(attribute.getName()); + attributeValues[index] = value; + attributeValueExplicitlySpecified.set(index, explicit); + } + + void setAttributeValueByName(String attrName, Object value) { + Integer index = ruleClass.getAttributeIndex(attrName); + attributeValues[index] = value; + attributeValueExplicitlySpecified.set(index); + } + + void setAttributeLocation(int attrIndex, Location location) { + attributeLocations[attrIndex] = location; + } + + void setAttributeLocation(Attribute attribute, Location location) { + Integer index = ruleClass.getAttributeIndex(attribute.getName()); + attributeLocations[index] = location; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AttributeMap.java b/src/main/java/com/google/devtools/build/lib/packages/AttributeMap.java new file mode 100644 index 0000000..eeb6951 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/AttributeMap.java
@@ -0,0 +1,108 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.syntax.Label; + +import javax.annotation.Nullable; + +/** + * The interface for accessing a {@link Rule}'s attributes. + * + * <p>Since what an attribute lookup should return can be ambiguous (e.g. for configurable + * attributes, should we return a configuration-resolved value or the original, unresolved + * selection expression?), different implementations can apply different policies for how to + * fulfill these methods. Calling code can then use the appropriate implementation for whatever + * its particular needs are. + */ +public interface AttributeMap { + /** + * Returns the name of the rule; this is equivalent to {@code getLabel().getName()}. + */ + String getName(); + + /** + * Returns the label of the rule. + */ + Label getLabel(); + + /** + * Returns the value of the named rule attribute, which must be of the given type. If it does not + * exist or has the wrong type, throws an {@link IllegalArgumentException}. + */ + <T> T get(String attributeName, Type<T> type); + + /** + * Returns the names of all attributes covered by this map. + */ + Iterable<String> getAttributeNames(); + + /** + * Returns the type of the given attribute, if it exists. Otherwise returns null. + */ + @Nullable Type<?> getAttributeType(String attrName); + + /** + * Returns the attribute definition whose name is {@code attrName}, or null + * if not found. + */ + @Nullable Attribute getAttributeDefinition(String attrName); + + /** + * Returns true iff the value of the specified attribute is explicitly set in the BUILD file (as + * opposed to its default value). This also returns true if the value from the BUILD file is the + * same as the default value. + * + * <p>It is probably a good idea to avoid this method in default value and implicit outputs + * computation, because it is confusing that setting an attribute to an empty list (for example) + * is different from not setting it at all. + */ + boolean isAttributeValueExplicitlySpecified(String attributeName); + + /** + * An interface which accepts {@link Attribute}s, used by {@link #visitLabels}. + */ + interface AcceptsLabelAttribute { + /** + * Accept a (Label, Attribute) pair describing a dependency edge. + * + * @param label the target node of the (Rule, Label) edge. + * The source node should already be known. + * @param attribute the attribute. + */ + void acceptLabelAttribute(Label label, Attribute attribute); + } + + /** + * For all attributes that contain labels in their values (either by *being* a label or + * being a collection that includes labels), visits every label and notifies the + * specified observer at each visit. + */ + void visitLabels(AcceptsLabelAttribute observer); + + // TODO(bazel-team): These methods are here to support computed defaults that inherit + // package-level default values. Instead, we should auto-inherit and remove the computed + // defaults. If we really need to give access to package-level defaults, we should come up with + // a more generic interface. + String getPackageDefaultHdrsCheck(); + + Boolean getPackageDefaultObsolete(); + + Boolean getPackageDefaultTestOnly(); + + String getPackageDefaultDeprecation(); + + ImmutableList<String> getPackageDefaultCopts(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/BuildFileContainsErrorsException.java b/src/main/java/com/google/devtools/build/lib/packages/BuildFileContainsErrorsException.java new file mode 100644 index 0000000..d9283c3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/BuildFileContainsErrorsException.java
@@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import javax.annotation.Nullable; + +/** + * Exception indicating a failed attempt to access a package that could not + * be read or had syntax errors. + */ +public class BuildFileContainsErrorsException extends NoSuchPackageException { + + private Package pkg; + + public BuildFileContainsErrorsException(String packageName, String message) { + super(packageName, "error loading package", message); + } + + public BuildFileContainsErrorsException(String packageName, String message, + Throwable cause) { + super(packageName, "error loading package", message, cause); + } + + public BuildFileContainsErrorsException(Package pkg, String msg) { + this(pkg.getName(), msg); + this.pkg = pkg; + } + + @Override + @Nullable + public Package getPackage() { + return pkg; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/BuildFileNotFoundException.java b/src/main/java/com/google/devtools/build/lib/packages/BuildFileNotFoundException.java new file mode 100644 index 0000000..2c70a4e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/BuildFileNotFoundException.java
@@ -0,0 +1,31 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +/** + * Exception indicating an attempt to access a package which is not found or + * does not exist. + */ +public class BuildFileNotFoundException extends NoSuchPackageException { + + public BuildFileNotFoundException(String packageName, String message) { + super(packageName, message); + } + + public BuildFileNotFoundException(String packageName, String message, + Throwable cause) { + super(packageName, message, cause); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/CachingPackageLocator.java b/src/main/java/com/google/devtools/build/lib/packages/CachingPackageLocator.java new file mode 100644 index 0000000..f6badad --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/CachingPackageLocator.java
@@ -0,0 +1,45 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.vfs.Path; + +/** + * CachingPackageLocator implementations locate a package by its name. + * + * <p> Similar to #pkgcache.PathPackageLocator, but implementations are required + * to cache results and handle deleted packages. + * + * <p> This interface exists for two reasons: (1) to avoid creating a bad dependency edge from the + * PythonPreprocessor to lib.pkgcache ("dependency injection") and (2) to allow Skyframe to use + * pieces of legacy code while still updating the Skyframe node graph. + */ +public interface CachingPackageLocator { + + /** + * Returns path of BUILD file for specified package iff the specified package exists, null + * otherwise (e.g. invalid package name, no build file, or package has been deleted via + * --deleted_packages).. + * + * <p> The package's root directory may be computed by calling getParentFile(). + * + * <p> Instances of this interface are required to cache the results. + * + * <p> This method must be thread-safe. + */ + @ThreadSafe + Path getBuildFileForPackage(String packageName); + +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/ConstantRuleVisibility.java b/src/main/java/com/google/devtools/build/lib/packages/ConstantRuleVisibility.java new file mode 100644 index 0000000..bb64015 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/ConstantRuleVisibility.java
@@ -0,0 +1,94 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.syntax.Label; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; + +/** + * A rule visibility that simply says yes or no. It corresponds to public, + * legacy_public and private visibilities. + */ +@Immutable @ThreadSafe +public class ConstantRuleVisibility implements RuleVisibility, Serializable { + static final Label LEGACY_PUBLIC_LABEL; // same as "public"; used for automated depot cleanup + private static final Label PUBLIC_LABEL; + private static final Label PRIVATE_LABEL; + + public static final ConstantRuleVisibility PUBLIC = + new ConstantRuleVisibility(true); + + public static final ConstantRuleVisibility PRIVATE = + new ConstantRuleVisibility(false); + + static { + try { + PUBLIC_LABEL = Label.parseAbsolute("//visibility:public"); + LEGACY_PUBLIC_LABEL = Label.parseAbsolute("//visibility:legacy_public"); + PRIVATE_LABEL = Label.parseAbsolute("//visibility:private"); + } catch (Label.SyntaxException e) { + throw new IllegalStateException(); + } + } + + private final boolean result; + + public ConstantRuleVisibility(boolean result) { + this.result = result; + } + + public boolean isPubliclyVisible() { + return result; + } + + @Override + public List<Label> getDependencyLabels() { + return Collections.emptyList(); + } + + @Override + public List<Label> getDeclaredLabels() { + return ImmutableList.of(result ? PUBLIC_LABEL : PRIVATE_LABEL); + } + + /** + * Tries to parse a list of labels into a {@link ConstantRuleVisibility}. + * + * @param labels the list of labels to parse + * @return The resulting visibility object, or null if the list of labels + * could not be parsed. + */ + public static ConstantRuleVisibility tryParse(List<Label> labels) { + if (labels.size() != 1) { + return null; + } + return tryParse(labels.get(0)); + } + + public static ConstantRuleVisibility tryParse(Label label) { + if (PUBLIC_LABEL.equals(label) || LEGACY_PUBLIC_LABEL.equals(label)) { + return PUBLIC; + } else if (PRIVATE_LABEL.equals(label)) { + return PRIVATE; + } else { + return null; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/DefaultSetting.java b/src/main/java/com/google/devtools/build/lib/packages/DefaultSetting.java new file mode 100644 index 0000000..1c89b4e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/DefaultSetting.java
@@ -0,0 +1,33 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +/** + * A feature of the build that can be switched on and off on a per-package + * basis. + * + * <p>This interface is only for marking targets as being affected by a feature; + * their implementation can be anywhere. + * + * Implementations of this interface must be immutable. + */ +public interface DefaultSetting { + String getSettingName(); + + /** + * Returns if the default setting in question affects the specific target. + */ + boolean appliesTo(Target target); +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/DuplicatePackageException.java b/src/main/java/com/google/devtools/build/lib/packages/DuplicatePackageException.java new file mode 100644 index 0000000..dc21cba --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/DuplicatePackageException.java
@@ -0,0 +1,32 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +/** + * Exception indicating that the same package (i.e. BUILD file) can be loaded + * via different package paths. + */ +// TODO(bazel-team): (2009) Change exception hierarchy so that DuplicatePackageException is no +// longer a child of NoSuchPackageException. +public class DuplicatePackageException extends NoSuchPackageException { + + public DuplicatePackageException(String packageName, String message) { + super(packageName, message); + } + + public DuplicatePackageException(String packageName, String message, Throwable cause) { + super(packageName, message, cause); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/EnumFilterConverter.java b/src/main/java/com/google/devtools/build/lib/packages/EnumFilterConverter.java new file mode 100644 index 0000000..31242e3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/EnumFilterConverter.java
@@ -0,0 +1,97 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.devtools.build.lib.util.StringUtil; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Converter that translates a string of the form "value1,value2,-value3,value4" + * into a corresponding set of allowed Enum values. + * + * <p>Values preceded by '-' are excluded from this set. So "value1,-value2,value3" + * translates to the set [EnumType.value1, EnumType.value3]. + * + * <p>If *all* values are exclusions (e.g. "-value1,-value2,-value3"), the returned + * set contains all values for the Enum type *except* those specified. + */ +class EnumFilterConverter<E extends Enum<E>> implements Converter<Set<E>> { + + private final Set<String> allowedValues = new LinkedHashSet<>(); + private final Class<E> typeClass; + private final String prettyEnumName; + + /** + * Constructor. + * + * @param typeClass this should be E.class (Java generics can't infer that directly) + * @param userFriendlyName a user-friendly description of this enum type + */ + public EnumFilterConverter(Class<E> typeClass, String userFriendlyName) { + this.typeClass = typeClass; + this.prettyEnumName = userFriendlyName; + for (E value : EnumSet.allOf(typeClass)) { + allowedValues.add(value.name()); + } + } + + /** + * Returns the set of allowed values for the option. + * + * Implements {@link #convert(String)}. + */ + @Override + public Set<E> convert(String input) throws OptionsParsingException { + if (input.equals("")) { + return Collections.emptySet(); + } + EnumSet<E> includedSet = EnumSet.noneOf(typeClass); + EnumSet<E> excludedSet = EnumSet.noneOf(typeClass); + for (String value : input.split(",", -1)) { + boolean excludeFlag = value.startsWith("-"); + String s = (excludeFlag ? value.substring(1) : value).toUpperCase(); + if (!allowedValues.contains(s)) { + throw new OptionsParsingException("Invalid " + prettyEnumName + " filter '" + value + + "' in the input '" + input + "'"); + } + (excludeFlag ? excludedSet : includedSet).add(E.valueOf(typeClass, s)); + } + if (includedSet.isEmpty()) { + includedSet = EnumSet.complementOf(excludedSet); + } else { + includedSet.removeAll(excludedSet); + } + if (includedSet.isEmpty()) { + throw new OptionsParsingException( + Character.toUpperCase(prettyEnumName.charAt(0)) + prettyEnumName.substring(1) + + " filter '" + input + "' definition cannot match any tests"); + } + return includedSet; + } + + /** + * Implements {@link #getTypeDescription()}. + */ + @Override + public final String getTypeDescription() { + return "comma-separated list of values: " + + StringUtil.joinEnglishList(allowedValues).toLowerCase(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/EnvironmentGroup.java b/src/main/java/com/google/devtools/build/lib/packages/EnvironmentGroup.java new file mode 100644 index 0000000..07da227 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/EnvironmentGroup.java
@@ -0,0 +1,241 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Model for the "environment_group' rule: the piece of Bazel's rule constraint system that binds + * thematically related environments together and determines which environments a rule supports + * by default. See {@link com.google.devtools.build.lib.analysis.constraints.ConstraintSemantics} + * for precise semantic details of how this information is used. + * + * <p>Note that "environment_group" is implemented as a loading-time function, not a rule. This is + * to support proper discovery of defaults: Say rule A has no explicit constraints and depends + * on rule B, which is explicitly constrained to environment ":bar". Since A declares nothing + * explicitly, it's implicitly constrained to DEFAULTS (whatever that is). Therefore, the + * dependency is only allowed if DEFAULTS doesn't include environments beyond ":bar". To figure + * that out, we need to be able to look up the environment group for ":bar", which is what this + * class provides. + * + * <p>If we implemented this as a rule, we'd have to provide that lookup via rule dependencies, + * e.g. something like: + * + * <code> + * environment( + * name = 'bar', + * group = [':sample_environments'], + * is_default = 1 + * ) + * </code> + * + * <p>But this won't work. This would let us find the environment group for ":bar", but the only way + * to determine what other environments belong to the group is to have the group somehow reference + * them. That would produce circular dependencies in the build graph, which is no good. + */ +@Immutable +public class EnvironmentGroup implements Target { + private final Label label; + private final Location location; + private final Package containingPackage; + private final Set<Label> environments; + private final Set<Label> defaults; + + /** + * Predicate that matches labels from a different package than the initialized package. + */ + private static final class DifferentPackage implements Predicate<Label> { + private final Package containingPackage; + + private DifferentPackage(Package containingPackage) { + this.containingPackage = containingPackage; + } + + @Override + public boolean apply(Label environment) { + return !environment.getPackageName().equals(containingPackage.getName()); + } + } + + /** + * Instantiates a new group without verifying the soundness of its contents. See the validation + * methods below for appropriate checks. + * + * @param label the build label identifying this group + * @param pkg the package this group belongs to + * @param environments the set of environments that belong to this group + * @param defaults the environments a rule implicitly supports unless otherwise specified + * @param location location in the BUILD file of this group + */ + EnvironmentGroup(Label label, Package pkg, final List<Label> environments, List<Label> defaults, + Location location) { + this.label = label; + this.location = location; + this.containingPackage = pkg; + this.environments = ImmutableSet.copyOf(environments); + this.defaults = ImmutableSet.copyOf(defaults); + } + + /** + * Checks that all environments declared by this group are in the same package as the group (so + * we can perform an environment --> environment_group lookup and know the package is available) + * and checks that all defaults are legitimate members of the group. + * + * <p>Does <b>not</b> check that the referenced environments exist (see + * {@link #checkEnvironmentsExist). + * + * @return a list of validation errors that occurred + */ + List<Event> validateMembership() { + List<Event> events = new ArrayList<>(); + + // All environments should belong to the same package as this group. + for (Label environment : + Iterables.filter(environments, new DifferentPackage(containingPackage))) { + events.add(Event.error(location, + environment + " is not in the same package as group " + label)); + } + + // The defaults must be a subset of the member environments. + for (Label unknownDefault : Sets.difference(defaults, environments)) { + events.add(Event.error(location, "default " + unknownDefault + " is not a " + + "declared environment for group " + getLabel())); + } + + return events; + } + + /** + * Given the set of targets in this group's package, checks that all of the group's declared + * environments are part of that set (i.e. the group doesn't reference non-existant labels). + * + * @param pkgTargets mapping from label name to target instance for this group's package + * @return a list of validation errors that occurred + */ + List<Event> checkEnvironmentsExist(Map<String, Target> pkgTargets) { + List<Event> events = new ArrayList<>(); + for (Label envName : environments) { + Target env = pkgTargets.get(envName.getName()); + if (env == null) { + events.add(Event.error(location, "environment " + envName + " does not exist")); + } else if (!env.getTargetKind().equals("environment rule")) { + events.add(Event.error(location, env.getLabel() + " is not a valid environment")); + } + } + return events; + } + + /** + * Returns the environments that belong to this group. + */ + public Set<Label> getEnvironments() { + return environments; + } + + /** + * Returns the environments a rule supports by default, i.e. if it has no explicit references to + * environments in this group. + */ + public Set<Label> getDefaults() { + return defaults; + } + + /** + * Determines whether or not an environment is a default. Returns false if the environment + * doesn't belong to this group. + */ + public boolean isDefault(Label environment) { + return defaults.contains(environment); + } + + @Override + public Label getLabel() { + return label; + } + + @Override + public String getName() { + return label.getName(); + } + + @Override + public Package getPackage() { + return containingPackage; + } + + @Override + public String getTargetKind() { + return targetKind(); + } + + @Override + public Rule getAssociatedRule() { + return null; + } + + @Override + public License getLicense() { + return License.NO_LICENSE; + } + + @Override + public Location getLocation() { + return location; + } + + @Override + public String toString() { + return targetKind() + " " + getLabel(); + } + + @Override + public Set<License.DistributionType> getDistributions() { + return Collections.emptySet(); + } + + @Override + public RuleVisibility getVisibility() { + return ConstantRuleVisibility.PRIVATE; // No rule should be referencing an environment_group. + } + + public static String targetKind() { + return "environment group"; + } + + @Override + public boolean equals(Object o) { + // In a distributed implementation these may not be the same object. + if (o == this) { + return true; + } else if (!(o instanceof EnvironmentGroup)) { + return false; + } else { + return ((EnvironmentGroup) o).getLabel().equals(getLabel()); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/ExternalPackage.java b/src/main/java/com/google/devtools/build/lib/packages/ExternalPackage.java new file mode 100644 index 0000000..9c92b33 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/ExternalPackage.java
@@ -0,0 +1,193 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; +import com.google.devtools.build.lib.packages.RuleFactory.InvalidRuleException; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.vfs.Path; + +import java.util.Map; +import java.util.Map.Entry; + +/** + * This creates the //external package, where targets not homed in this repository can be bound. + */ +public class ExternalPackage extends Package { + + private Map<RepositoryName, Rule> repositoryMap; + + ExternalPackage() { + super(PackageIdentifier.createInDefaultRepo("external")); + } + + /** + * Returns a description of the repository with the given name, or null if there's no such + * repository. + */ + public Rule getRepositoryInfo(RepositoryName repositoryName) { + return repositoryMap.get(repositoryName); + } + + /** + * Holder for a binding's actual label and location. + */ + public static class Binding { + private final Label actual; + private final Location location; + + public Binding(Label actual, Location location) { + this.actual = actual; + this.location = location; + } + + public Label getActual() { + return actual; + } + + public Location getLocation() { + return location; + } + + /** + * Checks if the label is bound, i.e., starts with //external:. + */ + public static boolean isBoundLabel(Label label) { + return label.getPackageName().equals("external"); + } + } + + /** + * Given a workspace file path, creates an ExternalPackage. + */ + public static class ExternalPackageBuilder + extends AbstractBuilder<ExternalPackage, ExternalPackageBuilder> { + private Map<Label, Binding> bindMap = Maps.newHashMap(); + private Map<RepositoryName, Rule> repositoryMap = Maps.newHashMap(); + + public ExternalPackageBuilder(Path workspacePath) { + super(new ExternalPackage()); + setFilename(workspacePath); + setMakeEnv(new MakeEnvironment.Builder()); + } + + @Override + protected ExternalPackageBuilder self() { + return this; + } + + @Override + public ExternalPackage build() { + pkg.repositoryMap = ImmutableMap.copyOf(repositoryMap); + return super.build(); + } + + public void addBinding(Label label, Binding binding) { + bindMap.put(label, binding); + } + + public void resolveBindTargets(RuleClass ruleClass) + throws EvalException, NoSuchBindingException { + for (Entry<Label, Binding> entry : bindMap.entrySet()) { + resolveLabel(entry.getKey(), entry.getValue()); + } + + for (Entry<Label, Binding> entry : bindMap.entrySet()) { + try { + addRule(ruleClass, entry); + } catch (NameConflictException | InvalidRuleException e) { + throw new EvalException(entry.getValue().location, e.getMessage()); + } + } + } + + // Uses tortoise and the hare algorithm to detect cycles. + private void resolveLabel(final Label virtual, Binding binding) + throws NoSuchBindingException { + Label actual = binding.getActual(); + Label tortoise = virtual; + Label hare = actual; + boolean moveTortoise = true; + while (Binding.isBoundLabel(actual)) { + if (tortoise == hare) { + throw new NoSuchBindingException("cycle detected resolving " + virtual + " binding"); + } + + Label previous = actual; // For the exception. + binding = bindMap.get(actual); + if (binding == null) { + throw new NoSuchBindingException("no binding found for target " + previous + " (via " + + virtual + ")"); + } + actual = binding.getActual(); + hare = actual; + moveTortoise = !moveTortoise; + if (moveTortoise) { + tortoise = bindMap.get(tortoise).getActual(); + } + } + bindMap.put(virtual, binding); + } + + private void addRule(RuleClass klass, Map.Entry<Label, Binding> bindingEntry) + throws InvalidRuleException, NameConflictException { + Label virtual = bindingEntry.getKey(); + Label actual = bindingEntry.getValue().actual; + Location location = bindingEntry.getValue().location; + + Map<String, Object> attributes = Maps.newHashMap(); + // Bound rules don't have a name field, but this works because we don't want more than one + // with the same virtual name. + attributes.put("name", virtual.getName()); + attributes.put("actual", actual); + StoredEventHandler handler = new StoredEventHandler(); + Rule rule = RuleFactory.createAndAddRule(this, klass, attributes, handler, null, location); + rule.setVisibility(ConstantRuleVisibility.PUBLIC); + } + + /** + * This is used when a binding is invalid, either because one of the targets is malformed, + * refers to a package that does not exist, or creates a circular dependency. + */ + public class NoSuchBindingException extends NoSuchThingException { + public NoSuchBindingException(String message) { + super(message); + } + } + + /** + * Creates an external repository rule. + * @throws SyntaxException if the repository name is invalid. + */ + public ExternalPackageBuilder createAndAddRepositoryRule(RuleClass ruleClass, + Map<String, Object> kwargs, FuncallExpression ast) + throws InvalidRuleException, NameConflictException, SyntaxException { + StoredEventHandler eventHandler = new StoredEventHandler(); + Rule rule = RuleFactory.createAndAddRule(this, ruleClass, kwargs, eventHandler, ast, + ast.getLocation()); + // Propagate Rule errors to the builder. + addEvents(eventHandler.getEvents()); + repositoryMap.put(RepositoryName.create("@" + rule.getName()), rule); + return this; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/FileTarget.java b/src/main/java/com/google/devtools/build/lib/packages/FileTarget.java new file mode 100644 index 0000000..60f30ce --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/FileTarget.java
@@ -0,0 +1,92 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.packages.License.DistributionType; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType.HasFilename; + +import java.util.Set; + +/** + * Common superclass for InputFile and OutputFile which provides implementation + * for the file operations in common. + */ +public abstract class FileTarget implements Target, HasFilename { + protected final Package pkg; + protected final Label label; + + /** + * Constructs a file with the given label, which must be in the given package. + */ + protected FileTarget(Package pkg, Label label) { + Preconditions.checkArgument(label.getPackageFragment().equals(pkg.getNameFragment())); + this.pkg = pkg; + this.label = label; + } + + @Override + public String getFilename() { + return label.getName(); + } + + @Override + public Label getLabel() { + return label; + } + + @Override + public String getName() { + return label.getName(); + } + + @Override + public Package getPackage() { + return pkg; + } + + @Override + public String toString() { + return getTargetKind() + "(" + getLabel() + ")"; // Just for debugging + } + + @Override + public int hashCode() { + return label.hashCode(); + } + + @Override + public Set<DistributionType> getDistributions() { + return getPackage().getDefaultDistribs(); + } + + /** + * {@inheritDoc} + * + * <p>File licenses are strange, and require some special handling. When + * you ask "What license covers this file?" in a query, the answer should + * be the license declared for the enclosing package. On the other hand, + * if the file is a source for a rule target, and the rule's license declares + * more exceptions than the default inherited by the file, the rule's + * more liberal target should override the stricter license of the file. In + * other words, the license of the rule always overrides the license of + * the non-rule file targets that are inputs to that rule. + */ + @Override + public License getLicense() { + return getPackage().getDefaultLicense(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java b/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java new file mode 100644 index 0000000..3e150b4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/GlobCache.java
@@ -0,0 +1,347 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Throwables; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.SettableFuture; +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.lib.vfs.UnixGlob; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Caches the results of glob expansion for a package. + */ +@ThreadSafety.ThreadCompatible +public class GlobCache { + public static class BadGlobException extends Exception { + BadGlobException(String message) { + super(message); + } + } + + /** + * A mapping from glob expressions (e.g. "*.java") to the list of files it + * matched (in the order returned by VFS) at the time the package was + * constructed. Required for sound dependency analysis. + * + * We don't use a Multimap because it provides no way to distinguish "key not + * present" from (key -> {}). + */ + private final Map<Pair<String, Boolean>, Future<List<Path>>> globCache = new HashMap<>(); + + /** + * The directory in which our package's BUILD file resides. + */ + private final Path packageDirectory; + + /** + * The name of the package we belong to. + */ + private final PackageIdentifier packageId; + + /** + * The package locator-based directory traversal predicate. + */ + private final Predicate<Path> childDirectoryPredicate; + + /** + * System call caching layer. + */ + private AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls; + + /** + * The thread pool for glob evaluation. + */ + private final ThreadPoolExecutor globExecutor; + + /** + * Create a glob expansion cache. + * @param packageDirectory globs will be expanded relatively to this + * directory. + * @param packageId the name of the package this cache belongs to. + * @param locator the package locator. + * @param globExecutor thread pool for glob evaluation. + */ + public GlobCache(final Path packageDirectory, + final PackageIdentifier packageId, + final CachingPackageLocator locator, + AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls, + ThreadPoolExecutor globExecutor) { + this.packageDirectory = Preconditions.checkNotNull(packageDirectory); + this.packageId = Preconditions.checkNotNull(packageId); + this.globExecutor = Preconditions.checkNotNull(globExecutor); + this.syscalls = syscalls == null ? new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS) : syscalls; + + Preconditions.checkNotNull(locator); + final PathFragment pkgNameFrag = packageId.getPackageFragment(); + childDirectoryPredicate = new Predicate<Path>() { + @Override + public boolean apply(Path directory) { + if (directory.equals(packageDirectory)) { + return true; + } + + PathFragment pkgName = pkgNameFrag.getRelative(directory.relativeTo(packageDirectory)); + UnixGlob.FilesystemCalls syscalls = GlobCache.this.syscalls.get(); + if (syscalls != UnixGlob.DEFAULT_SYSCALLS) { + // This is needed because in case the BUILD file exists, we do not call readdir() on its + // directory. However, the package needs to be re-evaluated if the BUILD file is removed. + // Therefore, we add this BUILD file to our dependencies by statting it through the + // recording syscall object so that the BUILD file itself is added to the dependencies of + // this package. + // + // The stat() call issued by locator.getBuildFileForPackage() does not quite cut it + // because 1. it is cached so there may not be a stat() call at all and 2. even if there + // is, it does not go through the proxy in GlobCache.this.syscalls. + // + // Note that this does not cause any significant slowdown; the BUILD file cache will have + // already evaluated the very same stat, so we don't pay any I/O cost, only a cache + // lookup. + syscalls.statNullable(directory.getChild("BUILD"), Symlinks.FOLLOW); + } + + return locator.getBuildFileForPackage(pkgName.getPathString()) == null; + } + }; + } + + /** + * Returns the future result of evaluating glob "pattern" against this + * package's directory, using the package's cache of previously-started + * globs if possible. + * + * @return the list of paths matching the pattern, relative to the package's + * directory. + * @throws BadGlobException if the glob was syntactically invalid, or + * contained uplevel references. + */ + Future<List<Path>> getGlobAsync(String pattern, boolean excludeDirs) + throws BadGlobException { + Future<List<Path>> cached = globCache.get(Pair.of(pattern, excludeDirs)); + if (cached == null) { + cached = safeGlob(pattern, excludeDirs); + setGlobPaths(pattern, excludeDirs, cached); + } + return cached; + } + + @VisibleForTesting + List<String> getGlob(String pattern) + throws IOException, BadGlobException, InterruptedException { + return getGlob(pattern, false); + } + + @VisibleForTesting + protected List<String> getGlob(String pattern, boolean excludeDirs) + throws IOException, BadGlobException, InterruptedException { + Future<List<Path>> futureResult = getGlobAsync(pattern, excludeDirs); + List<Path> globPaths = fromFuture(futureResult); + // Replace the UnixGlob.GlobFuture with a completed future object, to allow + // garbage collection of the GlobFuture and GlobVisitor objects. + if (!(futureResult instanceof SettableFuture<?>)) { + SettableFuture<List<Path>> completedFuture = SettableFuture.create(); + completedFuture.set(globPaths); + globCache.put(Pair.of(pattern, excludeDirs), completedFuture); + } + + List<String> result = Lists.newArrayListWithCapacity(globPaths.size()); + for (Path path : globPaths) { + String relative = path.relativeTo(packageDirectory).getPathString(); + // Don't permit "" (meaning ".") in the glob expansion, since it's + // invalid as a label, plus users should say explicitly if they + // really want to name the package directory. + if (!relative.isEmpty()) { + result.add(relative); + } + } + return result; + } + + /** + * Adds glob entries to the cache, making sure they are sorted first. + */ + @VisibleForTesting + void setGlobPaths(String pattern, boolean excludeDirectories, Future<List<Path>> result) { + globCache.put(Pair.of(pattern, excludeDirectories), result); + } + + /** + * Actually execute a glob against the filesystem. Otherwise similar to + * getGlob(). + */ + @VisibleForTesting + Future<List<Path>> safeGlob(String pattern, boolean excludeDirs) throws BadGlobException { + // Forbidden patterns: + if (pattern.indexOf('?') != -1) { + throw new BadGlobException("glob pattern '" + pattern + "' contains forbidden '?' wildcard"); + } + // Patterns forbidden by UnixGlob library: + String error = UnixGlob.checkPatternForError(pattern); + if (error != null) { + throw new BadGlobException(error + " (in glob pattern '" + pattern + "')"); + } + return UnixGlob.forPath(packageDirectory) + .addPattern(pattern) + .setExcludeDirectories(excludeDirs) + .setDirectoryFilter(childDirectoryPredicate) + .setThreadPool(globExecutor) + .setFilesystemCalls(syscalls) + .globAsync(true); + } + + /** + * Sanitize the future exceptions - the only expected checked exception + * is IOException. + */ + private static List<Path> fromFuture(Future<List<Path>> future) + throws IOException, InterruptedException { + try { + return future.get(); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + Throwables.propagateIfPossible(cause, + IOException.class, InterruptedException.class); + throw new RuntimeException(e); + } + } + + /** + * Returns true iff all this package's globs are up-to-date. That is, + * re-evaluating the package's BUILD file at this moment would yield an + * equivalent Package instance. (This call requires filesystem I/O to + * re-evaluate the globs.) + */ + public boolean globsUpToDate() throws InterruptedException { + // Start all globs in parallel. + Map<Pair<String, Boolean>, Future<List<Path>>> newGlobs = new HashMap<>(); + try { + for (Map.Entry<Pair<String, Boolean>, Future<List<Path>>> entry : globCache.entrySet()) { + Pair<String, Boolean> key = entry.getKey(); + try { + newGlobs.put(key, safeGlob(key.first, key.second)); + } catch (BadGlobException e) { + return false; + } + } + + for (Map.Entry<Pair<String, Boolean>, Future<List<Path>>> entry : globCache.entrySet()) { + try { + Pair<String, Boolean> key = entry.getKey(); + List<Path> newGlob = fromFuture(newGlobs.get(key)); + List<Path> oldGlob = fromFuture(entry.getValue()); + if (!oldGlob.equals(newGlob)) { + return false; + } + } catch (IOException e) { + return false; + } + } + + return true; + } finally { + finishBackgroundTasks(newGlobs.values()); + } + } + + /** + * Evaluate the build language expression "glob(includes, excludes)" in the + * context of this package. + * + * <p>Called by PackageFactory via Package. + */ + public List<String> glob(List<String> includes, List<String> excludes, boolean excludeDirs) + throws IOException, BadGlobException, InterruptedException { + // Start globbing all patterns in parallel. The getGlob() calls below will + // block on an individual pattern's results, but the other globs can + // continue in the background. + for (String pattern : Iterables.concat(includes, excludes)) { + getGlobAsync(pattern, excludeDirs); + } + + Set<String> results = new LinkedHashSet<>(); + for (String pattern : includes) { + results.addAll(getGlob(pattern, excludeDirs)); + } + for (String pattern : excludes) { + results.removeAll(getGlob(pattern, excludeDirs)); + } + + Preconditions.checkState(!results.contains(null), "glob returned null"); + return new ArrayList<>(results); + } + + public Set<Pair<String, Boolean>> getKeySet() { + return globCache.keySet(); + } + + /** + * Block on the completion of all potentially-abandoned background tasks. + */ + public void finishBackgroundTasks() { + finishBackgroundTasks(globCache.values()); + } + + public void cancelBackgroundTasks() { + cancelBackgroundTasks(globCache.values()); + } + + private static void finishBackgroundTasks(Collection<Future<List<Path>>> tasks) { + for (Future<List<Path>> task : tasks) { + try { + fromFuture(task); + } catch (IOException | InterruptedException e) { + // Ignore: If this was still going on in the background, some other + // failure already occurred. + } + } + } + + private static void cancelBackgroundTasks(Collection<Future<List<Path>>> tasks) { + for (Future<List<Path>> task : tasks) { + task.cancel(true); + + try { + task.get(); + } catch (ExecutionException | InterruptedException e) { + // We don't care. Point is, the task does not bother us anymore. + } + } + } + + @Override + public String toString() { + return "GlobCache for " + packageId + " in " + packageDirectory; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/ImplicitOutputsFunction.java b/src/main/java/com/google/devtools/build/lib/packages/ImplicitOutputsFunction.java new file mode 100644 index 0000000..9f72fd0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/ImplicitOutputsFunction.java
@@ -0,0 +1,421 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import static com.google.devtools.build.lib.syntax.SkylarkFunction.castMap; +import static java.util.Collections.singleton; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.escape.Escaper; +import com.google.common.escape.Escapers; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.ClassObject; +import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkCallbackFunction; +import com.google.devtools.build.lib.util.StringUtil; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A function interface allowing rules to specify their set of implicit outputs + * in a more dynamic way than just simple template-substitution. For example, + * the set of implicit outputs may be a function of rule attributes. + */ +public abstract class ImplicitOutputsFunction { + + /** + * Implicit output functions for Skylark supporting key value access of expanded implicit outputs. + */ + public abstract static class SkylarkImplicitOutputsFunction extends ImplicitOutputsFunction { + + public abstract ImmutableMap<String, String> calculateOutputs(AttributeMap map) + throws EvalException; + + @Override + public Iterable<String> getImplicitOutputs(AttributeMap map) throws EvalException { + return calculateOutputs(map).values(); + } + } + + /** + * Implicit output functions executing Skylark code. + */ + public static final class SkylarkImplicitOutputsFunctionWithCallback + extends SkylarkImplicitOutputsFunction { + + private final SkylarkCallbackFunction callback; + private final Location loc; + + public SkylarkImplicitOutputsFunctionWithCallback( + SkylarkCallbackFunction callback, Location loc) { + this.callback = callback; + this.loc = loc; + } + + @Override + public ImmutableMap<String, String> calculateOutputs(AttributeMap map) throws EvalException { + Map<String, Object> attrValues = new HashMap<>(); + for (String attrName : map.getAttributeNames()) { + // TODO(bazel-team): support configurable attributes - which value would we want to + // pass on to the child outputs function? Maybe implicit output functions shouldn't + // have access to configurable values (makes them too complicated?). Maybe they + // should have *full* access (gives them the most power?). + Object value = map.get(attrName, map.getAttributeType(attrName)); + if (value != null) { + attrValues.put(attrName, value); + } + } + ClassObject attrs = new SkylarkClassObject(attrValues, "No such attribute '%s'"); + try { + ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); + for (Map.Entry<String, String> entry : castMap(callback.call(attrs), + String.class, String.class, "implicit outputs function return value")) { + Iterable<String> substitutions = fromTemplates(entry.getValue()).getImplicitOutputs(map); + if (!Iterables.isEmpty(substitutions)) { + builder.put(entry.getKey(), Iterables.getOnlyElement(substitutions)); + } + } + return builder.build(); + } catch (IllegalArgumentException e) { + throw new EvalException(loc, e.getMessage()); + } + } + } + + /** + * Implicit output functions using a simple an output map. + */ + public static final class SkylarkImplicitOutputsFunctionWithMap + extends SkylarkImplicitOutputsFunction { + + private final ImmutableMap<String, String> outputMap; + + public SkylarkImplicitOutputsFunctionWithMap(ImmutableMap<String, String> outputMap) { + this.outputMap = outputMap; + } + + @Override + public ImmutableMap<String, String> calculateOutputs(AttributeMap map) { + ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); + for (Map.Entry<String, String> entry : outputMap.entrySet()) { + Iterable<String> substitutions = fromTemplates(entry.getValue()).getImplicitOutputs(map); + if (!Iterables.isEmpty(substitutions)) { + builder.put(entry.getKey(), Iterables.getOnlyElement(substitutions)); + } + } + return builder.build(); + } + } + + /** + * Implicit output functions which can not throw an EvalException. + */ + public abstract static class SafeImplicitOutputsFunction extends ImplicitOutputsFunction { + @Override + public abstract Iterable<String> getImplicitOutputs(AttributeMap map); + } + + /** + * An interface to objects that can retrieve rule attributes. + */ + public interface AttributeValueGetter { + /** + * Returns the value(s) of attribute "attr" in "rule", or empty set if attribute unknown. + */ + Set<String> get(AttributeMap rule, String attr); + } + + /** + * The default rule attribute retriever. + * + * <p>Custom {@link AttributeValueGetter} implementations may delegate to this object as a + * fallback mechanism. + */ + public static final AttributeValueGetter DEFAULT_RULE_ATTRIBUTE_GETTER = + new AttributeValueGetter() { + @Override + public Set<String> get(AttributeMap rule, String attr) { + return attributeValues(rule, attr); + } + }; + + private static final Escaper PERCENT_ESCAPER = Escapers.builder().addEscape('%', "%%").build(); + + /** + * Given a newly-constructed Rule instance (with attributes populated), + * returns the list of output files that this rule produces implicitly. + */ + public abstract Iterable<String> getImplicitOutputs(AttributeMap rule) throws EvalException; + + /** + * The implicit output function that returns no files. + */ + public static final SafeImplicitOutputsFunction NONE = new SafeImplicitOutputsFunction() { + @Override public Iterable<String> getImplicitOutputs(AttributeMap rule) { + return Collections.emptyList(); + } + }; + + /** + * A convenience wrapper for {@link #fromTemplates(Iterable)}. + */ + public static SafeImplicitOutputsFunction fromTemplates(String... templates) { + return fromTemplates(Arrays.asList(templates)); + } + + /** + * The implicit output function that generates files based on a set of + * template substitutions using rule attribute values. + * + * @param templates The templates used to construct the name of the implicit + * output file target. The substring "%{name}" will be replaced by the + * actual name of the rule, the substring "%{srcs}" will be replaced by the + * name of each source file without its extension. If multiple %{} + * substrings exist, the cross-product of them is generated. + */ + public static SafeImplicitOutputsFunction fromTemplates(final Iterable<String> templates) { + return new SafeImplicitOutputsFunction() { + // TODO(bazel-team): parse the templates already here + @Override + public Iterable<String> getImplicitOutputs(AttributeMap rule) { + Iterable<String> result = null; + for (String template : templates) { + List<String> substitutions = substitutePlaceholderIntoTemplate(template, rule); + if (substitutions.isEmpty()) { + continue; + } + if (result == null) { + result = substitutions; + } else { + result = Iterables.concat(result, substitutions); + } + } + if (result == null) { + return ImmutableList.<String>of(); + } else { + return Sets.newLinkedHashSet(result); + } + } + + @Override + public String toString() { + return StringUtil.joinEnglishList(templates); + } + }; + } + + /** + * A convenience wrapper for {@link #fromFunctions(Iterable)}. + */ + public static SafeImplicitOutputsFunction fromFunctions( + SafeImplicitOutputsFunction... functions) { + return fromFunctions(Arrays.asList(functions)); + } + + /** + * The implicit output function that generates files based on a set of + * template substitutions using rule attribute values. + * + * @param functions The functions used to construct the name of the implicit + * output file target. The substring "%{name}" will be replaced by the + * actual name of the rule, the substring "%{srcs}" will be replaced by the + * name of each source file without its extension. If multiple %{} + * substrings exist, the cross-product of them is generated. + */ + public static SafeImplicitOutputsFunction fromFunctions( + final Iterable<SafeImplicitOutputsFunction> functions) { + return new SafeImplicitOutputsFunction() { + @Override + public Iterable<String> getImplicitOutputs(AttributeMap rule) { + Collection<String> result = new LinkedHashSet<>(); + for (SafeImplicitOutputsFunction function : functions) { + Iterables.addAll(result, function.getImplicitOutputs(rule)); + } + return result; + } + @Override + public String toString() { + return StringUtil.joinEnglishList(functions); + } + }; + } + + /** + * Coerces attribute "attrName" of the specified rule into a sequence of + * strings. Helper function for {@link #fromTemplates(Iterable)}. + */ + private static Set<String> attributeValues(AttributeMap rule, String attrName) { + // Special case "name" since it's not treated as an attribute. + if (attrName.equals("name")) { + return singleton(rule.getName()); + } else if (attrName.equals("dirname")) { + PathFragment dir = new PathFragment(rule.getName()).getParentDirectory(); + return (dir.segmentCount() == 0) ? singleton("") : singleton(dir.getPathString() + "/"); + } else if (attrName.equals("basename")) { + return singleton(new PathFragment(rule.getName()).getBaseName()); + } + + Type<?> attrType = rule.getAttributeType(attrName); + if (attrType == null) { return Collections.emptySet(); } + // String attributes and lists are easy. + if (Type.STRING == attrType) { + return singleton(rule.get(attrName, Type.STRING)); + } else if (Type.STRING_LIST == attrType) { + Iterable<String> values = rule.get(attrName, Type.STRING_LIST); + // TODO(bazel-team): extract this for modularization + if ("locales".equals(attrName)) { + // Locales have to be lowercased before used in a file name for + // consistency with file naming guidelines, and convert dash-style + // (en-US-pseudo) to underscore-style (en_US_pseudo). + values = Iterables.transform(values, + new Function<String, String>() { + @Override + public String apply(String s) { + return s.toLowerCase().replace('-', '_'); + } + }); + } + return Sets.newLinkedHashSet(values); + } else if (Type.LABEL_LIST == attrType) { + // Labels are most often used to change the extension, + // e.g. %.foo -> %.java, so we return the basename w/o extension. + return Sets.newLinkedHashSet( + Iterables.transform(rule.get(attrName, Type.LABEL_LIST), + new Function<Label, String>() { + @Override + public String apply(Label label) { + return FileSystemUtils.removeExtension(label.getName()); + } + })); + } else if (Type.OUTPUT == attrType) { + Label out = rule.get(attrName, Type.OUTPUT); + return singleton(out.getName()); + } else if (Type.OUTPUT_LIST == attrType) { + return Sets.newLinkedHashSet( + Iterables.transform(rule.get(attrName, Type.OUTPUT_LIST), + new Function<Label, String>() { + @Override + public String apply(Label label) { + return label.getName(); + } + })); + } + throw new IllegalArgumentException( + "Don't know how to handle " + attrName + " : " + attrType); + } + + /** + * Collects all named placeholders from the template while replacing them with %s. + * + * <p>Example: for {@code template} "%{name}_%{locales}.foo", it will return "%s_%s.foo" and + * store "name" and "locales" in {@code placeholders}. + * + * <p>Incomplete placeholders are treated like text: for "a-%{x}-%{y" this method returns + * "a-%s-%%{y" and stores "x" in {@code placeholders}. + * + * @param template a string with placeholders of the format %{...} + * @param placeholders a collection to collect placeholders into; may contain duplicates if not a + * Set + * @return a format string for {@link String#format}, created from the template string with every + * placeholder replaced by %s + */ + public static String createPlaceholderSubstitutionFormatString(String template, + Collection<String> placeholders) { + return createPlaceholderSubstitutionFormatStringRecursive(template, placeholders, + new StringBuilder()); + } + + private static String createPlaceholderSubstitutionFormatStringRecursive(String template, + Collection<String> placeholders, StringBuilder formatBuilder) { + int start = template.indexOf("%{"); + if (start < 0) { + return formatBuilder.append(PERCENT_ESCAPER.escape(template)).toString(); + } + + int end = template.indexOf("}", start + 2); + if (end < 0) { + return formatBuilder.append(PERCENT_ESCAPER.escape(template)).toString(); + } + + formatBuilder.append(PERCENT_ESCAPER.escape(template.substring(0, start))).append("%s"); + placeholders.add(template.substring(start + 2, end)); + return createPlaceholderSubstitutionFormatStringRecursive(template.substring(end + 1), + placeholders, formatBuilder); + } + + /** + * Given a template string, replaces all placeholders of the form %{...} with + * the values from attributeSource. If there are multiple placeholders, then + * the output is the cross product of substitutions. + */ + public static ImmutableList<String> substitutePlaceholderIntoTemplate(String template, + AttributeMap rule) { + return substitutePlaceholderIntoTemplate(template, rule, DEFAULT_RULE_ATTRIBUTE_GETTER, null); + } + + /** + * Substitutes attribute-placeholders in a template string, producing all possible combinations. + * + * @param template the template string, may contain named placeholders for rule attributes, like + * <code>%{name}</code> or <code>%{deps}</code> + * @param rule the rule whose attributes the placeholders correspond to + * @param placeholdersInTemplate if specified, will contain all placeholders found in the + * template; may contain duplicates + * @return all possible combinations of the attributes referenced by the placeholders, + * substituted into the template; empty if any of the placeholders expands to no values + */ + public static ImmutableList<String> substitutePlaceholderIntoTemplate(String template, + AttributeMap rule, AttributeValueGetter attributeGetter, + @Nullable List<String> placeholdersInTemplate) { + List<String> placeholders = (placeholdersInTemplate == null) + ? Lists.<String>newArrayList() + : placeholdersInTemplate; + String formatStr = createPlaceholderSubstitutionFormatString(template, placeholders); + if (placeholders.isEmpty()) { + return ImmutableList.of(template); + } + + List<Set<String>> values = Lists.newArrayListWithCapacity(placeholders.size()); + for (String placeholder : placeholders) { + Set<String> attrValues = attributeGetter.get(rule, placeholder); + if (attrValues.isEmpty()) { + return ImmutableList.<String>of(); + } + values.add(attrValues); + } + ImmutableList.Builder<String> out = new ImmutableList.Builder<>(); + for (List<String> combination : Sets.cartesianProduct(values)) { + out.add(String.format(formatStr, combination.toArray())); + } + return out.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/InputFile.java b/src/main/java/com/google/devtools/build/lib/packages/InputFile.java new file mode 100644 index 0000000..130e2b7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/InputFile.java
@@ -0,0 +1,124 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * A file that is an input to the build system. + * + * <p>In the build system, a file is considered an <i>input</i> file iff it is + * not generated by the build system (e.g. it's maintained under version + * control, or created by the test harness). It has nothing to do with the + * type of the file; a generated file containing <code>Java</code> source code + * is an OutputFile, not an InputFile. + */ +@Immutable @ThreadSafe +public final class InputFile extends FileTarget { + private final Location location; + private final RuleVisibility visibility; + private final License license; + + /** + * Constructs an input file with the given label, which must be a label for + * the given package, and package-default visibility. + */ + InputFile(Package pkg, Label label, Location location) { + this(pkg, label, location, null, License.NO_LICENSE); + } + + /** + * Constructs an input file with the given label, which must be a label for the given package + * that was parsed from the specified location, and has the specified visibility. + */ + InputFile(Package pkg, Label label, Location location, RuleVisibility visibility, + License license) { + super(pkg, label); + Preconditions.checkNotNull(location); + this.location = location; + this.visibility = visibility; + this.license = license; + } + + public boolean isVisibilitySpecified() { + return visibility != null; + } + + @Override + public RuleVisibility getVisibility() { + if (visibility != null) { + return visibility; + } else { + return pkg.getDefaultVisibility(); + } + } + + public boolean isLicenseSpecified() { + return license != null && license != License.NO_LICENSE; + } + + @Override + public License getLicense() { + if (license != null) { + return license; + } else { + return pkg.getDefaultLicense(); + } + } + + /** + * Returns the path to the location of the input file (which is necessarily + * within the source tree, not beneath <code>bin</code> or + * <code>genfiles</code>. + * + * <p>Prefer {@link #getExecPath} if possible. + */ + public Path getPath() { + return pkg.getPackageDirectory().getRelative(label.getName()); + } + + /** + * Returns the exec path of the file, i.e. the path relative to the package source root. + */ + public PathFragment getExecPath() { + return label.toPathFragment(); + } + + @Override + public int hashCode() { + return label.hashCode(); + } + + @Override + public String getTargetKind() { + return "source file"; + } + + @Override + public Rule getAssociatedRule() { + return null; + } + + @Override + public Location getLocation() { + return location; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/InvalidPackageNameException.java b/src/main/java/com/google/devtools/build/lib/packages/InvalidPackageNameException.java new file mode 100644 index 0000000..69561b8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/InvalidPackageNameException.java
@@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +/** + * Exception indicating that a package name was invalid. + */ +public class InvalidPackageNameException extends NoSuchPackageException { + + public InvalidPackageNameException(String packageName, String message) { + super(packageName, message); + } + + public InvalidPackageNameException(String packageName, String message, Throwable cause) { + super(packageName, message, cause); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/License.java b/src/main/java/com/google/devtools/build/lib/packages/License.java new file mode 100644 index 0000000..fe63c9c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/License.java
@@ -0,0 +1,327 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableTable; +import com.google.common.collect.Sets; +import com.google.common.collect.Table; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; + +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +/** + * Support for license and distribution checking. + */ +@Immutable @ThreadSafe +public final class License { + + private final Set<LicenseType> licenseTypes; + private final Set<Label> exceptions; + + /** + * The error that's thrown if a build file contains an invalid license string. + */ + public static class LicenseParsingException extends Exception { + public LicenseParsingException(String s) { + super(s); + } + } + + /** + * LicenseType is the basis of the License lattice - stricter licenses should + * be declared before less-strict licenses in the enum. + * + * <p>Note that the order is important for the purposes of finding the least + * restrictive license. + */ + public enum LicenseType { + BY_EXCEPTION_ONLY, + RESTRICTED, + RESTRICTED_IF_STATICALLY_LINKED, + RECIPROCAL, + NOTICE, + PERMISSIVE, + UNENCUMBERED, + NONE + } + + /** + * Gets the least restrictive license type from the list of licenses declared + * for a target. For the purposes of license checking, the license type set of + * a declared license can be reduced to its least restrictive member. + * + * @param types a collection of license types + * @return the least restrictive license type + */ + @VisibleForTesting + static LicenseType leastRestrictive(Collection<LicenseType> types) { + return types.isEmpty() ? LicenseType.BY_EXCEPTION_ONLY : Collections.max(types); + } + + /** + * An instance of LicenseType.None with no exceptions, used for packages + * outside of third_party which have no license clause in their BUILD files. + */ + public static final License NO_LICENSE = + new License(ImmutableSet.of(LicenseType.NONE), Collections.<Label>emptySet()); + + /** + * A default instance of Distributions which is used for packages which + * have no "distribs" declaration. If nothing is declared, we opt for the + * most permissive kind of distribution, which is the internal-only distrib. + */ + public static final Set<DistributionType> DEFAULT_DISTRIB = + Collections.singleton(DistributionType.INTERNAL); + + /** + * The types of distribution that are supported. + */ + public enum DistributionType { + INTERNAL, + WEB, + CLIENT, + EMBEDDED + } + + /** + * Parses a set of strings declaring distribution types. + * + * @param distStrings strings containing distribution declarations from BUILD + * files + * @return a new, unmodifiable set of DistributionTypes + * @throws LicenseParsingException + */ + public static Set<DistributionType> parseDistributions(Collection<String> distStrings) + throws LicenseParsingException { + if (distStrings.isEmpty()) { + return Collections.unmodifiableSet(EnumSet.of(DistributionType.INTERNAL)); + } else { + Set<DistributionType> result = EnumSet.noneOf(DistributionType.class); + for (String distStr : distStrings) { + try { + DistributionType dist = Enum.valueOf(DistributionType.class, distStr.toUpperCase()); + result.add(dist); + } catch (IllegalArgumentException e) { + throw new LicenseParsingException("Invalid distribution type '" + distStr + "'"); + } + } + return Collections.unmodifiableSet(result); + } + } + + private static final Object MARKER = new Object(); + + /** + * The license incompatibility set. This contains the set of + * (Distribution,License) pairs that should generate errors. + */ + private static Table<DistributionType, LicenseType, Object> LICENSE_INCOMPATIBILIES = + createLicenseIncompatibilitySet(); + + private static Table<DistributionType, LicenseType, Object> createLicenseIncompatibilitySet() { + Table<DistributionType, LicenseType, Object> result = HashBasedTable.create(); + result.put(DistributionType.CLIENT, LicenseType.RESTRICTED, MARKER); + result.put(DistributionType.EMBEDDED, LicenseType.RESTRICTED, MARKER); + result.put(DistributionType.INTERNAL, LicenseType.BY_EXCEPTION_ONLY, MARKER); + result.put(DistributionType.CLIENT, LicenseType.BY_EXCEPTION_ONLY, MARKER); + result.put(DistributionType.WEB, LicenseType.BY_EXCEPTION_ONLY, MARKER); + result.put(DistributionType.EMBEDDED, LicenseType.BY_EXCEPTION_ONLY, MARKER); + return ImmutableTable.copyOf(result); + } + + /** + * The license warning set. This contains the set of + * (Distribution,License) pairs that should generate warnings when the user + * requests verbose license checking. + */ + private static Table<DistributionType, LicenseType, Object> LICENSE_WARNINGS = + createLicenseWarningsSet(); + + private static Table<DistributionType, LicenseType, Object> createLicenseWarningsSet() { + Table<DistributionType, LicenseType, Object> result = HashBasedTable.create(); + result.put(DistributionType.CLIENT, LicenseType.RECIPROCAL, MARKER); + result.put(DistributionType.EMBEDDED, LicenseType.RECIPROCAL, MARKER); + result.put(DistributionType.CLIENT, LicenseType.NOTICE, MARKER); + result.put(DistributionType.EMBEDDED, LicenseType.NOTICE, MARKER); + return ImmutableTable.copyOf(result); + } + + private License(Set<LicenseType> licenseTypes, Set<Label> exceptions) { + // Defensive copy is done in .of() + this.licenseTypes = licenseTypes; + this.exceptions = exceptions; + } + + public static License of(Collection<LicenseType> licenses, Collection<Label> exceptions) { + Set<LicenseType> licenseSet = ImmutableSet.copyOf(licenses); + Set<Label> exceptionSet = ImmutableSet.copyOf(exceptions); + + if (exceptionSet.isEmpty() && licenseSet.equals(ImmutableSet.of(LicenseType.NONE))) { + return License.NO_LICENSE; + } + + return new License(licenseSet, exceptionSet); + } + /** + * Computes a license which can be used to check if a package is compatible + * with some kinds of distribution. The list of licenses is scanned for the + * least restrictive, and the exceptions are added. + * + * @param licStrings the list of license strings declared for the package + * @throws LicenseParsingException if there are any parsing problems + */ + public static License parseLicense(List<String> licStrings) throws LicenseParsingException { + /* + * The semantics of comparison for licenses depends on a stable iteration + * order for both license types and exceptions. For licenseTypes, it will be + * the comparison order from the enumerated types; for exceptions, it will + * be lexicographic order achieved using TreeSets. + */ + Set<LicenseType> licenseTypes = EnumSet.noneOf(LicenseType.class); + Set<Label> exceptions = Sets.newTreeSet(); + for (String str : licStrings) { + if (str.startsWith("exception=")) { + try { + Label label = Label.parseAbsolute(str.substring("exception=".length())); + exceptions.add(label); + } catch (SyntaxException e) { + throw new LicenseParsingException(e.getMessage()); + } + } else { + try { + licenseTypes.add(LicenseType.valueOf(str.toUpperCase())); + } catch (IllegalArgumentException e) { + throw new LicenseParsingException("invalid license type: '" + str + "'"); + } + } + } + + return License.of(licenseTypes, exceptions); + } + + /** + * Checks if this license is compatible with distributing a particular target + * in some set of distribution modes. + * + * @param dists the modes of distribution + * @param target the target which is being checked, and which will be used for + * checking exceptions + * @param licensedTarget the target which declared the license being checked. + * @param eventHandler a reporter where any licensing issues discovered should be + * reported + * @param staticallyLinked whether the target is statically linked under this command + * @return true if the license is compatible with the distributions + */ + public boolean checkCompatibility(Set<DistributionType> dists, + Target target, Label licensedTarget, EventHandler eventHandler, + boolean staticallyLinked) { + Location location = (target instanceof Rule) ? ((Rule) target).getLocation() : null; + + LicenseType leastRestrictiveLicense; + if (licenseTypes.contains(LicenseType.RESTRICTED_IF_STATICALLY_LINKED)) { + Set<LicenseType> tempLicenses = EnumSet.copyOf(licenseTypes); + tempLicenses.remove(LicenseType.RESTRICTED_IF_STATICALLY_LINKED); + if (staticallyLinked) { + tempLicenses.add(LicenseType.RESTRICTED); + } else { + tempLicenses.add(LicenseType.UNENCUMBERED); + } + leastRestrictiveLicense = leastRestrictive(tempLicenses); + } else { + leastRestrictiveLicense = leastRestrictive(licenseTypes); + } + for (DistributionType dt : dists) { + if (LICENSE_INCOMPATIBILIES.contains(dt, leastRestrictiveLicense)) { + if (!exceptions.contains(target.getLabel())) { + eventHandler.handle(Event.error(location, "Build target '" + target.getLabel() + + "' is not compatible with license '" + this + "' from target '" + + licensedTarget + "'")); + return false; + } + } else if (LICENSE_WARNINGS.contains(dt, leastRestrictiveLicense)) { + eventHandler.handle( + Event.warn(location, "Build target '" + target + + "' has a potential licensing issue " + + "with a '" + this + "' license from target '" + licensedTarget + "'")); + } + } + return true; + } + + /** + * @return an immutable set of {@link LicenseType}s contained in this {@code + * License} + */ + public Set<LicenseType> getLicenseTypes() { + return licenseTypes; + } + + /** + * @return an immutable set of {@link Label}s that describe exceptions to the + * {@code License} + */ + public Set<Label> getExceptions() { + return exceptions; + } + + /** + * A simple toString implementation which generates a canonical form of the + * license. (The order of license types is guaranteed to be canonical by + * EnumSet, and the order of exceptions is guaranteed to be lexicographic + * order by TreeSet.) + */ + @Override + public String toString() { + if (exceptions.isEmpty()) { + return licenseTypes.toString().toLowerCase(); + } else { + return licenseTypes.toString().toLowerCase() + " with exceptions " + exceptions.toString(); + } + } + + /** + * A simple equals implementation leveraging the support built into Set that + * delegates to its contents. + */ + @Override + public boolean equals(Object o) { + return o == this || + o instanceof License && + ((License) o).licenseTypes.equals(this.licenseTypes) && + ((License) o).exceptions.equals(this.exceptions); + } + + /** + * A simple hashCode implementation leveraging the support built into Set that + * delegates to its contents. + */ + @Override + public int hashCode() { + return licenseTypes.hashCode() * 43 + exceptions.hashCode(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/MakeEnvironment.java b/src/main/java/com/google/devtools/build/lib/packages/MakeEnvironment.java new file mode 100644 index 0000000..6c0d849 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/MakeEnvironment.java
@@ -0,0 +1,184 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Environment for varref variables (formerly called "Makefile + * variables"). + * + * <p><code>update</code> emulates a very restricted subset of the behaviour of + * GNU Make's environment. In particular, does not attempt to simulate Make's + * complex range of assigment operators. + */ +@Immutable @ThreadSafe +public class MakeEnvironment { + /** + * The platform set regexp that matches all platforms. Canonical. + */ + public static final String MATCH_ANY = ".*"; + + // A platform-specific binding of a value for a given variable. + static class Binding { + private final String value; + private final String platformSetRegexp; + + Binding(String value, String platformSetRegexp) { + this.value = value; + this.platformSetRegexp = platformSetRegexp; + } + + @Override + public String toString() { + return value + " (" + platformSetRegexp + ")"; + } + + String getValue() { + return value; + } + + String getPlatformSetRegexp() { + return platformSetRegexp; + } + } + + // Maps each variable name to the [short] list of platform-specific bindings + // for it. The first matching binding is definitive. + private final ImmutableMap<String, ImmutableList<Binding>> env; + + private MakeEnvironment(ImmutableMap<String, ImmutableList<Binding>> env) { + this.env = env; + } + + /** + * @return the "Make" value from the environment whose name is "varname", or + * null iff the variable is not defined in the environment. + */ + public String lookup(String varname, String platform) { + List<Binding> bindings = env.get(varname); + if (bindings == null) { + return null; + } + // First, look for a matching non-default binding. + // (The order in 'bindings' is the reverse of the order of vardefs in the BUILD file, so + // the first match in this for loop selects the last matching definition in the BUILD file.) + for (Binding binding : bindings) { + if (!binding.platformSetRegexp.equals(MATCH_ANY) && + platform.matches(binding.platformSetRegexp)) { + return binding.value; + } + } + // If we didn't find a matching non-default binding, + // try using the last default binding. + for (Binding binding : bindings) { + if (binding.platformSetRegexp.equals(MATCH_ANY)) { + return binding.value; + } + } + return null; + } + + Map<String, ImmutableList<Binding>> getBindings() { + return env; + } + + /** + * Interface for creating a MakeEnvironment, settings its environment values, + * and exposing it in immutable state. + */ + public static class Builder { + private final Map<String, LinkedList<Binding>> env = new HashMap<>(); + private Map<String, String> platformSets = ImmutableMap.<String, String>of("any", MATCH_ANY); + + /** + * Performs an update of Makefile variable 'var' to value 'value' for all + * platforms belonging to the specified 'platformSetRegexp'. Corresponds to + * vardef. We explicitly do not support the various complex nuances of + * Make's assignment operator. + * + * <p>The most recent binding for a particular variable takes precedence, even if + * a more specific binding came earlier. + * + * @param varname the name of the Makefile variable; + * @param value the string value to assign; + * @param platformSetRegexp a set of platforms for which this variable definition + * should take effect. This is expressed as a regexp over gplatform + * strings. + */ + public void update(String varname, String value, String platformSetRegexp) { + if (varname == null || value == null || platformSetRegexp == null) { + throw new NullPointerException(); + } + LinkedList<Binding> bindings = env.get(varname); + if (bindings == null) { + bindings = new LinkedList<Binding>(); + env.put(varname, bindings); + } + // push new bindings onto head of list (=> most recent binding is + // definitive): + bindings.addFirst(new Binding(value, platformSetRegexp)); + } + + /** + * Sets the nickname to regexp mapping for <tt>vardef</tt>. + */ + public void setPlatformSetRegexps(Map<String, String> sets) { + this.platformSets = sets; + } + + @Nullable + public String getPlatformSetRegexp(String nickname) { + return this.platformSets.get(nickname); + } + + /** + * Returns a new MakeEnvironment with environment settings corresponding + * to the cumulative results of this builder's {@link #update} calls. + */ + public MakeEnvironment build() { + Map<String, ImmutableList<Binding>> newMap = new HashMap<>(); + for (Map.Entry<String, LinkedList<Binding>> entry : env.entrySet()) { + newMap.put(entry.getKey(), ImmutableList.copyOf(entry.getValue())); + } + return new MakeEnvironment(ImmutableMap.copyOf(newMap)); + } + } + + @Override + public int hashCode() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean equals(Object that) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return "MakeEnvironment=" + env; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/MethodLibrary.java b/src/main/java/com/google/devtools/build/lib/packages/MethodLibrary.java new file mode 100644 index 0000000..5969697 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/MethodLibrary.java
@@ -0,0 +1,1053 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import static com.google.devtools.build.lib.syntax.SkylarkFunction.cast; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.Type.ConversionException; +import com.google.devtools.build.lib.syntax.AbstractFunction; +import com.google.devtools.build.lib.syntax.AbstractFunction.NoArgFunction; +import com.google.devtools.build.lib.syntax.ClassObject; +import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject; +import com.google.devtools.build.lib.syntax.DotExpression; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.EvalUtils; +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.Function; +import com.google.devtools.build.lib.syntax.MixedModeFunction; +import com.google.devtools.build.lib.syntax.SelectorValue; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; +import com.google.devtools.build.lib.syntax.SkylarkType; +import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ExecutionException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A helper class containing built in functions for the Build and the Build Extension Language. + */ +public class MethodLibrary { + + private MethodLibrary() {} + + // Convert string index in the same way Python does. + // If index is negative, starts from the end. + // If index is outside bounds, it is restricted to the valid range. + private static int getPythonStringIndex(int index, int stringLength) { + if (index < 0) { + index += stringLength; + } + return Math.max(Math.min(index, stringLength), 0); + } + + // Emulate Python substring function + // It converts out of range indices, and never fails + private static String getPythonSubstring(String str, int start, int end) { + start = getPythonStringIndex(start, str.length()); + end = getPythonStringIndex(end, str.length()); + if (start > end) { + return ""; + } else { + return str.substring(start, end); + } + } + + public static int getListIndex(Object key, int listSize, FuncallExpression ast) + throws ConversionException, EvalException { + // Get the nth element in the list + int index = Type.INTEGER.convert(key, "index operand"); + if (index < 0) { + index += listSize; + } + if (index < 0 || index >= listSize) { + throw new EvalException(ast.getLocation(), "List index out of range (index is " + + index + ", but list has " + listSize + " elements)"); + } + return index; + } + + // supported string methods + + @SkylarkBuiltin(name = "join", objectType = StringModule.class, returnType = String.class, + doc = "Returns a string in which the string elements of the argument have been " + + "joined by this string as a separator. Example:<br>" + + "<pre class=language-python>\"|\".join([\"a\", \"b\", \"c\"]) == \"a|b|c\"</pre>", + mandatoryParams = { + @Param(name = "elements", type = SkylarkList.class, doc = "The objects to join.")}) + private static Function join = new MixedModeFunction("join", + ImmutableList.of("this", "elements"), 2, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws ConversionException { + String thiz = Type.STRING.convert(args[0], "'join' operand"); + List<?> seq = Type.OBJECT_LIST.convert(args[1], "'join' argument"); + StringBuilder sb = new StringBuilder(); + for (Iterator<?> i = seq.iterator(); i.hasNext();) { + sb.append(i.next().toString()); + if (i.hasNext()) { + sb.append(thiz); + } + } + return sb.toString(); + } + }; + + @SkylarkBuiltin(name = "lower", objectType = StringModule.class, returnType = String.class, + doc = "Returns the lower case version of this string.") + private static Function lower = new MixedModeFunction("lower", + ImmutableList.of("this"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws ConversionException { + String thiz = Type.STRING.convert(args[0], "'lower' operand"); + return thiz.toLowerCase(); + } + }; + + @SkylarkBuiltin(name = "upper", objectType = StringModule.class, returnType = String.class, + doc = "Returns the upper case version of this string.") + private static Function upper = new MixedModeFunction("upper", + ImmutableList.of("this"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws ConversionException { + String thiz = Type.STRING.convert(args[0], "'upper' operand"); + return thiz.toUpperCase(); + } + }; + + @SkylarkBuiltin(name = "replace", objectType = StringModule.class, returnType = String.class, + doc = "Returns a copy of the string in which the occurrences " + + "of <code>old</code> have been replaced with <code>new</code>, optionally restricting " + + "the number of replacements to <code>maxsplit</code>.", + mandatoryParams = { + @Param(name = "old", type = String.class, doc = "The string to be replaced."), + @Param(name = "new", type = String.class, doc = "The string to replace with.")}, + optionalParams = { + @Param(name = "maxsplit", type = Integer.class, doc = "The maximum number of replacements.")}) + private static Function replace = + new MixedModeFunction("replace", ImmutableList.of("this", "old", "new", "maxsplit"), 3, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws EvalException, + ConversionException { + String thiz = Type.STRING.convert(args[0], "'replace' operand"); + String old = Type.STRING.convert(args[1], "'replace' argument"); + String neww = Type.STRING.convert(args[2], "'replace' argument"); + int maxsplit = + args[3] != null ? Type.INTEGER.convert(args[3], "'replace' argument") + : Integer.MAX_VALUE; + StringBuffer sb = new StringBuffer(); + try { + Matcher m = Pattern.compile(old, Pattern.LITERAL).matcher(thiz); + for (int i = 0; i < maxsplit && m.find(); i++) { + m.appendReplacement(sb, Matcher.quoteReplacement(neww)); + } + m.appendTail(sb); + } catch (IllegalStateException e) { + throw new EvalException(ast.getLocation(), e.getMessage() + " in call to replace"); + } + return sb.toString(); + } + }; + + @SkylarkBuiltin(name = "split", objectType = StringModule.class, returnType = SkylarkList.class, + doc = "Returns a list of all the words in the string, using <code>sep</code> " + + "as the separator, optionally limiting the number of splits to <code>maxsplit</code>.", + optionalParams = { + @Param(name = "sep", type = String.class, + doc = "The string to split on, default is space (\" \")."), + @Param(name = "maxsplit", type = Integer.class, doc = "The maximum number of splits.")}) + private static Function split = new MixedModeFunction("split", + ImmutableList.of("this", "sep", "maxsplit"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast, Environment env) + throws ConversionException { + String thiz = Type.STRING.convert(args[0], "'split' operand"); + String sep = args[1] != null + ? Type.STRING.convert(args[1], "'split' argument") + : " "; + int maxsplit = args[2] != null + ? Type.INTEGER.convert(args[2], "'split' argument") + 1 // last is remainder + : -1; + String[] ss = Pattern.compile(sep, Pattern.LITERAL).split(thiz, + maxsplit); + List<String> result = java.util.Arrays.asList(ss); + return env.isSkylarkEnabled() ? SkylarkList.list(result, String.class) : result; + } + }; + + @SkylarkBuiltin(name = "rfind", objectType = StringModule.class, returnType = Integer.class, + doc = "Returns the last index where <code>sub</code> is found, " + + "or -1 if no such index exists, optionally restricting to " + + "[<code>start</code>:<code>end</code>], " + + "<code>start</code> being inclusive and <code>end</code> being exclusive.", + mandatoryParams = { + @Param(name = "sub", type = String.class, doc = "The substring to find.")}, + optionalParams = { + @Param(name = "start", type = Integer.class, doc = "Restrict to search from this position."), + @Param(name = "end", type = Integer.class, doc = "Restrict to search before this position.")}) + private static Function rfind = + new MixedModeFunction("rfind", ImmutableList.of("this", "sub", "start", "end"), 2, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) + throws ConversionException { + String thiz = Type.STRING.convert(args[0], "'rfind' operand"); + String sub = Type.STRING.convert(args[1], "'rfind' argument"); + int start = 0; + if (args[2] != null) { + start = Type.INTEGER.convert(args[2], "'rfind' argument"); + } + int end = thiz.length(); + if (args[3] != null) { + end = Type.INTEGER.convert(args[3], "'rfind' argument"); + } + int subpos = getPythonSubstring(thiz, start, end).lastIndexOf(sub); + start = getPythonStringIndex(start, thiz.length()); + return subpos < 0 ? subpos : subpos + start; + } + }; + + @SkylarkBuiltin(name = "find", objectType = StringModule.class, returnType = Integer.class, + doc = "Returns the first index where <code>sub</code> is found, " + + "or -1 if no such index exists, optionally restricting to " + + "[<code>start</code>:<code>end]</code>, " + + "<code>start</code> being inclusive and <code>end</code> being exclusive.", + mandatoryParams = { + @Param(name = "sub", type = String.class, doc = "The substring to find.")}, + optionalParams = { + @Param(name = "start", type = Integer.class, doc = "Restrict to search from this position."), + @Param(name = "end", type = Integer.class, doc = "Restrict to search before this position.")}) + private static Function find = + new MixedModeFunction("find", ImmutableList.of("this", "sub", "start", "end"), 2, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) + throws ConversionException { + String thiz = Type.STRING.convert(args[0], "'find' operand"); + String sub = Type.STRING.convert(args[1], "'find' argument"); + int start = 0; + if (args[2] != null) { + start = Type.INTEGER.convert(args[2], "'find' argument"); + } + int end = thiz.length(); + if (args[3] != null) { + end = Type.INTEGER.convert(args[3], "'find' argument"); + } + int subpos = getPythonSubstring(thiz, start, end).indexOf(sub); + start = getPythonStringIndex(start, thiz.length()); + return subpos < 0 ? subpos : subpos + start; + } + }; + + @SkylarkBuiltin(name = "count", objectType = StringModule.class, returnType = Integer.class, + doc = "Returns the number of (non-overlapping) occurrences of substring <code>sub</code> in " + + "string, optionally restricting to [<code>start</code>:<code>end</code>], " + + "<code>start</code> being inclusive and <code>end</code> being exclusive.", + mandatoryParams = { + @Param(name = "sub", type = String.class, doc = "The substring to count.")}, + optionalParams = { + @Param(name = "start", type = Integer.class, doc = "Restrict to search from this position."), + @Param(name = "end", type = Integer.class, doc = "Restrict to search before this position.")}) + private static Function count = + new MixedModeFunction("count", ImmutableList.of("this", "sub", "start", "end"), 2, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) + throws ConversionException { + String thiz = Type.STRING.convert(args[0], "'count' operand"); + String sub = Type.STRING.convert(args[1], "'count' argument"); + int start = 0; + if (args[2] != null) { + start = Type.INTEGER.convert(args[2], "'count' argument"); + } + int end = thiz.length(); + if (args[3] != null) { + end = Type.INTEGER.convert(args[3], "'count' argument"); + } + String str = getPythonSubstring(thiz, start, end); + if (sub.equals("")) { + return str.length() + 1; + } + int count = 0; + int index = -1; + while ((index = str.indexOf(sub)) >= 0) { + count++; + str = str.substring(index + sub.length()); + } + return count; + } + }; + + @SkylarkBuiltin(name = "endswith", objectType = StringModule.class, returnType = Boolean.class, + doc = "Returns True if the string ends with <code>sub</code>, " + + "otherwise False, optionally restricting to [<code>start</code>:<code>end</code>], " + + "<code>start</code> being inclusive and <code>end</code> being exclusive.", + mandatoryParams = { + @Param(name = "sub", type = String.class, doc = "The substring to check.")}, + optionalParams = { + @Param(name = "start", type = Integer.class, doc = "Test beginning at this position."), + @Param(name = "end", type = Integer.class, doc = "Stop comparing at this position.")}) + private static Function endswith = + new MixedModeFunction("endswith", ImmutableList.of("this", "sub", "start", "end"), 2, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) + throws ConversionException { + String thiz = Type.STRING.convert(args[0], "'endswith' operand"); + String sub = Type.STRING.convert(args[1], "'endswith' argument"); + int start = 0; + if (args[2] != null) { + start = Type.INTEGER.convert(args[2], "'endswith' argument"); + } + int end = thiz.length(); + if (args[3] != null) { + end = Type.INTEGER.convert(args[3], ""); + } + + return getPythonSubstring(thiz, start, end).endsWith(sub); + } + }; + + @SkylarkBuiltin(name = "startswith", objectType = StringModule.class, returnType = Boolean.class, + doc = "Returns True if the string starts with <code>sub</code>, " + + "otherwise False, optionally restricting to [<code>start</code>:<code>end</code>], " + + "<code>start</code> being inclusive and <code>end</code> being exclusive.", + mandatoryParams = { + @Param(name = "sub", type = String.class, doc = "The substring to check.")}, + optionalParams = { + @Param(name = "start", type = Integer.class, doc = "Test beginning at this position."), + @Param(name = "end", type = Integer.class, doc = "Stop comparing at this position.")}) + private static Function startswith = + new MixedModeFunction("startswith", ImmutableList.of("this", "sub", "start", "end"), 2, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws ConversionException { + String thiz = Type.STRING.convert(args[0], "'startswith' operand"); + String sub = Type.STRING.convert(args[1], "'startswith' argument"); + int start = 0; + if (args[2] != null) { + start = Type.INTEGER.convert(args[2], "'startswith' argument"); + } + int end = thiz.length(); + if (args[3] != null) { + end = Type.INTEGER.convert(args[3], "'startswith' argument"); + } + return getPythonSubstring(thiz, start, end).startsWith(sub); + } + }; + + // TODO(bazel-team): Maybe support an argument to tell the type of the whitespace. + @SkylarkBuiltin(name = "strip", objectType = StringModule.class, returnType = String.class, + doc = "Returns a copy of the string in which all whitespace characters " + + "have been stripped from the beginning and the end of the string.") + private static Function strip = + new MixedModeFunction("strip", ImmutableList.of("this"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) + throws ConversionException { + String operand = Type.STRING.convert(args[0], "'strip' operand"); + return operand.trim(); + } + }; + + // substring operator + @SkylarkBuiltin(name = "$substring", hidden = true, + doc = "String[<code>start</code>:<code>end</code>] returns a substring.") + private static Function substring = new MixedModeFunction("$substring", + ImmutableList.of("this", "start", "end"), 3, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws ConversionException { + String thiz = Type.STRING.convert(args[0], "substring operand"); + int left = Type.INTEGER.convert(args[1], "substring operand"); + int right = Type.INTEGER.convert(args[2], "substring operand"); + return getPythonSubstring(thiz, left, right); + } + }; + + // supported list methods + @SkylarkBuiltin(name = "append", hidden = true, + doc = "Adds an item to the end of the list.") + private static Function append = new MixedModeFunction("append", + ImmutableList.of("this", "x"), 2, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws EvalException, + ConversionException { + List<Object> thiz = Type.OBJECT_LIST.convert(args[0], "'append' operand"); + thiz.add(args[1]); + return Environment.NONE; + } + }; + + @SkylarkBuiltin(name = "extend", hidden = true, + doc = "Adds all items to the end of the list.") + private static Function extend = new MixedModeFunction("extend", + ImmutableList.of("this", "x"), 2, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws EvalException, + ConversionException { + List<Object> thiz = Type.OBJECT_LIST.convert(args[0], "'extend' operand"); + List<Object> l = Type.OBJECT_LIST.convert(args[1], "'extend' argument"); + thiz.addAll(l); + return Environment.NONE; + } + }; + + // dictionary access operator + @SkylarkBuiltin(name = "$index", hidden = true, + doc = "Returns the nth element of a list or string, " + + "or looks up a value in a dictionary.") + private static Function index = new MixedModeFunction("$index", + ImmutableList.of("this", "index"), 2, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws EvalException, + ConversionException { + Object collectionCandidate = args[0]; + Object key = args[1]; + + if (collectionCandidate instanceof Map<?, ?>) { + Map<?, ?> dictionary = (Map<?, ?>) collectionCandidate; + if (!dictionary.containsKey(key)) { + throw new EvalException(ast.getLocation(), "Key '" + key + "' not found in dictionary"); + } + return dictionary.get(key); + } else if (collectionCandidate instanceof List<?>) { + + List<Object> list = Type.OBJECT_LIST.convert(collectionCandidate, "index operand"); + + if (!list.isEmpty()) { + int index = getListIndex(key, list.size(), ast); + return list.get(index); + } + + throw new EvalException(ast.getLocation(), "List is empty"); + } else if (collectionCandidate instanceof SkylarkList) { + SkylarkList list = (SkylarkList) collectionCandidate; + + if (!list.isEmpty()) { + int index = getListIndex(key, list.size(), ast); + return list.get(index); + } + + throw new EvalException(ast.getLocation(), "List is empty"); + } else if (collectionCandidate instanceof String) { + String str = (String) collectionCandidate; + int index = getListIndex(key, str.length(), ast); + return str.substring(index, index + 1); + + } else { + // TODO(bazel-team): This is dead code, get rid of it. + throw new EvalException(ast.getLocation(), String.format( + "Unsupported datatype (%s) for indexing, only works for dict and list", + EvalUtils.getDatatypeName(collectionCandidate))); + } + } + }; + + @SkylarkBuiltin(name = "values", objectType = DictModule.class, returnType = SkylarkList.class, + doc = "Return the list of values.") + private static Function values = new NoArgFunction("values") { + @Override + public Object call(Object self, FuncallExpression ast, Environment env) + throws EvalException, InterruptedException { + Map<?, ?> dict = (Map<?, ?>) self; + return convert(dict.values(), env, ast.getLocation()); + } + }; + + @SkylarkBuiltin(name = "items", objectType = DictModule.class, returnType = SkylarkList.class, + doc = "Return the list of key-value tuples.") + private static Function items = new NoArgFunction("items") { + @Override + public Object call(Object self, FuncallExpression ast, Environment env) + throws EvalException, InterruptedException { + Map<?, ?> dict = (Map<?, ?>) self; + List<Object> list = Lists.newArrayListWithCapacity(dict.size()); + for (Map.Entry<?, ?> entries : dict.entrySet()) { + List<?> item = ImmutableList.of(entries.getKey(), entries.getValue()); + list.add(env.isSkylarkEnabled() ? SkylarkList.tuple(item) : item); + } + return convert(list, env, ast.getLocation()); + } + }; + + @SkylarkBuiltin(name = "keys", objectType = DictModule.class, returnType = SkylarkList.class, + doc = "Return the list of keys.") + private static Function keys = new NoArgFunction("keys") { + @Override + public Object call(Object self, FuncallExpression ast, Environment env) + throws EvalException, InterruptedException { + Map<?, ?> dict = (Map<?, ?>) self; + return convert(dict.keySet(), env, ast.getLocation()); + } + }; + + @SuppressWarnings("unchecked") + private static Iterable<Object> convert(Collection<?> list, Environment env, Location loc) + throws EvalException { + if (env.isSkylarkEnabled()) { + return SkylarkList.list(list, loc); + } else { + return Lists.newArrayList(list); + } + } + + // unary minus + @SkylarkBuiltin(name = "-", hidden = true, doc = "Unary minus operator.") + private static Function minus = new MixedModeFunction("-", ImmutableList.of("this"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws ConversionException { + int num = Type.INTEGER.convert(args[0], "'unary minus' argument"); + return -num; + } + }; + + @SkylarkBuiltin(name = "list", returnType = SkylarkList.class, + doc = "Converts a collection (e.g. set or dictionary) to a list.", + mandatoryParams = {@Param(name = "x", doc = "The object to convert.")}) + private static Function list = new MixedModeFunction("list", + ImmutableList.of("list"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws EvalException { + Location loc = ast.getLocation(); + return SkylarkList.list(EvalUtils.toCollection(args[0], loc), loc); + } + }; + + @SkylarkBuiltin(name = "len", returnType = Integer.class, doc = + "Returns the length of a string, list, tuple, set, or dictionary.", + mandatoryParams = {@Param(name = "x", doc = "The object to check length of.")}) + private static Function len = new MixedModeFunction("len", + ImmutableList.of("list"), 1, false) { + + @Override + public Object call(Object[] args, FuncallExpression ast) throws EvalException { + Object arg = args[0]; + int l = EvalUtils.size(arg); + if (l == -1) { + throw new EvalException(ast.getLocation(), + EvalUtils.getDatatypeName(arg) + " is not iterable"); + } + return l; + } + }; + + @SkylarkBuiltin(name = "str", returnType = String.class, doc = + "Converts any object to string. This is useful for debugging.", + mandatoryParams = {@Param(name = "x", doc = "The object to convert.")}) + private static Function str = new MixedModeFunction("str", ImmutableList.of("this"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws EvalException { + return EvalUtils.printValue(args[0]); + } + }; + + @SkylarkBuiltin(name = "bool", returnType = Boolean.class, doc = "Converts an object to boolean. " + + "It returns False if the object is None, False, an empty string, the number 0, or an " + + "empty collection. Otherwise, it returns True.", + mandatoryParams = {@Param(name = "x", doc = "The variable to convert.")}) + private static Function bool = new MixedModeFunction("bool", + ImmutableList.of("this"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws EvalException { + return EvalUtils.toBoolean(args[0]); + } + }; + + @SkylarkBuiltin(name = "struct", returnType = SkylarkClassObject.class, doc = + "Creates an immutable struct using the keyword arguments as fields. It is used to group " + + "multiple values together.Example:<br>" + + "<pre class=language-python>s = struct(x = 2, y = 3)\n" + + "return s.x + s.y # returns 5</pre>") + private static Function struct = new AbstractFunction("struct") { + + @Override + public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast, + Environment env) throws EvalException, InterruptedException { + if (args.size() > 0) { + throw new EvalException(ast.getLocation(), "struct only supports keyword arguments"); + } + return new SkylarkClassObject(kwargs, ast.getLocation()); + } + }; + + @SkylarkBuiltin(name = "set", returnType = SkylarkNestedSet.class, + doc = "Creates a set from the <code>items</code>, that supports nesting. " + + "The nesting is applied to other nested sets among <code>items</code>.<br>" + + "Examples:<br>" + + "<pre class=language-python>set([1, set([2, 3]), 2])\n" + + "set([1, 2, 3], order=\"compile\")</pre>", + optionalParams = { + @Param(name = "items", type = SkylarkList.class, + doc = "The items to initialize the set with."), + @Param(name = "order", type = String.class, + doc = "The ordering strategy for the set if it's nested, " + + "possible values are: <code>stable</code> (default), <code>compile</code>, " + + "<code>link</code> or <code>naive_link</code>.")}) + private static final Function set = + new MixedModeFunction("set", ImmutableList.of("items", "order"), 0, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws EvalException, + ConversionException { + Order order; + if (args[1] == null || args[1].equals("stable")) { + order = Order.STABLE_ORDER; + } else if (args[1].equals("compile")) { + order = Order.COMPILE_ORDER; + } else if (args[1].equals("link")) { + order = Order.LINK_ORDER; + } else if (args[1].equals("naive_link")) { + order = Order.NAIVE_LINK_ORDER; + } else { + throw new EvalException(ast.getLocation(), "Invalid order: " + args[1]); + } + + if (args[0] == null) { + return new SkylarkNestedSet(order, SkylarkList.EMPTY_LIST, ast.getLocation()); + } + return new SkylarkNestedSet(order, args[0], ast.getLocation()); + } + }; + + @SkylarkBuiltin(name = "enumerate", returnType = SkylarkList.class, + doc = "Return a list of pairs, with the index (int) and the item from the input list.\n" + + "<pre class=language-python>" + + "enumerate([24, 21, 84]) == [[0, 24], [1, 21], [2, 84]]</pre>\n", + mandatoryParams = { + @Param(name = "list", type = SkylarkList.class, + doc = "input list"), + }) + private static Function enumerate = new MixedModeFunction("enumerate", + ImmutableList.of("list"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws EvalException, + ConversionException { + List<Object> input = Type.OBJECT_LIST.convert(args[0], "'enumerate' operand"); + List<List<Object>> result = Lists.newArrayList(); + int count = 0; + for (Object obj : input) { + result.add(Lists.newArrayList(count, obj)); + count++; + } + return result; + } + }; + + @SkylarkBuiltin(name = "range", returnType = SkylarkList.class, + doc = "Creates a list where items go from <code>start</code> to <end>, using a " + + "<code>step</code> increment. If a single argument is provided, items will " + + "range from 0 to that element." + + "<pre class=language-python>range(4) == [0, 1, 2, 3]\n" + + "range(3, 9, 2) == [3, 5, 7]\n" + + "range(3, 0, -1) == [3, 2, 1]</pre>", + mandatoryParams = { + @Param(name = "start", type = Integer.class, + doc = "Value of the first element"), + }, + optionalParams = { + @Param(name = "end", type = SkylarkList.class, + doc = "Generation of the list stops before <code>end</code> is reached."), + @Param(name = "step", type = String.class, + doc = "The increment (default is 1). It may be negative.")}) + private static final Function range = + new MixedModeFunction("range", ImmutableList.of("start", "stop", "step"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws EvalException, + ConversionException { + int start; + int stop; + if (args[1] == null) { + start = 0; + stop = Type.INTEGER.convert(args[0], "stop"); + } else { + start = Type.INTEGER.convert(args[0], "start"); + stop = Type.INTEGER.convert(args[1], "stop"); + } + int step = args[2] == null ? 1 : Type.INTEGER.convert(args[2], "step"); + if (step == 0) { + throw new EvalException(ast.getLocation(), "step cannot be 0"); + } + List<Integer> result = Lists.newArrayList(); + if (step > 0) { + while (start < stop) { + result.add(start); + start += step; + } + } else { + while (start > stop) { + result.add(start); + start += step; + } + } + return SkylarkList.list(result, Integer.class); + } + }; + + /** + * Returns a function-value implementing "select" (i.e. configurable attributes) + * in the specified package context. + */ + @SkylarkBuiltin(name = "select", + doc = "Creates a SelectorValue from the dict parameter.", + mandatoryParams = {@Param(name = "x", type = Map.class, doc = "The parameter to convert.")}) + private static final Function select = new MixedModeFunction("select", + ImmutableList.of("x"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) + throws EvalException, ConversionException { + Object dict = args[0]; + if (!(dict instanceof Map<?, ?>)) { + throw new EvalException(ast.getLocation(), + "select({...}) argument isn't a dictionary"); + } + return new SelectorValue((Map<?, ?>) dict); + } + }; + + /** + * Returns true if the object has a field of the given name, otherwise false. + */ + @SkylarkBuiltin(name = "hasattr", returnType = Boolean.class, + doc = "Returns True if the object <code>x</code> has a field of the given <code>name</code>, " + + "otherwise False. Example:<br>" + + "<pre class=language-python>hasattr(ctx.attr, \"myattr\")</pre>", + mandatoryParams = { + @Param(name = "object", doc = "The object to check."), + @Param(name = "name", type = String.class, doc = "The name of the field.")}) + private static final Function hasattr = + new MixedModeFunction("hasattr", ImmutableList.of("object", "name"), 2, false) { + + @Override + public Object call(Object[] args, FuncallExpression ast, Environment env) + throws EvalException, ConversionException { + Object obj = args[0]; + String name = cast(args[1], String.class, "name", ast.getLocation()); + + if (obj instanceof ClassObject && ((ClassObject) obj).getValue(name) != null) { + return true; + } + + if (env.getFunctionNames(obj.getClass()).contains(name)) { + return true; + } + + try { + return FuncallExpression.getMethodNames(obj.getClass()).contains(name); + } catch (ExecutionException e) { + // This shouldn't happen + throw new EvalException(ast.getLocation(), e.getMessage()); + } + } + }; + + @SkylarkBuiltin(name = "getattr", + doc = "Returns the struct's field of the given name if exists, otherwise <code>default</code>" + + " if specified, otherwise rasies an error. For example, <code>getattr(x, \"foobar\")" + + "</code> is equivalent to <code>x.foobar</code>." + + "Example:<br>" + + "<pre class=language-python>getattr(ctx.attr, \"myattr\")\n" + + "getattr(ctx.attr, \"myattr\", \"mydefault\")</pre>", + mandatoryParams = { + @Param(name = "object", doc = "The struct which's field is accessed."), + @Param(name = "name", doc = "The name of the struct field.")}, + optionalParams = { + @Param(name = "default", doc = "The default value to return in case the struct " + + "doesn't have a field of the given name.")}) + private static final Function getattr = new MixedModeFunction( + "getattr", ImmutableList.of("object", "name", "default"), 2, false) { + @Override + public Object call(Object[] args, FuncallExpression ast, Environment env) + throws EvalException { + Object obj = args[0]; + String name = cast(args[1], String.class, "name", ast.getLocation()); + Object result = DotExpression.eval(obj, name, ast.getLocation()); + if (result == null) { + if (args[2] != null) { + return args[2]; + } else { + throw new EvalException(ast.getLocation(), "Object of type '" + + EvalUtils.getDatatypeName(obj) + "' has no field '" + name + "'"); + } + } + return result; + } + }; + + @SkylarkBuiltin(name = "dir", returnType = SkylarkList.class, + doc = "Returns the list of the names (list of strings) of the fields and " + + "methods of the parameter object.", + mandatoryParams = {@Param(name = "object", doc = "The object to check.")}) + private static final Function dir = new MixedModeFunction( + "dir", ImmutableList.of("object"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast, Environment env) + throws EvalException { + Object obj = args[0]; + // Order the fields alphabetically. + Set<String> fields = new TreeSet<>(); + if (obj instanceof ClassObject) { + fields.addAll(((ClassObject) obj).getKeys()); + } + fields.addAll(env.getFunctionNames(obj.getClass())); + try { + fields.addAll(FuncallExpression.getMethodNames(obj.getClass())); + } catch (ExecutionException e) { + // This shouldn't happen + throw new EvalException(ast.getLocation(), e.getMessage()); + } + return SkylarkList.list(fields, String.class); + } + }; + + @SkylarkBuiltin(name = "type", returnType = String.class, + doc = "Returns the type name of its argument.", + mandatoryParams = {@Param(name = "object", doc = "The object to check type of.")}) + private static final Function type = new MixedModeFunction("type", + ImmutableList.of("object"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) throws EvalException { + // There is no 'type' type in Skylark, so we return a string with the type name. + return EvalUtils.getDatatypeName(args[0]); + } + }; + + @SkylarkBuiltin(name = "fail", + doc = "Raises an error (the execution stops), except if the <code>when</code> condition " + + "is False.", + returnType = Environment.NoneType.class, + mandatoryParams = { + @Param(name = "msg", type = String.class, doc = "Error message to display for the user")}, + optionalParams = { + @Param(name = "attr", type = String.class, + doc = "The name of the attribute that caused the error"), + @Param(name = "when", type = Boolean.class, + doc = "When False, the function does nothing. Default is True.")}) + private static final Function fail = new MixedModeFunction( + "fail", ImmutableList.of("msg", "attr", "when"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast, Environment env) + throws EvalException { + if (args[2] != null) { + if (!EvalUtils.toBoolean(args[2])) { + return Environment.NONE; + } + } + String msg = cast(args[0], String.class, "msg", ast.getLocation()); + if (args[1] != null) { + msg = "attribute " + cast(args[1], String.class, "attr", ast.getLocation()) + + ": " + msg; + } + throw new EvalException(ast.getLocation(), msg); + } + }; + + @SkylarkBuiltin(name = "print", returnType = Environment.NoneType.class, + doc = "Prints <code>msg</code> to the console.", + mandatoryParams = { + @Param(name = "*args", doc = "The objects to print.")}, + optionalParams = { + @Param(name = "sep", type = String.class, + doc = "The separator string between the objects, default is space (\" \").")}) + private static final Function print = new AbstractFunction("print") { + @Override + public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast, + Environment env) throws EvalException, InterruptedException { + String sep = " "; + if (kwargs.containsKey("sep")) { + sep = cast(kwargs.remove("sep"), String.class, "sep", ast.getLocation()); + } + if (kwargs.size() > 0) { + throw new EvalException(ast.getLocation(), + "unexpected keywords: '" + kwargs.keySet() + "'"); + } + String msg = Joiner.on(sep).join(Iterables.transform(args, + new com.google.common.base.Function<Object, String>() { + @Override + public String apply(Object input) { + return EvalUtils.printValue(input); + } + })); + ((SkylarkEnvironment) env).handleEvent(Event.warn(ast.getLocation(), msg)); + return Environment.NONE; + } + }; + + /** + * Skylark String module. + */ + @SkylarkModule(name = "string", doc = + "A language built-in type to support strings. " + + "Example of string literals:<br>" + + "<pre class=language-python>a = 'abc\\ndef'\n" + + "b = \"ab'cd\"\n" + + "c = \"\"\"multiline string\"\"\"</pre>" + + "Strings are iterable and support the <code>in</code> operator. Examples:<br>" + + "<pre class=language-python>\"a\" in \"abc\" # evaluates as True\n" + + "l = []\n" + + "for s in \"abc\":\n" + + " l += [s] # l == [\"a\", \"b\", \"c\"]</pre>") + public static final class StringModule {} + + /** + * Skylark Dict module. + */ + @SkylarkModule(name = "dict", doc = + "A language built-in type to support dicts. " + + "Example of dict literal:<br>" + + "<pre class=language-python>d = {\"a\": 2, \"b\": 5}</pre>" + + "Accessing elements works just like in Python:<br>" + + "<pre class=language-python>e = d[\"a\"] # e == 2</pre>" + + "Dicts support the <code>+</code> operator to concatenate two dicts. In case of multiple " + + "keys the second one overrides the first one. Examples:<br>" + + "<pre class=language-python>" + + "d = {\"a\" : 1} + {\"b\" : 2} # d == {\"a\" : 1, \"b\" : 2}\n" + + "d += {\"c\" : 3} # d == {\"a\" : 1, \"b\" : 2, \"c\" : 3}\n" + + "d = d + {\"c\" : 5} # d == {\"a\" : 1, \"b\" : 2, \"c\" : 5}</pre>" + + "Since the language doesn't have mutable objects <code>d[\"a\"] = 5</code> automatically " + + "translates to <code>d = d + {\"a\" : 5}</code>.<br>" + + "Dicts are iterable, the iteration works on their keyset.<br>" + + "Dicts support the <code>in</code> operator, testing membership in the keyset of the dict. " + + "Example:<br>" + + "<pre class=language-python>\"a\" in {\"a\" : 2, \"b\" : 5} # evaluates as True</pre>") + public static final class DictModule {} + + public static final Map<Function, SkylarkType> stringFunctions = ImmutableMap + .<Function, SkylarkType>builder() + .put(join, SkylarkType.STRING) + .put(lower, SkylarkType.STRING) + .put(upper, SkylarkType.STRING) + .put(replace, SkylarkType.STRING) + .put(split, SkylarkType.of(List.class, String.class)) + .put(rfind, SkylarkType.INT) + .put(find, SkylarkType.INT) + .put(endswith, SkylarkType.BOOL) + .put(startswith, SkylarkType.BOOL) + .put(strip, SkylarkType.STRING) + .put(substring, SkylarkType.STRING) + .put(count, SkylarkType.INT) + .build(); + + public static final List<Function> listFunctions = ImmutableList + .<Function>builder() + .add(append) + .add(extend) + .build(); + + public static final Map<Function, SkylarkType> dictFunctions = ImmutableMap + .<Function, SkylarkType>builder() + .put(items, SkylarkType.of(List.class)) + .put(keys, SkylarkType.of(Set.class)) + .put(values, SkylarkType.of(List.class)) + .build(); + + private static final Map<Function, SkylarkType> pureGlobalFunctions = ImmutableMap + .<Function, SkylarkType>builder() + // TODO(bazel-team): String methods are added two times, because there are + // a lot of cases when they are used as global functions in the depot. Those + // should be cleaned up first. + .put(minus, SkylarkType.INT) + .put(select, SkylarkType.of(SelectorValue.class)) + .put(len, SkylarkType.INT) + .put(str, SkylarkType.STRING) + .put(bool, SkylarkType.BOOL) + .build(); + + private static final Map<Function, SkylarkType> skylarkGlobalFunctions = ImmutableMap + .<Function, SkylarkType>builder() + .putAll(pureGlobalFunctions) + .put(list, SkylarkType.of(SkylarkList.class)) + .put(struct, SkylarkType.of(ClassObject.class)) + .put(hasattr, SkylarkType.BOOL) + .put(getattr, SkylarkType.UNKNOWN) + .put(set, SkylarkType.of(SkylarkNestedSet.class)) + .put(dir, SkylarkType.of(SkylarkList.class, String.class)) + .put(enumerate, SkylarkType.of(SkylarkList.class)) + .put(range, SkylarkType.of(SkylarkList.class, Integer.class)) + .put(type, SkylarkType.of(String.class)) + .put(fail, SkylarkType.NONE) + .put(print, SkylarkType.NONE) + .build(); + + /** + * Set up a given environment for supported class methods. + */ + public static void setupMethodEnvironment(Environment env) { + env.registerFunction(Map.class, index.getName(), index); + setupMethodEnvironment(env, Map.class, dictFunctions.keySet()); + env.registerFunction(String.class, index.getName(), index); + setupMethodEnvironment(env, String.class, stringFunctions.keySet()); + if (env.isSkylarkEnabled()) { + env.registerFunction(SkylarkList.class, index.getName(), index); + setupMethodEnvironment(env, skylarkGlobalFunctions.keySet()); + } else { + env.registerFunction(List.class, index.getName(), index); + env.registerFunction(ImmutableList.class, index.getName(), index); + // TODO(bazel-team): listFunctions are not allowed in Skylark extensions (use += instead). + // It is allowed in BUILD files only for backward-compatibility. + setupMethodEnvironment(env, List.class, listFunctions); + setupMethodEnvironment(env, stringFunctions.keySet()); + setupMethodEnvironment(env, pureGlobalFunctions.keySet()); + } + } + + private static void setupMethodEnvironment( + Environment env, Class<?> nameSpace, Iterable<Function> functions) { + for (Function function : functions) { + env.registerFunction(nameSpace, function.getName(), function); + } + } + + private static void setupMethodEnvironment(Environment env, Iterable<Function> functions) { + for (Function function : functions) { + env.update(function.getName(), function); + } + } + + private static void setupValidationEnvironment( + Map<Function, SkylarkType> functions, Map<String, SkylarkType> result) { + for (Map.Entry<Function, SkylarkType> function : functions.entrySet()) { + String name = function.getKey().getName(); + result.put(name, SkylarkFunctionType.of(name, function.getValue())); + } + } + + public static void setupValidationEnvironment( + Map<SkylarkType, Map<String, SkylarkType>> builtIn) { + Map<String, SkylarkType> global = builtIn.get(SkylarkType.GLOBAL); + setupValidationEnvironment(skylarkGlobalFunctions, global); + + Map<String, SkylarkType> dict = new HashMap<>(); + setupValidationEnvironment(dictFunctions, dict); + builtIn.put(SkylarkType.of(Map.class), dict); + + Map<String, SkylarkType> string = new HashMap<>(); + setupValidationEnvironment(stringFunctions, string); + builtIn.put(SkylarkType.STRING, string); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/NoSuchPackageException.java b/src/main/java/com/google/devtools/build/lib/packages/NoSuchPackageException.java new file mode 100644 index 0000000..720d3d5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/NoSuchPackageException.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import javax.annotation.Nullable; + +/** + * Exception indicating an attempt to access a package which is not found, does + * not exist, or can't be parsed into a package. + */ +public abstract class NoSuchPackageException extends NoSuchThingException { + + private final String packageName; + + public NoSuchPackageException(String packageName, String message) { + this(packageName, "no such package", message); + } + + public NoSuchPackageException(String packageName, String message, + Throwable cause) { + this(packageName, "no such package", message, cause); + } + + protected NoSuchPackageException(String packageName, String messagePrefix, String message) { + super(messagePrefix + " '" + packageName + "': " + message); + this.packageName = packageName; + } + + protected NoSuchPackageException(String packageName, String messagePrefix, String message, + Throwable cause) { + super(messagePrefix + " '" + packageName + "': " + message, cause); + this.packageName = packageName; + } + + public String getPackageName() { + return packageName; + } + + /** + * Return the package if parsing completed enough to construct it. May return null. + */ + @Nullable + public Package getPackage() { + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/NoSuchTargetException.java b/src/main/java/com/google/devtools/build/lib/packages/NoSuchTargetException.java new file mode 100644 index 0000000..fa180fa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/NoSuchTargetException.java
@@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.devtools.build.lib.syntax.Label; + +import javax.annotation.Nullable; + +/** + * Exception indicating an attempt to access a target which is not found or does + * not exist. + */ +public class NoSuchTargetException extends NoSuchThingException { + + @Nullable private final Label label; + // TODO(bazel-team): rename/refactor this class and NoSuchPackageException since it's confusing + // that they embed Target/Package instances. + @Nullable private final Target target; + private final boolean packageLoadedSuccessfully; + + public NoSuchTargetException(String message) { + this(null, message); + } + + public NoSuchTargetException(@Nullable Label label, String message) { + this((label != null ? "no such target '" + label + "': " : "") + message, label, null, null); + } + + public NoSuchTargetException(Target targetInError, NoSuchPackageException nspe) { + this(String.format("Target '%s' contains an error and its package is in error", + targetInError.getLabel()), targetInError.getLabel(), targetInError, nspe); + } + + private NoSuchTargetException(String message, @Nullable Label label, @Nullable Target target, + @Nullable NoSuchPackageException nspe) { + super(message, nspe); + this.label = label; + this.target = target; + this.packageLoadedSuccessfully = nspe != null ? false : true; + } + + @Nullable + public Label getLabel() { + return label; + } + + /** + * Return the target (in error) if parsing completed enough to construct it. May return null. + */ + @Nullable + public Target getTarget() { + return target; + } + + public boolean getPackageLoadedSuccessfully() { + return packageLoadedSuccessfully; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/NoSuchThingException.java b/src/main/java/com/google/devtools/build/lib/packages/NoSuchThingException.java new file mode 100644 index 0000000..49e703e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/NoSuchThingException.java
@@ -0,0 +1,31 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +/** + * Exception indicating an attempt to access something which is not found or + * does not exist. + */ +public class NoSuchThingException extends Exception { + + public NoSuchThingException(String message) { + super(message); + } + + public NoSuchThingException(String message, Throwable cause) { + super(message, cause); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/NonconfigurableAttributeMapper.java b/src/main/java/com/google/devtools/build/lib/packages/NonconfigurableAttributeMapper.java new file mode 100644 index 0000000..d54c847 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/NonconfigurableAttributeMapper.java
@@ -0,0 +1,56 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +/** + * {@link AttributeMap} implementation that triggers an {@link IllegalStateException} if called + * on any attribute that supports configurable values, as determined by + * {@link Attribute#isConfigurable()}. + * + * <p>This is particularly useful for logic that doesn't have access to configurations - it + * protects against undefined behavior in response to unexpected configuration-dependent inputs. + */ +public class NonconfigurableAttributeMapper extends AbstractAttributeMapper { + private NonconfigurableAttributeMapper(Rule rule) { + super(rule.getPackage(), rule.getRuleClassObject(), rule.getLabel(), + rule.getAttributeContainer()); + } + + /** + * Example usage: + * + * <pre> + * Label fooLabel = NonconfigurableAttributeMapper.of(rule).get("foo", Type.LABEL); + * </pre> + */ + public static NonconfigurableAttributeMapper of (Rule rule) { + return new NonconfigurableAttributeMapper(rule); + } + + @Override + public <T> T get(String attributeName, Type<T> type) { + Preconditions.checkState(!getAttributeDefinition(attributeName).isConfigurable(), + "Attribute '" + attributeName + "' is potentially configurable - not allowed here"); + return super.get(attributeName, type); + } + + @Override + protected <T> Iterable<T> visitAttribute(String attributeName, Type<T> type) { + T value = get(attributeName, type); + return value == null ? ImmutableList.<T>of() : ImmutableList.of(value); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/OutputFile.java b/src/main/java/com/google/devtools/build/lib/packages/OutputFile.java new file mode 100644 index 0000000..9c91afb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/OutputFile.java
@@ -0,0 +1,67 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Label; + +/** + * A generated file that is the output of a rule. + */ +public final class OutputFile extends FileTarget { + + private final Rule generatingRule; + + /** + * Constructs an output file with the given label, which must be in the given + * package. + */ + OutputFile(Package pkg, Label label, Rule generatingRule) { + super(pkg, label); + this.generatingRule = generatingRule; + } + + @Override + public RuleVisibility getVisibility() { + return generatingRule.getVisibility(); + } + + /** + * Returns the rule which generates this output file. + */ + public Rule getGeneratingRule() { + return generatingRule; + } + + @Override + public String getTargetKind() { + return "generated file"; + } + + @Override + public Rule getAssociatedRule() { + return getGeneratingRule(); + } + + @Override + public Location getLocation() { + return generatingRule.getLocation(); + } + + @Override + public int hashCode() { + return label.hashCode(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/Package.java b/src/main/java/com/google/devtools/build/lib/packages/Package.java new file mode 100644 index 0000000..a2216e0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/Package.java
@@ -0,0 +1,1516 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.collect.ImmutableSortedKeyMap; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.AttributeMap.AcceptsLabelAttribute; +import com.google.devtools.build.lib.packages.License.DistributionType; +import com.google.devtools.build.lib.packages.PackageDeserializer.PackageDeserializationException; +import com.google.devtools.build.lib.packages.PackageFactory.Globber; + +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.Canonicalizer; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.PrintStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * A package, which is a container of {@link Rule}s, each of + * which contains a dictionary of named attributes. + * + * <p>Package instances are intended to be immutable and for all practical + * purposes can be treated as such. Note, however, that some member variables + * exposed via the public interface are not strictly immutable, so until their + * types are guaranteed immutable we're not applying the {@code @Immutable} + * annotation here. + */ +public class Package implements Serializable { + + /** + * Common superclass for all name-conflict exceptions. + */ + public static class NameConflictException extends Exception { + protected NameConflictException(String message) { + super(message); + } + } + + /** + * The repository identifier for this package. + */ + private final PackageIdentifier packageIdentifier; + + /** + * The name of the package, e.g. "foo/bar". + */ + protected final String name; + + /** + * Like name, but in the form of a PathFragment. + */ + private final PathFragment nameFragment; + + /** + * The filename of this package's BUILD file. + */ + protected Path filename; + + /** + * The directory in which this package's BUILD file resides. All InputFile + * members of the packages are located relative to this directory. + */ + private Path packageDirectory; + + /** + * The root of the source tree in which this package was found. It is an invariant that + * {@code sourceRoot.getRelative(name).equals(packageDirectory)}. + */ + private Path sourceRoot; + + /** + * The "Make" environment of this package, containing package-local + * definitions of "Make" variables. + */ + private MakeEnvironment makeEnv; + + /** + * The collection of all targets defined in this package, indexed by name. + */ + protected Map<String, Target> targets; + + /** + * Default visibility for rules that do not specify it. null is interpreted + * as VISIBILITY_PRIVATE. + */ + private RuleVisibility defaultVisibility; + private boolean defaultVisibilitySet; + + /** + * Default package-level 'obsolete' value for rules that do not specify it. + */ + private boolean defaultObsolete = false; + + /** + * Default package-level 'testonly' value for rules that do not specify it. + */ + private boolean defaultTestOnly = false; + + /** + * Default package-level 'deprecation' value for rules that do not specify it. + */ + private String defaultDeprecation; + + /** + * Default header strictness checking for rules that do not specify it. + */ + private String defaultHdrsCheck; + + /** + * Default copts for cc_* rules. The rules' individual copts will append to + * this value. + */ + private ImmutableList<String> defaultCopts; + + /** + * The InputFile target corresponding to this package's BUILD file. + */ + private InputFile buildFile; + + /** + * True iff this package's BUILD files contained lexical or grammatical + * errors, or experienced errors during evaluation, or semantic errors during + * the construction of any rule. + * + * <p>Note: A package containing errors does not necessarily prevent a build; + * if all the rules needed for a given build were constructed prior to the + * first error, the build may proceed. + */ + private boolean containsErrors; + + /** + * True iff this package contains errors that were caused by temporary conditions (e.g. an I/O + * error). If this is true, {@link #containsErrors} is also true. + */ + private boolean containsTemporaryErrors; + + /** + * The set of labels subincluded by this package. + */ + private Set<Label> subincludes; + + /** + * The list of transitive closure of the Skylark file dependencies. + */ + private ImmutableList<Label> skylarkFileDependencies; + + /** + * The package's default "licenses" and "distribs" attributes, as specified + * in calls to licenses() and distribs() in the BUILD file. + */ + // These sets contain the values specified by the most recent licenses() or + // distribs() declarations encountered during package parsing: + private License defaultLicense; + private Set<License.DistributionType> defaultDistributionSet; + + + /** + * The names of the package() attributes that declare default values for rule + * {@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR} and {@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR} + * values when not explicitly specified. + */ + public static final String DEFAULT_COMPATIBLE_WITH_ATTRIBUTE = "default_compatible_with"; + public static final String DEFAULT_RESTRICTED_TO_ATTRIBUTE = "default_restricted_to"; + + private Set<Label> defaultCompatibleWith = ImmutableSet.of(); + private Set<Label> defaultRestrictedTo = ImmutableSet.of(); + + private ImmutableSet<String> features; + + private ImmutableList<Event> events; + + // Hack to avoid having to copy every attribute. See #readObject and #readResolve. + // This will always be null for externally observable instances. + private Package deserializedPkg = null; + + /** + * Package initialization, part 1 of 3: instantiates a new package with the + * given name. + * + * <p>As part of initialization, {@link Builder} constructs {@link InputFile} + * and {@link PackageGroup} instances that require a valid Package instance where + * {@link Package#getNameFragment()} is accessible. That's why these settings are + * applied here at the start. + * + * @precondition {@code name} must be a suffix of + * {@code filename.getParentDirectory())}. + */ + protected Package(PackageIdentifier packageId) { + this.packageIdentifier = packageId; + this.nameFragment = Canonicalizer.fragments().intern(packageId.getPackageFragment()); + this.name = nameFragment.getPathString(); + } + + private void writeObject(ObjectOutputStream out) { + com.google.devtools.build.lib.query2.proto.proto2api.Build.Package pb = + PackageSerializer.serializePackage(this); + try { + pb.writeDelimitedTo(out); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private void readObject(ObjectInputStream in) throws IOException { + com.google.devtools.build.lib.query2.proto.proto2api.Build.Package pb = + com.google.devtools.build.lib.query2.proto.proto2api.Build.Package.parseDelimitedFrom(in); + Package pkg; + try { + pkg = new PackageDeserializer(null, null).deserialize(pb); + } catch (PackageDeserializationException e) { + throw new IllegalStateException(e); + } + deserializedPkg = pkg; + } + + protected Object readResolve() { + // This method needs to be protected so serialization works for subclasses. + return deserializedPkg; + } + + // See: http://docs.oracle.com/javase/6/docs/platform/serialization/spec/input.html#6053 + @SuppressWarnings("unused") + private void readObjectNoData() { + throw new IllegalStateException(); + } + + /** Returns this packages' identifier. */ + public PackageIdentifier getPackageIdentifier() { + return packageIdentifier; + } + + /** + * Package initialization: part 2 of 3: sets this package's default header + * strictness checking. + * + * <p>This is needed to support C++-related rule classes + * which accesses {@link #getDefaultHdrsCheck} from the still-under-construction + * package. + */ + protected void setDefaultHdrsCheck(String defaultHdrsCheck) { + this.defaultHdrsCheck = defaultHdrsCheck; + } + + /** + * Set the default 'obsolete' value for this package. + */ + protected void setDefaultObsolete(boolean obsolete) { + defaultObsolete = obsolete; + } + + /** + * Set the default 'testonly' value for this package. + */ + protected void setDefaultTestOnly(boolean testOnly) { + defaultTestOnly = testOnly; + } + + /** + * Set the default 'deprecation' value for this package. + */ + protected void setDefaultDeprecation(String deprecation) { + defaultDeprecation = deprecation; + } + + /** + * Sets the default value to use for a rule's {@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR} + * attribute when not explicitly specified by the rule. + */ + protected void setDefaultCompatibleWith(Set<Label> environments) { + defaultCompatibleWith = environments; + } + + /** + * Sets the default value to use for a rule's {@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR} + * attribute when not explicitly specified by the rule. + */ + protected void setDefaultRestrictedTo(Set<Label> environments) { + defaultRestrictedTo = environments; + } + + public static Path getSourceRoot(Path buildFile, PathFragment nameFragment) { + Path current = buildFile.getParentDirectory(); + for (int i = 0, len = nameFragment.segmentCount(); i < len && current != null; i++) { + current = current.getParentDirectory(); + } + return current; + } + + /** + * Package initialization: part 3 of 3: applies all other settings and completes + * initialization of the package. + * + * <p>Only after this method is called can this package be considered "complete" + * and be shared publicly. + */ + protected void finishInit(AbstractBuilder<?, ?> builder) { + // If any error occurred during evaluation of this package, consider all + // rules in the package to be "in error" also (even if they were evaluated + // prior to the error). This behaviour is arguably stricter than need be, + // but stopping a build only for some errors but not others creates user + // confusion. + if (builder.containsErrors) { + for (Rule rule : builder.getTargets(Rule.class)) { + rule.setContainsErrors(); + } + } + this.filename = builder.filename; + this.packageDirectory = filename.getParentDirectory(); + + this.sourceRoot = getSourceRoot(filename, nameFragment); + if ((sourceRoot == null + || !sourceRoot.getRelative(nameFragment).equals(packageDirectory)) + && !filename.getBaseName().equals("WORKSPACE")) { + throw new IllegalArgumentException( + "Invalid BUILD file name for package '" + name + "': " + filename); + } + + this.makeEnv = builder.makeEnv.build(); + this.targets = ImmutableSortedKeyMap.copyOf(builder.targets); + this.defaultVisibility = builder.defaultVisibility; + this.defaultVisibilitySet = builder.defaultVisibilitySet; + if (builder.defaultCopts == null) { + this.defaultCopts = ImmutableList.of(); + } else { + this.defaultCopts = ImmutableList.copyOf(builder.defaultCopts); + } + this.buildFile = builder.buildFile; + this.containsErrors = builder.containsErrors; + this.containsTemporaryErrors = builder.containsTemporaryErrors; + this.subincludes = builder.subincludes.keySet(); + this.skylarkFileDependencies = builder.skylarkFileDependencies; + this.defaultLicense = builder.defaultLicense; + this.defaultDistributionSet = builder.defaultDistributionSet; + this.features = ImmutableSortedSet.copyOf(builder.features); + this.events = ImmutableList.copyOf(builder.events); + } + + /** + * Returns the list of subincluded labels on which the validity of this package depends. + */ + public Set<Label> getSubincludeLabels() { + return subincludes; + } + + /** + * Returns the list of transitive closure of the Skylark file dependencies of this package. + */ + public ImmutableList<Label> getSkylarkFileDependencies() { + return skylarkFileDependencies; + } + + /** + * Returns the filename of the BUILD file which defines this package. The + * parent directory of the BUILD file is the package directory. + */ + public Path getFilename() { + return filename; + } + + /** + * Returns the source root (a directory) beneath which this package's BUILD file was found. + * + * Assumes invariant: + * {@code getSourceRoot().getRelative(getName()).equals(getPackageDirectory())} + */ + public Path getSourceRoot() { + return sourceRoot; + } + + /** + * Returns the directory containing the package's BUILD file. + */ + public Path getPackageDirectory() { + return packageDirectory; + } + + /** + * Returns the name of this package. If this build is using external repositories then this name + * may not be unique! + */ + public String getName() { + return name; + } + + /** + * Like {@link #getName}, but has type {@code PathFragment}. + */ + public PathFragment getNameFragment() { + return nameFragment; + } + + /** + * Returns the "Make" value from the package's make environment whose name + * is "varname", or null iff the variable is not defined in the environment. + */ + public String lookupMakeVariable(String varname, String platform) { + return makeEnv.lookup(varname, platform); + } + + /** + * Returns the make environment. This should only ever be used for serialization -- how the + * make variables are implemented is an implementation detail. + */ + MakeEnvironment getMakeEnvironment() { + return makeEnv; + } + + /** + * Returns the label of this package's BUILD file. + * + * Typically <code>getBuildFileLabel().getName().equals("BUILD")</code> -- + * though not necessarily: data in a subdirectory of a test package may use a + * different filename to avoid inadvertently creating a new package. + */ + Label getBuildFileLabel() { + return buildFile.getLabel(); + } + + /** + * Returns the InputFile target for this package's BUILD file. + */ + public InputFile getBuildFile() { + return buildFile; + } + + /** + * Returns true if errors were encountered during evaluation of this package. + * (The package may be incomplete and its contents should not be relied upon + * for critical operations. However, any Rules belonging to the package are + * guaranteed to be intact, unless their <code>containsErrors()</code> flag + * is set.) + */ + public boolean containsErrors() { + return containsErrors; + } + + /** + * True iff this package contains errors that were caused by temporary conditions (e.g. an I/O + * error). If this is true, {@link #containsErrors()} also returns true. + */ + public boolean containsTemporaryErrors() { + return containsTemporaryErrors; + } + + public List<Event> getEvents() { + return events; + } + + /** + * Returns an (immutable, unordered) view of all the targets belonging to this package. + */ + public Collection<Target> getTargets() { + return getTargets(targets); + } + + /** + * Common getTargets implementation, accessible by both {@link Package} and + * {@link Package.AbstractBuilder}. + */ + private static Collection<Target> getTargets(Map<String, Target> targetMap) { + return Collections.unmodifiableCollection(targetMap.values()); + } + + /** + * Returns a (read-only, unordered) iterator of all the targets belonging + * to this package which are instances of the specified class. + */ + public <T extends Target> Iterable<T> getTargets(Class<T> targetClass) { + return getTargets(targets, targetClass); + } + + /** + * Common getTargets implementation, accessible by both {@link Package} and + * {@link Package.AbstractBuilder}. + */ + private static <T extends Target> Iterable<T> getTargets(Map<String, Target> targetMap, + Class<T> targetClass) { + return Iterables.filter(targetMap.values(), targetClass); + } + + /** + * Returns a (read-only, unordered) iterator over the rules in this package. + */ + @VisibleForTesting // Legacy. Production code should use getTargets(Class) instead + Iterable<? extends Rule> getRules() { + return getTargets(Rule.class); + } + + /** + * Returns a (read-only, unordered) iterator over the files in this package. + */ + @VisibleForTesting // Legacy. Production code should use getTargets(Class) instead + Iterable<? extends FileTarget> getFiles() { + return getTargets(FileTarget.class); + } + + /** + * Returns the rule that corresponds to a particular BUILD target name. Useful + * for walking through the dependency graph of a target. + * Fails if the target is not a Rule. + */ + @VisibleForTesting + Rule getRule(String targetName) { + return (Rule) targets.get(targetName); + } + + /** + * Returns the features specified in the <code>package()</code> declaration. + */ + public ImmutableSet<String> getFeatures() { + return features; + } + + /** + * Returns the target (a member of this package) whose name is "targetName". + * First rules are searched, then output files, then input files. The target + * name must be valid, as defined by {@code LabelValidator#validateTargetName}. + * + * @throws NoSuchTargetException if the specified target was not found. + */ + public Target getTarget(String targetName) throws NoSuchTargetException { + Target target = targets.get(targetName); + if (target != null) { + return target; + } + + // No such target. + + // If there's a file on the disk that's not mentioned in the BUILD file, + // produce a more informative error. NOTE! this code path is only executed + // on failure, which is (relatively) very rare. In the common case no + // stat(2) is executed. + Path filename = getPackageDirectory().getRelative(targetName); + String suffix; + if (!new PathFragment(targetName).isNormalized()) { + // Don't check for file existence in this case because the error message + // would be confusing and wrong. If the targetName is "foo/bar/.", and + // there is a directory "foo/bar", it doesn't mean that "//pkg:foo/bar/." + // is a valid label. + suffix = ""; + } else if (filename.isDirectory()) { + suffix = "; however, a source directory of this name exists. (Perhaps add " + + "'exports_files([\"" + targetName + "\"])' to " + name + "/BUILD, or define a " + + "filegroup?)"; + } else if (filename.exists()) { + suffix = "; however, a source file of this name exists. (Perhaps add " + + "'exports_files([\"" + targetName + "\"])' to " + name + "/BUILD?)"; + } else { + suffix = ""; + } + + try { + throw new NoSuchTargetException(createLabel(targetName), "target '" + targetName + + "' not declared in package '" + name + "'" + suffix + " defined by " + + this.filename); + } catch (Label.SyntaxException e) { + throw new IllegalArgumentException(targetName); + } + } + + /** + * Creates a label for a target inside this package. + * + * @throws SyntaxException if the {@code targetName} is invalid + */ + public Label createLabel(String targetName) throws SyntaxException { + return Label.create(packageIdentifier, targetName); + } + + /** + * Returns the default visibility for this package. + */ + public RuleVisibility getDefaultVisibility() { + if (defaultVisibility != null) { + return defaultVisibility; + } else { + return ConstantRuleVisibility.PRIVATE; + } + } + + /** + * Returns the default obsolete value. + */ + public Boolean getDefaultObsolete() { + return defaultObsolete; + } + + /** + * Returns the default testonly value. + */ + public Boolean getDefaultTestOnly() { + return defaultTestOnly; + } + + /** + * Returns the default obsolete value. + */ + public String getDefaultDeprecation() { + return defaultDeprecation; + } + + /** + * Gets the default header checking mode. + */ + public String getDefaultHdrsCheck() { + return defaultHdrsCheck != null ? defaultHdrsCheck : "loose"; + } + + /** + * Returns the default copts value, to which rules should append their + * specific copts. + */ + public ImmutableList<String> getDefaultCopts() { + return defaultCopts; + } + + /** + * Returns whether the default header checking mode has been set or it is the + * default value. + */ + public boolean isDefaultHdrsCheckSet() { + return defaultHdrsCheck != null; + } + + public boolean isDefaultVisibilitySet() { + return defaultVisibilitySet; + } + + /** + * Gets the parsed license object for the default license + * declared by this package. + */ + public License getDefaultLicense() { + return defaultLicense; + } + + /** + * Returns the parsed set of distributions declared as the default for this + * package. + */ + public Set<License.DistributionType> getDefaultDistribs() { + return defaultDistributionSet; + } + + /** + * Returns the default value to use for a rule's {@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR} + * attribute when not explicitly specified by the rule. + */ + public Set<Label> getDefaultCompatibleWith() { + return defaultCompatibleWith; + } + + /** + * Returns the default value to use for a rule's {@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR} + * attribute when not explicitly specified by the rule. + */ + public Set<Label> getDefaultRestrictedTo() { + return defaultRestrictedTo; + } + + @Override + public String toString() { + return "Package(" + name + ")=" + (targets != null ? getRules() : "initializing..."); + } + + /** + * Dumps the package for debugging. Do not depend on the exact format/contents of this debugging + * output. + */ + public void dump(PrintStream out) { + out.println(" Package " + getName() + " (" + getFilename() + ")"); + + // Rules: + out.println(" Rules"); + for (Rule rule : getTargets(Rule.class)) { + out.println(" " + rule.getTargetKind() + " " + rule.getLabel()); + for (Attribute attr : rule.getAttributes()) { + for (Object possibleValue : AggregatingAttributeMapper.of(rule) + .visitAttribute(attr.getName(), attr.getType())) { + out.println(" " + attr.getName() + " = " + possibleValue); + } + } + } + + // Files: + out.println(" Files"); + for (FileTarget file : getTargets(FileTarget.class)) { + out.print(" " + file.getTargetKind() + " " + file.getLabel()); + if (file instanceof OutputFile) { + out.println(" (generated by " + ((OutputFile) file).getGeneratingRule().getLabel() + ")"); + } else { + out.println(); + } + } + + // TODO(bazel-team): (2009) perhaps dump also: + // - subincludes + // - globs + // - containsErrors + // - makeEnv + } + + /** + * Builder class for {@link Package}. + * + * <p>Should only be used by the package loading and the package deserialization machineries. + */ + static class Builder extends AbstractBuilder<Package, Builder> { + Builder(PackageIdentifier packageId) { + super(new Package(packageId)); + } + + @Override + protected Builder self() { + return this; + } + } + + /** Builder class for {@link Package} that does its own globbing. */ + public static class LegacyBuilder extends AbstractBuilder<Package, LegacyBuilder> { + + private Globber globber = null; + + LegacyBuilder(PackageIdentifier packageId) { + super(AbstractBuilder.newPackage(packageId)); + } + + @Override + protected LegacyBuilder self() { + return this; + } + + /** + * Sets the globber used for this package's glob expansions. + */ + LegacyBuilder setGlobber(Globber globber) { + this.globber = globber; + return this; + } + + /** + * Removes a target from the {@link Package} under construction. Intended to be used only by + * {@link PackageFunction} to remove targets whose labels cross subpackage boundaries. + */ + public void removeTarget(Target target) { + if (target.getPackage() == pkg) { + this.targets.remove(target.getName()); + } + } + + /** + * Returns the glob patterns requested by {@link PackageFactory} during evaluation of this + * package's BUILD file. Intended to be used only by {@link PackageFunction} to mark the + * appropriate Skyframe dependencies after the fact. + */ + public Set<Pair<String, Boolean>> getGlobPatterns() { + return globber.getGlobPatterns(); + } + } + + abstract static class AbstractBuilder<P extends Package, B extends AbstractBuilder<P, B>> { + /** + * The output instance for this builder. Needs to be instantiated and + * available with name info throughout initialization. All other settings + * are applied during {@link #build}. See {@link Package#Package(String)} + * and {@link Package#finishInit} for details. + */ + protected P pkg; + + protected Path filename = null; + private Label buildFileLabel = null; + private InputFile buildFile = null; + private MakeEnvironment.Builder makeEnv = null; + private RuleVisibility defaultVisibility = null; + private boolean defaultVisibilitySet; + private List<String> defaultCopts = null; + private List<String> features = new ArrayList<>(); + private List<Event> events = Lists.newArrayList(); + private boolean containsErrors = false; + private boolean containsTemporaryErrors = false; + + private License defaultLicense = License.NO_LICENSE; + private Set<License.DistributionType> defaultDistributionSet = License.DEFAULT_DISTRIB; + + protected Map<String, Target> targets = new HashMap<>(); + protected Map<Label, EnvironmentGroup> environmentGroups = new HashMap<>(); + + protected Map<Label, Path> subincludes = null; + protected ImmutableList<Label> skylarkFileDependencies = null; + + /** + * True iff the "package" function has already been called in this package. + */ + private boolean packageFunctionUsed; + + /** + * The collection of the prefixes of every output file. Maps every prefix + * to an output file whose prefix it is. + * + * <p>This is needed to make the output file prefix conflict check be + * reasonably fast. However, since it can potentially take a lot of memory and + * is useless after the package has been loaded, it isn't passed to the + * package itself. + */ + private Map<String, OutputFile> outputFilePrefixes = new HashMap<>(); + + private boolean alreadyBuilt = false; + + private EventHandler builderEventHandler = new EventHandler() { + @Override + public void handle(Event event) { + addEvent(event); + } + }; + + protected AbstractBuilder(P pkg) { + this.pkg = pkg; + if (pkg.getName().startsWith("javatests/")) { + setDefaultTestonly(true); + } + } + + protected static Package newPackage(PackageIdentifier packageId) { + return new Package(packageId); + } + + protected abstract B self(); + + protected PackageIdentifier getPackageIdentifier() { + return pkg.getPackageIdentifier(); + } + + /** + * Sets the name of this package's BUILD file. + */ + B setFilename(Path filename) { + this.filename = filename; + try { + buildFileLabel = createLabel(filename.getBaseName()); + addInputFile(buildFileLabel, Location.fromFile(filename)); + } catch (Label.SyntaxException e) { + // This can't actually happen. + throw new AssertionError("Package BUILD file has an illegal name: " + filename); + } + return self(); + } + + public Label getBuildFileLabel() { + return buildFileLabel; + } + + Path getFilename() { + return filename; + } + + /** + * Sets this package's Make environment. + */ + B setMakeEnv(MakeEnvironment.Builder makeEnv) { + this.makeEnv = makeEnv; + return self(); + } + + /** + * Sets the default visibility for this package. Called at most once per + * package from PackageFactory. + */ + B setDefaultVisibility(RuleVisibility visibility) { + this.defaultVisibility = visibility; + this.defaultVisibilitySet = true; + return self(); + } + + /** + * Sets whether the default visibility is set in the BUILD file. + */ + B setDefaultVisibilitySet(boolean defaultVisibilitySet) { + this.defaultVisibilitySet = defaultVisibilitySet; + return self(); + } + + /** + * Sets the default value of 'obsolete'. Rule-level 'obsolete' will override this. + */ + B setDefaultObsolete(boolean defaultObsolete) { + pkg.setDefaultObsolete(defaultObsolete); + return self(); + } + + /** Sets the default value of 'testonly'. Rule-level 'testonly' will override this. */ + B setDefaultTestonly(boolean defaultTestonly) { + pkg.setDefaultTestOnly(defaultTestonly); + return self(); + } + + /** + * Sets the default value of 'deprecation'. Rule-level 'deprecation' will append to this. + */ + B setDefaultDeprecation(String defaultDeprecation) { + pkg.setDefaultDeprecation(defaultDeprecation); + return self(); + } + + /** + * Returns whether the "package" function has been called yet + */ + public boolean isPackageFunctionUsed() { + return packageFunctionUsed; + } + + public void setPackageFunctionUsed() { + packageFunctionUsed = true; + } + + /** + * Sets the default header checking mode. + */ + public B setDefaultHdrsCheck(String hdrsCheck) { + // Note that this setting is propagated directly to the package because + // other code needs the ability to read this info directly from the + // under-construction package. See {@link Package#setDefaultHdrsCheck}. + pkg.setDefaultHdrsCheck(hdrsCheck); + return self(); + } + + /** + * Sets the default value of copts. Rule-level copts will append to this. + */ + public B setDefaultCopts(List<String> defaultCopts) { + this.defaultCopts = defaultCopts; + return self(); + } + + public B addFeatures(Iterable<String> features) { + Iterables.addAll(this.features, features); + return self(); + } + + /** + * Declares that errors were encountering while loading this package. + */ + public B setContainsErrors() { + containsErrors = true; + return self(); + } + + public boolean containsErrors() { + return containsErrors; + } + + B setContainsTemporaryErrors() { + setContainsErrors(); + containsTemporaryErrors = true; + return self(); + } + + public B addEvents(Iterable<Event> events) { + for (Event event : events) { + addEvent(event); + } + return self(); + } + + public B addEvent(Event event) { + this.events.add(event); + return self(); + } + + B setSkylarkFileDependencies(ImmutableList<Label> skylarkFileDependencies) { + this.skylarkFileDependencies = skylarkFileDependencies; + return self(); + } + + /** + * Sets the default license for this package. + */ + void setDefaultLicense(License license) { + this.defaultLicense = license; + } + + License getDefaultLicense() { + return defaultLicense; + } + + /** + * Initializes the default set of distributions for targets in this package. + * + * TODO(bazel-team): (2011) consider moving the license & distribs info into Metadata--maybe + * even in the Build language. + */ + void setDefaultDistribs(Set<DistributionType> dists) { + this.defaultDistributionSet = dists; + } + + Set<DistributionType> getDefaultDistribs() { + return defaultDistributionSet; + } + + /** + * Sets the default value to use for a rule's {@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR} + * attribute when not explicitly specified by the rule. Records a package error if + * any labels are duplicated. + */ + void setDefaultCompatibleWith(List<Label> environments, String attrName, Location location) { + if (!checkForDuplicateLabels(environments, "package " + pkg.getName(), attrName, location, + builderEventHandler)) { + setContainsErrors(); + } + pkg.setDefaultCompatibleWith(ImmutableSet.copyOf(environments)); + } + + /** + * Sets the default value to use for a rule's {@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR} + * attribute when not explicitly specified by the rule. Records a package error if + * any labels are duplicated. + */ + void setDefaultRestrictedTo(List<Label> environments, String attrName, Location location) { + if (!checkForDuplicateLabels(environments, "package " + pkg.getName(), attrName, location, + builderEventHandler)) { + setContainsErrors(); + } + + pkg.setDefaultRestrictedTo(ImmutableSet.copyOf(environments)); + } + + /** + * Returns a new Rule belonging to this package instance, and uses the given Label. + * + * <p>Useful for RuleClass instantiation, where the rule name is checked by trying to create a + * Label. This label can then be used again here. + */ + Rule newRuleWithLabel(Label label, RuleClass ruleClass, FuncallExpression ast, + Location location) { + return new Rule(pkg, label, ruleClass, ast, location); + } + + /** + * Called by the parser when a "mocksubinclude" is encountered, to record the + * mappings from labels to absolute paths upon which that the validity of + * this package depends. + */ + void addSubinclude(Label label, Path resolvedPath) { + if (subincludes == null) { + // This is a TreeMap because the order needs to be deterministic. + subincludes = Maps.newTreeMap(); + } + + Path oldResolvedPath = subincludes.put(label, resolvedPath); + if (oldResolvedPath != null && !oldResolvedPath.equals(resolvedPath)){ + // The same label should have been resolved to the same path + throw new IllegalStateException("Ambiguous subinclude path"); + } + } + + public Set<Label> getSubincludeLabels() { + return subincludes == null ? Sets.<Label>newHashSet() : subincludes.keySet(); + } + + public Map<Label, Path> getSubincludes() { + return subincludes == null ? Maps.<Label, Path>newHashMap() : subincludes; + } + + public Collection<Target> getTargets() { + return Package.getTargets(targets); + } + + /** + * Returns an (immutable, unordered) view of all the targets belonging to + * this package which are instances of the specified class. + */ + <T extends Target> Iterable<T> getTargets(Class<T> targetClass) { + return Package.getTargets(targets, targetClass); + } + + /** + * An input file name conflicts with an existing package member. + */ + static class GeneratedLabelConflict extends NameConflictException { + private GeneratedLabelConflict(String message) { + super(message); + } + } + + /** + * Creates an input file target in this package with the specified name. + * + * @param targetName name of the input file. This must be a valid target + * name as defined by {@link + * com.google.devtools.build.lib.cmdline.LabelValidator#validateTargetName}. + * @return the newly-created InputFile, or the old one if it already existed. + * @throws GeneratedLabelConflict if the name was already taken by a Rule or + * an OutputFile target. + * @throws IllegalArgumentException if the name is not a valid label + */ + InputFile createInputFile(String targetName, Location location) + throws GeneratedLabelConflict { + Target existing = targets.get(targetName); + if (existing == null) { + try { + return addInputFile(createLabel(targetName), location); + } catch (Label.SyntaxException e) { + throw new IllegalArgumentException("FileTarget in package " + pkg.getName() + + " has illegal name: " + targetName); + } + } else if (existing instanceof InputFile) { + return (InputFile) existing; // idempotent + } else { + throw new GeneratedLabelConflict("generated label '//" + pkg.getName() + ":" + + targetName + "' conflicts with existing " + + existing.getTargetKind()); + } + } + + /** + * Sets the visibility and license for an input file. The input file must already exist as + * a member of this package. + * @throws IllegalArgumentException if the input file doesn't exist in this + * package's target map. + */ + void setVisibilityAndLicense(InputFile inputFile, RuleVisibility visibility, License license) { + String filename = inputFile.getName(); + Target cacheInstance = targets.get(filename); + if (cacheInstance == null || !(cacheInstance instanceof InputFile)) { + throw new IllegalArgumentException("Can't set visibility for nonexistent FileTarget " + + filename + " in package " + pkg.getName() + "."); + } + if (!((InputFile) cacheInstance).isVisibilitySpecified() + || cacheInstance.getVisibility() != visibility + || cacheInstance.getLicense() != license) { + targets.put(filename, new InputFile( + pkg, cacheInstance.getLabel(), cacheInstance.getLocation(), visibility, license)); + } + } + + /** + * Creates a label for a target inside this package. + * + * @throws SyntaxException if the {@code targetName} is invalid + */ + Label createLabel(String targetName) throws SyntaxException { + return Label.create(pkg.getPackageIdentifier(), targetName); + } + + /** + * Adds a package group to the package. + */ + void addPackageGroup(String name, Collection<String> packages, Collection<Label> includes, + EventHandler eventHandler, Location location) + throws NameConflictException, Label.SyntaxException { + PackageGroup group = + new PackageGroup(createLabel(name), pkg, packages, includes, eventHandler, location); + Target existing = targets.get(group.getName()); + if (existing != null) { + throw nameConflict(group, existing); + } + + targets.put(group.getName(), group); + + if (group.containsErrors()) { + setContainsErrors(); + } + } + + /** + * Checks if any labels in the given list appear multiple times and reports an appropriate + * error message if so. Returns true if no duplicates were found, false otherwise. + * + * TODO(bazel-team): apply this to all build functions (maybe automatically?), possibly + * integrate with RuleClass.checkForDuplicateLabels. + */ + private static boolean checkForDuplicateLabels(Collection<Label> labels, String owner, + String attrName, Location location, EventHandler eventHandler) { + Set<Label> dupes = CollectionUtils.duplicatedElementsOf(labels); + for (Label dupe : dupes) { + eventHandler.handle(Event.error(location, String.format( + "label '%s' is duplicated in the '%s' list of '%s'", dupe, attrName, owner))); + } + return dupes.isEmpty(); + } + + /** + * Adds an environment group to the package. + */ + void addEnvironmentGroup(String name, List<Label> environments, List<Label> defaults, + EventHandler eventHandler, Location location) + throws NameConflictException, SyntaxException { + + if (!checkForDuplicateLabels(environments, name, "environments", location, eventHandler) + || !checkForDuplicateLabels(defaults, name, "defaults", location, eventHandler)) { + setContainsErrors(); + return; + } + + EnvironmentGroup group = new EnvironmentGroup(createLabel(name), pkg, environments, + defaults, location); + Target existing = targets.get(group.getName()); + if (existing != null) { + throw nameConflict(group, existing); + } + + targets.put(group.getName(), group); + Collection<Event> membershipErrors = group.validateMembership(); + if (!membershipErrors.isEmpty()) { + for (Event error : membershipErrors) { + eventHandler.handle(error); + } + setContainsErrors(); + return; + } + + // For each declared environment, make sure it doesn't also belong to some other group. + for (Label environment : group.getEnvironments()) { + EnvironmentGroup otherGroup = environmentGroups.get(environment); + if (otherGroup != null) { + eventHandler.handle(Event.error(location, "environment " + environment + " belongs to" + + " both " + group.getLabel() + " and " + otherGroup.getLabel())); + setContainsErrors(); + } else { + environmentGroups.put(environment, group); + } + } + } + + void addRule(Rule rule) throws NameConflictException { + checkForConflicts(rule); + // Now, modify the package: + for (OutputFile outputFile : rule.getOutputFiles()) { + targets.put(outputFile.getName(), outputFile); + PathFragment outputFileFragment = new PathFragment(outputFile.getName()); + for (int i = 1; i < outputFileFragment.segmentCount(); i++) { + String prefix = outputFileFragment.subFragment(0, i).toString(); + if (!outputFilePrefixes.containsKey(prefix)) { + outputFilePrefixes.put(prefix, outputFile); + } + } + } + targets.put(rule.getName(), rule); + if (rule.containsErrors()) { + this.setContainsErrors(); + } + } + + private B beforeBuild() { + Preconditions.checkNotNull(pkg); + Preconditions.checkNotNull(filename); + Preconditions.checkNotNull(buildFileLabel); + Preconditions.checkNotNull(makeEnv); + // Freeze subincludes. + subincludes = (subincludes == null) + ? Collections.<Label, Path>emptyMap() + : Collections.unmodifiableMap(subincludes); + + // We create the original BUILD InputFile when the package filename is set; however, the + // visibility may be overridden with an exports_files directive, so we need to obtain the + // current instance here. + buildFile = (InputFile) Preconditions.checkNotNull(targets.get(buildFileLabel.getName())); + + List<Rule> rules = Lists.newArrayList(getTargets(Rule.class)); + + // All labels mentioned in a rule that refer to an unknown target in the + // current package are assumed to be InputFiles, so let's create them: + for (final Rule rule : rules) { + AggregatingAttributeMapper.of(rule).visitLabels(new AcceptsLabelAttribute() { + @Override + public void acceptLabelAttribute(Label label, Attribute attribute) { + createInputFileMaybe(label, rule.getAttributeLocation(attribute.getName())); + } + }); + } + + // "test_suite" rules have the idiosyncratic semantics of implicitly + // depending on all tests in the package, iff tests=[] and suites=[]. + // Note, we implement this here when the Package is fully constructed, + // since clearly this information isn't available at Rule construction + // time, as forward references are permitted. + List<Label> allTests = new ArrayList<>(); + for (Rule rule : rules) { + if (TargetUtils.isTestRule(rule) && !TargetUtils.hasManualTag(rule) + && !TargetUtils.isObsolete(rule)) { + allTests.add(rule.getLabel()); + } + } + for (Rule rule : rules) { + AttributeMap attributes = NonconfigurableAttributeMapper.of(rule); + if (rule.getRuleClass().equals("test_suite") + && attributes.get("tests", Type.LABEL_LIST).isEmpty() + && attributes.get("suites", Type.LABEL_LIST).isEmpty()) { + rule.setAttributeValueByName("$implicit_tests", allTests); + } + } + return self(); + } + + /** Intended to be used only by {@link PackageFunction}. */ + public B buildPartial() { + if (alreadyBuilt) { + return self(); + } + return beforeBuild(); + } + + /** Intended to be used only by {@link PackageFunction}. */ + public P finishBuild() { + if (alreadyBuilt) { + return pkg; + } + // Freeze targets and distributions. + targets = ImmutableMap.copyOf(targets); + defaultDistributionSet = + Collections.unmodifiableSet(defaultDistributionSet); + + // Now all targets have been loaded, so we can check all declared environments in an + // environment group exist. + for (EnvironmentGroup envGroup : ImmutableSet.copyOf(environmentGroups.values())) { + Collection<Event> errors = envGroup.checkEnvironmentsExist(targets); + if (!errors.isEmpty()) { + addEvents(errors); + setContainsErrors(); + } + } + + // Build the package. + pkg.finishInit(this); + alreadyBuilt = true; + return pkg; + } + + public P build() { + if (alreadyBuilt) { + return pkg; + } + beforeBuild(); + return finishBuild(); + } + + /** + * If "label" refers to a non-existent target in the current package, create + * an InputFile target. + */ + void createInputFileMaybe(Label label, Location location) { + if (label != null && label.getPackageFragment().equals(pkg.getNameFragment())) { + if (!targets.containsKey(label.getName())) { + addInputFile(label, location); + } + } + } + + private InputFile addInputFile(Label label, Location location) { + InputFile inputFile = new InputFile(pkg, label, location); + Target prev = targets.put(label.getName(), inputFile); + Preconditions.checkState(prev == null); + return inputFile; + } + + /** + * Precondition check for addRule. We must maintain these invariants of the + * package: + * - Each name refers to at most one target. + * - No rule with errors is inserted into the package. + * - The generating rule of every output file in the package must itself be + * in the package. + */ + private void checkForConflicts(Rule rule) throws NameConflictException { + String name = rule.getName(); + Target existing = targets.get(name); + if (existing != null) { + throw nameConflict(rule, existing); + } + Map<String, OutputFile> outputFiles = new HashMap<>(); + + for (OutputFile outputFile : rule.getOutputFiles()) { + String outputFileName = outputFile.getName(); + if (outputFiles.put(outputFileName, outputFile) != null) { // dups within a single rule: + throw duplicateOutputFile(outputFile, outputFile); + } + existing = targets.get(outputFileName); + if (existing != null) { + throw duplicateOutputFile(outputFile, existing); + } + + // Check if this output file is the prefix of an already existing one + if (outputFilePrefixes.containsKey(outputFileName)) { + throw conflictingOutputFile(outputFile, outputFilePrefixes.get(outputFileName)); + } + + // Check if a prefix of this output file matches an already existing one + PathFragment outputFileFragment = new PathFragment(outputFileName); + for (int i = 1; i < outputFileFragment.segmentCount(); i++) { + String prefix = outputFileFragment.subFragment(0, i).toString(); + if (outputFiles.containsKey(prefix)) { + throw conflictingOutputFile(outputFile, outputFiles.get(prefix)); + } + if (targets.containsKey(prefix) + && targets.get(prefix) instanceof OutputFile) { + throw conflictingOutputFile(outputFile, (OutputFile) targets.get(prefix)); + } + + if (!outputFilePrefixes.containsKey(prefix)) { + outputFilePrefixes.put(prefix, outputFile); + } + } + } + + checkForInputOutputConflicts(rule, outputFiles.keySet()); + } + + /** + * A utility method that checks for conflicts between + * input file names and output file names for a rule from a build + * file. + * @param rule the rule whose inputs and outputs are + * to be checked for conflicts. + * @param outputFiles a set containing the names of output + * files to be generated by the rule. + * @throws NameConflictException if a conflict is found. + */ + private void checkForInputOutputConflicts(Rule rule, Set<String> outputFiles) + throws NameConflictException { + PathFragment packageFragment = rule.getLabel().getPackageFragment(); + for (Label inputLabel : rule.getLabels()) { + if (packageFragment.equals(inputLabel.getPackageFragment()) + && outputFiles.contains(inputLabel.getName())) { + throw inputOutputNameConflict(rule, inputLabel.getName()); + } + } + } + + /** An output file conflicts with another output file or the BUILD file. */ + private NameConflictException duplicateOutputFile(OutputFile duplicate, Target existing) { + return new NameConflictException(duplicate.getTargetKind() + " '" + duplicate.getName() + + "' in rule '" + duplicate.getGeneratingRule().getName() + "' " + + conflictsWith(existing)); + } + + /** The package contains two targets with the same name. */ + private NameConflictException nameConflict(Target duplicate, Target existing) { + return new NameConflictException(duplicate.getTargetKind() + " '" + duplicate.getName() + + "' in package '" + duplicate.getLabel().getPackageName() + "' " + + conflictsWith(existing)); + } + + /** A a rule has a input/output name conflict. */ + private NameConflictException inputOutputNameConflict(Rule rule, String conflictingName) { + return new NameConflictException("rule '" + rule.getName() + "' has file '" + + conflictingName + "' as both an input and an output"); + } + + private static NameConflictException conflictingOutputFile( + OutputFile added, OutputFile existing) { + if (added.getGeneratingRule() == existing.getGeneratingRule()) { + return new NameConflictException(String.format( + "rule '%s' has conflicting output files '%s' and '%s'", added.getGeneratingRule() + .getName(), added.getName(), existing.getName())); + } else { + return new NameConflictException(String.format( + "output file '%s' of rule '%s' conflicts with output file '%s' of rule '%s'", added + .getName(), added.getGeneratingRule().getName(), existing.getName(), existing + .getGeneratingRule().getName())); + } + } + + /** + * Utility function for generating exception messages. + */ + private static String conflictsWith(Target target) { + String message = "conflicts with existing "; + if (target instanceof OutputFile) { + return message + "generated file from rule '" + + ((OutputFile) target).getGeneratingRule().getName() + + "'"; + } else { + return message + target.getTargetKind(); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageDeserializer.java b/src/main/java/com/google/devtools/build/lib/packages/PackageDeserializer.java new file mode 100644 index 0000000..5eca0f4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/PackageDeserializer.java
@@ -0,0 +1,536 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.events.NullEventHandler; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.packages.License.DistributionType; +import com.google.devtools.build.lib.packages.License.LicenseParsingException; +import com.google.devtools.build.lib.packages.Package.AbstractBuilder.GeneratedLabelConflict; +import com.google.devtools.build.lib.packages.Package.NameConflictException; +import com.google.devtools.build.lib.packages.RuleClass.ParsedAttributeValue; +import com.google.devtools.build.lib.query2.proto.proto2api.Build; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.StringDictUnaryEntry; +import com.google.devtools.build.lib.syntax.FilesetEntry; +import com.google.devtools.build.lib.syntax.GlobCriteria; +import com.google.devtools.build.lib.syntax.GlobList; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Functionality to deserialize loaded packages. + */ +public class PackageDeserializer { + + // Workaround for Java serialization not allowing to pass in a context manually. + // volatile is needed to ensure that the objects are published safely. + // TODO(bazel-team): Subclass ObjectOutputStream to pass through environment variables. + public static volatile RuleClassProvider defaultRuleClassProvider; + public static volatile FileSystem defaultDeserializerFileSystem; + + private class Context { + private final Package.Builder packageBuilder; + private final Path buildFilePath; + + public Context(Path buildFilePath, Package.Builder packageBuilder) { + this.buildFilePath = buildFilePath; + this.packageBuilder = packageBuilder; + } + + Location deserializeLocation(Build.Location location) { + return new ExplicitLocation(buildFilePath, location); + } + + ParsedAttributeValue deserializeAttribute(Type<?> expectedType, + Build.Attribute attrPb) + throws PackageDeserializationException { + Object value = deserializeAttributeValue(expectedType, attrPb); + return new ParsedAttributeValue( + attrPb.hasExplicitlySpecified() ? attrPb.getExplicitlySpecified() : false, + value, + deserializeLocation(attrPb.getParseableLocation())); + } + + void deserializeInputFile(Build.SourceFile sourceFile) + throws PackageDeserializationException { + InputFile inputFile; + try { + inputFile = packageBuilder.createInputFile( + deserializeLabel(sourceFile.getName()).getName(), + deserializeLocation(sourceFile.getParseableLocation())); + } catch (GeneratedLabelConflict e) { + throw new PackageDeserializationException(e); + } + + if (!sourceFile.getVisibilityLabelList().isEmpty() || sourceFile.hasLicense()) { + packageBuilder.setVisibilityAndLicense(inputFile, + PackageFactory.getVisibility(deserializeLabels(sourceFile.getVisibilityLabelList())), + deserializeLicense(sourceFile.getLicense())); + } + } + + void deserializePackageGroup(Build.PackageGroup packageGroupPb) + throws PackageDeserializationException { + List<String> specifications = new ArrayList<>(); + for (String containedPackage : packageGroupPb.getContainedPackageList()) { + specifications.add("//" + containedPackage); + } + + try { + packageBuilder.addPackageGroup( + deserializeLabel(packageGroupPb.getName()).getName(), + specifications, + deserializeLabels(packageGroupPb.getIncludedPackageGroupList()), + NullEventHandler.INSTANCE, // TODO(bazel-team): Handle errors properly + deserializeLocation(packageGroupPb.getParseableLocation())); + } catch (Label.SyntaxException | Package.NameConflictException e) { + throw new PackageDeserializationException(e); + } + } + + void deserializeRule(Build.Rule rulePb) + throws PackageDeserializationException { + RuleClass ruleClass = ruleClassProvider.getRuleClassMap().get(rulePb.getRuleClass()); + if (ruleClass == null) { + throw new PackageDeserializationException( + String.format("Invalid rule class '%s'", ruleClass)); + } + + Map<String, ParsedAttributeValue> attributeValues = new HashMap<>(); + for (Build.Attribute attrPb : rulePb.getAttributeList()) { + Type<?> type = ruleClass.getAttributeByName(attrPb.getName()).getType(); + attributeValues.put(attrPb.getName(), deserializeAttribute(type, attrPb)); + } + + Label ruleLabel = deserializeLabel(rulePb.getName()); + Location ruleLocation = deserializeLocation(rulePb.getParseableLocation()); + try { + Rule rule = ruleClass.createRuleWithParsedAttributeValues( + ruleLabel, packageBuilder, ruleLocation, attributeValues, + NullEventHandler.INSTANCE); + packageBuilder.addRule(rule); + + Preconditions.checkState(!rule.containsErrors()); + } catch (NameConflictException | SyntaxException e) { + throw new PackageDeserializationException(e); + } + } + } + + private final FileSystem fileSystem; + private final RuleClassProvider ruleClassProvider; + + @Immutable + private static final class ExplicitLocation extends Location { + private final PathFragment path; + private final int startLine; + private final int startColumn; + private final int endLine; + private final int endColumn; + + private ExplicitLocation(Path path, Build.Location location) { + super( + location.hasStartOffset() && location.hasEndOffset() ? location.getStartOffset() : 0, + location.hasStartOffset() && location.hasEndOffset() ? location.getEndOffset() : 0); + this.path = path.asFragment(); + if (location.hasStartLine() && location.hasStartColumn() && + location.hasEndLine() && location.hasEndColumn()) { + this.startLine = location.getStartLine(); + this.startColumn = location.getStartColumn(); + this.endLine = location.getEndLine(); + this.endColumn = location.getEndColumn(); + } else { + this.startLine = 0; + this.startColumn = 0; + this.endLine = 0; + this.endColumn = 0; + } + } + + @Override + public PathFragment getPath() { + return path; + } + + @Override + public LineAndColumn getStartLineAndColumn() { + return new LineAndColumn(startLine, startColumn); + } + + @Override + public LineAndColumn getEndLineAndColumn() { + return new LineAndColumn(endLine, endColumn); + } + } + + public PackageDeserializer(FileSystem fileSystem, RuleClassProvider ruleClassProvider) { + if (fileSystem == null) { + fileSystem = defaultDeserializerFileSystem; + } + this.fileSystem = Preconditions.checkNotNull(fileSystem); + if (ruleClassProvider == null) { + ruleClassProvider = defaultRuleClassProvider; + } + this.ruleClassProvider = Preconditions.checkNotNull(ruleClassProvider); + } + + /** + * Exception thrown when something goes wrong during package deserialization. + */ + public static class PackageDeserializationException extends Exception { + private PackageDeserializationException(String message) { + super(message); + } + + private PackageDeserializationException(String message, Exception reason) { + super(message, reason); + } + + private PackageDeserializationException(Exception reason) { + super(reason); + } + } + + private static Label deserializeLabel(String labelName) throws PackageDeserializationException { + try { + return Label.parseRepositoryLabel(labelName); + } catch (Label.SyntaxException e) { + throw new PackageDeserializationException("Invalid label: " + e.getMessage(), e); + } + } + + private static List<Label> deserializeLabels(List<String> labelNames) + throws PackageDeserializationException { + ImmutableList.Builder<Label> result = ImmutableList.builder(); + for (String labelName : labelNames) { + result.add(deserializeLabel(labelName)); + } + + return result.build(); + } + + private static License deserializeLicense(Build.License licensePb) + throws PackageDeserializationException { + List<String> licenseStrings = new ArrayList<>(); + licenseStrings.addAll(licensePb.getLicenseTypeList()); + for (String exception : licensePb.getExceptionList()) { + licenseStrings.add("exception=" + exception); + } + + try { + return License.parseLicense(licenseStrings); + } catch (LicenseParsingException e) { + throw new PackageDeserializationException(e); + } + } + + private static Set<DistributionType> deserializeDistribs(List<String> distributions) + throws PackageDeserializationException { + try { + return License.parseDistributions(distributions); + } catch (LicenseParsingException e) { + throw new PackageDeserializationException(e); + } + } + + private static TriState deserializeTriStateValue(String value) + throws PackageDeserializationException { + if (value.equals("yes")) { + return TriState.YES; + } else if (value.equals("no")) { + return TriState.NO; + } else if (value.equals("auto")) { + return TriState.AUTO; + } else { + throw new PackageDeserializationException( + String.format("Invalid tristate value: '%s'", value)); + } + } + + private static List<FilesetEntry> deserializeFilesetEntries( + List<Build.FilesetEntry> filesetPbs) + throws PackageDeserializationException { + ImmutableList.Builder<FilesetEntry> result = ImmutableList.builder(); + for (Build.FilesetEntry filesetPb : filesetPbs) { + Label srcLabel = deserializeLabel(filesetPb.getSource()); + List<Label> files = + filesetPb.getFilesPresent() ? deserializeLabels(filesetPb.getFileList()) : null; + List<String> excludes = + filesetPb.getExcludeList().isEmpty() ? + null : ImmutableList.copyOf(filesetPb.getExcludeList()); + String destDir = filesetPb.getDestinationDirectory(); + FilesetEntry.SymlinkBehavior symlinkBehavior = + pbToSymlinkBehavior(filesetPb.getSymlinkBehavior()); + String stripPrefix = filesetPb.hasStripPrefix() ? filesetPb.getStripPrefix() : null; + + result.add( + new FilesetEntry(srcLabel, files, excludes, destDir, symlinkBehavior, stripPrefix)); + } + + return result.build(); + } + + /** + * Deserialize a package from its representation as a protocol message. The inverse of + * {@link PackageSerializer#serializePackage}. + */ + private void deserializeInternal(Build.Package packagePb, StoredEventHandler eventHandler, + Package.Builder builder) throws PackageDeserializationException { + Path buildFile = fileSystem.getPath(packagePb.getBuildFilePath()); + Preconditions.checkNotNull(buildFile); + Context context = new Context(buildFile, builder); + builder.setFilename(buildFile); + + if (packagePb.hasDefaultVisibilitySet() && packagePb.getDefaultVisibilitySet()) { + builder.setDefaultVisibility( + PackageFactory.getVisibility( + deserializeLabels(packagePb.getDefaultVisibilityLabelList()))); + } + + // It's important to do this after setting the default visibility, since that implicitly sets + // this bit to true + builder.setDefaultVisibilitySet(packagePb.getDefaultVisibilitySet()); + if (packagePb.hasDefaultObsolete()) { + builder.setDefaultObsolete(packagePb.getDefaultObsolete()); + } + if (packagePb.hasDefaultTestonly()) { + builder.setDefaultTestonly(packagePb.getDefaultTestonly()); + } + if (packagePb.hasDefaultDeprecation()) { + builder.setDefaultDeprecation(packagePb.getDefaultDeprecation()); + } + + builder.setDefaultCopts(packagePb.getDefaultCoptList()); + if (packagePb.hasDefaultHdrsCheck()) { + builder.setDefaultHdrsCheck(packagePb.getDefaultHdrsCheck()); + } + if (packagePb.hasDefaultLicense()) { + builder.setDefaultLicense(deserializeLicense(packagePb.getDefaultLicense())); + } + builder.setDefaultDistribs(deserializeDistribs(packagePb.getDefaultDistribList())); + + for (String subinclude : packagePb.getSubincludeLabelList()) { + Label label = deserializeLabel(subinclude); + builder.addSubinclude(label, null); + } + + ImmutableList.Builder<Label> skylarkFileDependencies = ImmutableList.builder(); + for (String skylarkFile : packagePb.getSkylarkLabelList()) { + skylarkFileDependencies.add(deserializeLabel(skylarkFile)); + } + builder.setSkylarkFileDependencies(skylarkFileDependencies.build()); + + MakeEnvironment.Builder makeEnvBuilder = new MakeEnvironment.Builder(); + for (Build.MakeVar makeVar : packagePb.getMakeVariableList()) { + for (Build.MakeVarBinding binding : makeVar.getBindingList()) { + makeEnvBuilder.update( + makeVar.getName(), binding.getValue(), binding.getPlatformSetRegexp()); + } + } + builder.setMakeEnv(makeEnvBuilder); + + for (Build.SourceFile sourceFile : packagePb.getSourceFileList()) { + context.deserializeInputFile(sourceFile); + } + + for (Build.PackageGroup packageGroupPb : + packagePb.getPackageGroupList()) { + context.deserializePackageGroup(packageGroupPb); + } + + for (Build.Rule rulePb : packagePb.getRuleList()) { + context.deserializeRule(rulePb); + } + + for (Build.Event event : packagePb.getEventList()) { + deserializeEvent(context, eventHandler, event); + } + + if (packagePb.hasContainsErrors() && packagePb.getContainsErrors()) { + builder.setContainsErrors(); + } + if (packagePb.hasContainsTemporaryErrors() && packagePb.getContainsTemporaryErrors()) { + builder.setContainsTemporaryErrors(); + } + } + + /** + * Deserialize a protocol message to a package. The inverse of + * {@link PackageSerializer#serializePackage}. + */ + public Package deserialize(Build.Package packagePb) + throws PackageDeserializationException { + Package.Builder builder; + try { + builder = new Package.Builder( + new PackageIdentifier(packagePb.getRepository(), new PathFragment(packagePb.getName()))); + } catch (SyntaxException e) { + throw new PackageDeserializationException(e); + } + StoredEventHandler eventHandler = new StoredEventHandler(); + deserializeInternal(packagePb, eventHandler, builder); + builder.addEvents(eventHandler.getEvents()); + return builder.build(); + } + + private static void deserializeEvent( + Context context, StoredEventHandler eventHandler, Build.Event event) { + Location location = null; + if (event.hasLocation()) { + location = context.deserializeLocation(event.getLocation()); + } + + String message = event.getMessage(); + switch (event.getKind()) { + case ERROR: eventHandler.handle(Event.error(location, message)); break; + case WARNING: eventHandler.handle(Event.warn(location, message)); break; + case INFO: eventHandler.handle(Event.info(location, message)); break; + case PROGRESS: eventHandler.handle(Event.progress(location, message)); break; + default: break; // Ignore + } + } + + private static List<?> deserializeGlobs(List<?> matches, + Build.Attribute attrPb) { + if (attrPb.getGlobCriteriaCount() == 0) { + return matches; + } + + Builder<GlobCriteria> criteriaBuilder = ImmutableList.builder(); + for (Build.GlobCriteria criteriaPb : attrPb.getGlobCriteriaList()) { + if (criteriaPb.hasGlob() && criteriaPb.getGlob()) { + criteriaBuilder.add(GlobCriteria.fromGlobCall( + ImmutableList.copyOf(criteriaPb.getIncludeList()), + ImmutableList.copyOf(criteriaPb.getExcludeList()))); + } else { + criteriaBuilder.add( + GlobCriteria.fromList(ImmutableList.copyOf(criteriaPb.getIncludeList()))); + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) GlobList<?> result = + new GlobList(criteriaBuilder.build(), matches); + return result; + } + + // TODO(bazel-team): Verify that these put sane values in the attribute + private static Object deserializeAttributeValue(Type<?> expectedType, + Build.Attribute attrPb) + throws PackageDeserializationException { + switch (attrPb.getType()) { + case INTEGER: + return new Integer(attrPb.getIntValue()); + + case STRING: + if (expectedType == Type.NODEP_LABEL) { + return deserializeLabel(attrPb.getStringValue()); + } else { + return attrPb.getStringValue(); + } + + case LABEL: + case OUTPUT: + return deserializeLabel(attrPb.getStringValue()); + + case STRING_LIST: + if (expectedType == Type.NODEP_LABEL_LIST) { + return deserializeGlobs(deserializeLabels(attrPb.getStringListValueList()), attrPb); + } else { + return deserializeGlobs(ImmutableList.copyOf(attrPb.getStringListValueList()), attrPb); + } + + case LABEL_LIST: + case OUTPUT_LIST: + return deserializeGlobs(deserializeLabels(attrPb.getStringListValueList()), attrPb); + + case DISTRIBUTION_SET: + return deserializeDistribs(attrPb.getStringListValueList()); + + case LICENSE: + return deserializeLicense(attrPb.getLicense()); + + case STRING_DICT: { + ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); + for (Build.StringDictEntry entry : attrPb.getStringDictValueList()) { + builder.put(entry.getKey(), entry.getValue()); + } + return builder.build(); + } + + case STRING_DICT_UNARY: { + ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); + for (StringDictUnaryEntry entry : attrPb.getStringDictUnaryValueList()) { + builder.put(entry.getKey(), entry.getValue()); + } + return builder.build(); + } + + case FILESET_ENTRY_LIST: + return deserializeFilesetEntries(attrPb.getFilesetListValueList()); + + case LABEL_LIST_DICT: { + ImmutableMap.Builder<String, List<Label>> builder = ImmutableMap.builder(); + for (Build.LabelListDictEntry entry : attrPb.getLabelListDictValueList()) { + builder.put(entry.getKey(), deserializeLabels(entry.getValueList())); + } + return builder.build(); + } + + case STRING_LIST_DICT: { + ImmutableMap.Builder<String, List<String>> builder = ImmutableMap.builder(); + for (Build.StringListDictEntry entry : attrPb.getStringListDictValueList()) { + builder.put(entry.getKey(), ImmutableList.copyOf(entry.getValueList())); + } + return builder.build(); + } + + case BOOLEAN: + return attrPb.getBooleanValue(); + + case TRISTATE: + return deserializeTriStateValue(attrPb.getStringValue()); + + default: + throw new PackageDeserializationException("Invalid discriminator: " + attrPb.getType()); + } + } + + private static FilesetEntry.SymlinkBehavior pbToSymlinkBehavior( + Build.FilesetEntry.SymlinkBehavior symlinkBehavior) { + switch (symlinkBehavior) { + case COPY: + return FilesetEntry.SymlinkBehavior.COPY; + case DEREFERENCE: + return FilesetEntry.SymlinkBehavior.DEREFERENCE; + default: + throw new IllegalStateException(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java new file mode 100644 index 0000000..abf63f9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java
@@ -0,0 +1,1272 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.devtools.build.lib.cmdline.LabelValidator; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.events.NullEventHandler; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.packages.GlobCache.BadGlobException; +import com.google.devtools.build.lib.packages.License.DistributionType; +import com.google.devtools.build.lib.packages.Type.ConversionException; +import com.google.devtools.build.lib.syntax.AbstractFunction; +import com.google.devtools.build.lib.syntax.AssignmentStatement; +import com.google.devtools.build.lib.syntax.BuildFileAST; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.Environment.NoSuchVariableException; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Expression; +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.Function; +import com.google.devtools.build.lib.syntax.GlobList; +import com.google.devtools.build.lib.syntax.Ident; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.MixedModeFunction; +import com.google.devtools.build.lib.syntax.ParserInputSource; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.Statement; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.UnixGlob; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +/** + * The package factory is responsible for constructing Package instances + * from a BUILD file's abstract syntax tree (AST). + * + * <p>A PackageFactory is a heavy-weight object; create them sparingly. + * Typically only one is needed per client application. + */ +public final class PackageFactory { + /** + * An argument to the {@code package()} function. + */ + public abstract static class PackageArgument<T> { + private final String name; + private final Type<T> type; + + protected PackageArgument(String name, Type<T> type) { + this.name = name; + this.type = type; + } + + public String getName() { + return name; + } + + private void convertAndProcess( + Package.LegacyBuilder pkgBuilder, Location location, Object value) + throws EvalException, ConversionException { + T typedValue = type.convert(value, "'package' argument", pkgBuilder.getBuildFileLabel()); + process(pkgBuilder, location, typedValue); + } + + /** + * Process an argument. + * + * @param pkgBuilder the package builder to be mutated + * @param location the location of the {@code package} function for error reporting + * @param value the value of the argument. Typically passed to {@link Type#convert} + */ + protected abstract void process( + Package.LegacyBuilder pkgBuilder, Location location, T value) + throws EvalException; + } + + /** Interface for evaluating globs during package loading. */ + public static interface Globber { + /** An opaque token for fetching the result of a glob computation. */ + abstract static class Token {} + + /** + * Asynchronously starts the given glob computation and returns a token for fetching the + * result. + */ + Token runAsync(List<String> includes, List<String> excludes, boolean excludeDirs) + throws BadGlobException; + + /** Fetches the result of a previously started glob computation. */ + List<String> fetch(Token token) throws IOException, InterruptedException; + + /** Should be called when the globber is about to be discarded due to an interrupt. */ + void onInterrupt(); + + /** Should be called when the globber is no longer needed. */ + void onCompletion(); + + /** Returns all the glob computations requested before {@link #onCompletion} was called. */ + Set<Pair<String, Boolean>> getGlobPatterns(); + } + + /** + * An extension to the global namespace of the BUILD language. + */ + public interface EnvironmentExtension { + /** + * Update the global environment with the identifiers this extension contributes. + */ + void update(Environment environment, MakeEnvironment.Builder pkgMakeEnv, + Label buildFileLabel); + + Iterable<PackageArgument<?>> getPackageArguments(); + } + + private static final int EXCLUDE_DIR_DEFAULT = 1; + + private static class DefaultVisibility extends PackageArgument<List<Label>> { + private DefaultVisibility() { + super("default_visibility", Type.LABEL_LIST); + } + + @Override + protected void process(Package.LegacyBuilder pkgBuilder, Location location, + List<Label> value) { + pkgBuilder.setDefaultVisibility(getVisibility(value)); + } + } + + private static class DefaultObsolete extends PackageArgument<Boolean> { + private DefaultObsolete() { + super("default_obsolete", Type.BOOLEAN); + } + + @Override + protected void process(Package.LegacyBuilder pkgBuilder, Location location, + Boolean value) { + pkgBuilder.setDefaultObsolete(value); + } + } + + private static class DefaultTestOnly extends PackageArgument<Boolean> { + private DefaultTestOnly() { + super("default_testonly", Type.BOOLEAN); + } + + @Override + protected void process(Package.LegacyBuilder pkgBuilder, Location location, + Boolean value) { + pkgBuilder.setDefaultTestonly(value); + } + } + + private static class DefaultDeprecation extends PackageArgument<String> { + private DefaultDeprecation() { + super("default_deprecation", Type.STRING); + } + + @Override + protected void process(Package.LegacyBuilder pkgBuilder, Location location, + String value) { + pkgBuilder.setDefaultDeprecation(value); + } + } + + private static class Features extends PackageArgument<List<String>> { + private Features() { + super("features", Type.STRING_LIST); + } + + @Override + protected void process(Package.LegacyBuilder pkgBuilder, Location location, + List<String> value) { + pkgBuilder.addFeatures(value); + } + } + + private static class DefaultLicenses extends PackageArgument<License> { + private DefaultLicenses() { + super("licenses", Type.LICENSE); + } + + @Override + protected void process(Package.LegacyBuilder pkgBuilder, Location location, + License value) { + pkgBuilder.setDefaultLicense(value); + } + } + + private static class DefaultDistribs extends PackageArgument<Set<DistributionType>> { + private DefaultDistribs() { + super("distribs", Type.DISTRIBUTIONS); + } + + @Override + protected void process(Package.LegacyBuilder pkgBuilder, Location location, + Set<DistributionType> value) { + pkgBuilder.setDefaultDistribs(value); + } + } + + /** + * Declares the package() attribute specifying the default value for + * {@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR} when not explicitly specified. + */ + private static class DefaultCompatibleWith extends PackageArgument<List<Label>> { + private DefaultCompatibleWith() { + super(Package.DEFAULT_COMPATIBLE_WITH_ATTRIBUTE, Type.LABEL_LIST); + } + + @Override + protected void process(Package.LegacyBuilder pkgBuilder, Location location, + List<Label> value) { + pkgBuilder.setDefaultCompatibleWith(value, Package.DEFAULT_COMPATIBLE_WITH_ATTRIBUTE, + location); + } + } + + /** + * Declares the package() attribute specifying the default value for + * {@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR} when not explicitly specified. + */ + private static class DefaultRestrictedTo extends PackageArgument<List<Label>> { + private DefaultRestrictedTo() { + super(Package.DEFAULT_RESTRICTED_TO_ATTRIBUTE, Type.LABEL_LIST); + } + + @Override + protected void process(Package.LegacyBuilder pkgBuilder, Location location, + List<Label> value) { + pkgBuilder.setDefaultRestrictedTo(value, Package.DEFAULT_RESTRICTED_TO_ATTRIBUTE, location); + } + } + + public static final String PKG_CONTEXT = "$pkg_context"; + + /** {@link Globber} that uses the legacy GlobCache. */ + public static class LegacyGlobber implements Globber { + + private final GlobCache globCache; + + public LegacyGlobber(GlobCache globCache) { + this.globCache = globCache; + } + + private class Token extends Globber.Token { + public final List<String> includes; + public final List<String> excludes; + public final boolean excludeDirs; + + public Token(List<String> includes, List<String> excludes, boolean excludeDirs) { + this.includes = includes; + this.excludes = excludes; + this.excludeDirs = excludeDirs; + } + } + + @Override + public Set<Pair<String, Boolean>> getGlobPatterns() { + return globCache.getKeySet(); + } + + @Override + public Token runAsync(List<String> includes, List<String> excludes, boolean excludeDirs) + throws BadGlobException { + for (String pattern : Iterables.concat(includes, excludes)) { + globCache.getGlobAsync(pattern, excludeDirs); + } + return new Token(includes, excludes, excludeDirs); + } + + @Override + public List<String> fetch(Globber.Token token) throws IOException, InterruptedException { + Token legacyToken = (Token) token; + try { + return globCache.glob(legacyToken.includes, legacyToken.excludes, + legacyToken.excludeDirs); + } catch (BadGlobException e) { + throw new IllegalStateException(e); + } + } + + @Override + public void onInterrupt() { + globCache.cancelBackgroundTasks(); + } + + @Override + public void onCompletion() { + globCache.finishBackgroundTasks(); + } + } + + private static final Logger LOG = Logger.getLogger(PackageFactory.class.getName()); + + private final RuleFactory ruleFactory; + private final RuleClassProvider ruleClassProvider; + private final Environment globalEnv; + + private AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls; + private Preprocessor.Factory preprocessorFactory = Preprocessor.Factory.NullFactory.INSTANCE; + + private final ThreadPoolExecutor threadPool; + private Map<String, String> platformSetRegexps; + + private final ImmutableList<EnvironmentExtension> environmentExtensions; + private final ImmutableMap<String, PackageArgument<?>> packageArguments; + + /** + * Constructs a {@code PackageFactory} instance with the given rule factory. + */ + public PackageFactory(RuleClassProvider ruleClassProvider) { + this(ruleClassProvider, null, ImmutableList.<EnvironmentExtension>of()); + } + + @VisibleForTesting + public PackageFactory(RuleClassProvider ruleClassProvider, + EnvironmentExtension environmentExtensions) { + this(ruleClassProvider, null, ImmutableList.of(environmentExtensions)); + } + + /** + * Constructs a {@code PackageFactory} instance with a specific glob path translator + * and rule factory. + */ + @VisibleForTesting + public PackageFactory(RuleClassProvider ruleClassProvider, + Map<String, String> platformSetRegexps, + Iterable<EnvironmentExtension> environmentExtensions) { + this.platformSetRegexps = platformSetRegexps; + this.ruleFactory = new RuleFactory(ruleClassProvider); + this.ruleClassProvider = ruleClassProvider; + globalEnv = newGlobalEnvironment(); + threadPool = new ThreadPoolExecutor(100, 100, 3L, TimeUnit.SECONDS, + new LinkedBlockingQueue<Runnable>(), + new ThreadFactoryBuilder().setNameFormat("PackageFactory %d").build()); + // Do not consume threads when not in use. + threadPool.allowCoreThreadTimeOut(true); + this.environmentExtensions = ImmutableList.copyOf(environmentExtensions); + this.packageArguments = createPackageArguments(); + } + + /** + * Sets the preprocessor used. + */ + public void setPreprocessorFactory(Preprocessor.Factory preprocessorFactory) { + this.preprocessorFactory = preprocessorFactory; + } + + /** + * Sets the syscalls cache used in globbing. + */ + public void setSyscalls(AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls) { + this.syscalls = Preconditions.checkNotNull(syscalls); + } + + /** + * Returns the static environment initialized once and shared by all packages + * created by this factory. No updates occur to this environment once created. + */ + @VisibleForTesting + public Environment getEnvironment() { + return globalEnv; + } + + /** + * Returns the immutable, unordered set of names of all the known rule + * classes. + */ + public Set<String> getRuleClassNames() { + return ruleFactory.getRuleClassNames(); + } + + /** + * Returns the {@link RuleClass} for the specified rule class name. + */ + public RuleClass getRuleClass(String ruleClassName) { + return ruleFactory.getRuleClass(ruleClassName); + } + + /** + * Returns the {@link RuleClassProvider} of this {@link PackageFactory}. + */ + public RuleClassProvider getRuleClassProvider() { + return ruleClassProvider; + } + + /** + * Creates the list of arguments for the 'package' function. + */ + private ImmutableMap<String, PackageArgument<?>> createPackageArguments() { + ImmutableList.Builder<PackageArgument<?>> arguments = + ImmutableList.<PackageArgument<?>>builder() + .add(new DefaultDeprecation()) + .add(new DefaultDistribs()) + .add(new DefaultLicenses()) + .add(new DefaultObsolete()) + .add(new DefaultTestOnly()) + .add(new DefaultVisibility()) + .add(new Features()) + .add(new DefaultCompatibleWith()) + .add(new DefaultRestrictedTo()); + + for (EnvironmentExtension extension : environmentExtensions) { + arguments.addAll(extension.getPackageArguments()); + } + + ImmutableMap.Builder<String, PackageArgument<?>> packageArguments = ImmutableMap.builder(); + for (PackageArgument<?> argument : arguments.build()) { + packageArguments.put(argument.getName(), argument); + } + return packageArguments.build(); + } + + /**************************************************************************** + * Environment function factories. + */ + + /** + * Returns a function-value implementing "glob" in the specified package + * context. + * + * @param async if true, start globs in the background but don't block on their completion. + * Only use this for heuristic preloading. + */ + private static Function newGlobFunction( + final PackageContext originalContext, final boolean async) { + List<String> params = ImmutableList.of("include", "exclude", "exclude_directories"); + return new MixedModeFunction("glob", params, 1, false) { + @Override + public Object call(Object[] namedArguments, FuncallExpression ast, Environment env) + throws EvalException, ConversionException, InterruptedException { + + // Skylark build extensions need to get the PackageContext from the Environment; + // async glob functions cannot do the same because the Environment is not thread safe. + PackageContext context; + if (originalContext == null) { + Preconditions.checkArgument(!async); + try { + context = (PackageContext) env.lookup(PKG_CONTEXT); + } catch (NoSuchVariableException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + } else { + context = originalContext; + } + + List<String> includes = Type.STRING_LIST.convert(namedArguments[0], "'glob' argument"); + List<String> excludes = namedArguments[1] == null + ? Collections.<String>emptyList() + : Type.STRING_LIST.convert(namedArguments[1], "'glob' argument"); + int excludeDirs = namedArguments[2] == null + ? EXCLUDE_DIR_DEFAULT + : Type.INTEGER.convert(namedArguments[2], "'glob' argument"); + + if (async) { + try { + context.globber.runAsync(includes, excludes, excludeDirs != 0); + } catch (GlobCache.BadGlobException e) { + // Ignore: errors will appear during the actual evaluation of the package. + } + return GlobList.captureResults(includes, excludes, ImmutableList.<String>of()); + } else { + return handleGlob(includes, excludes, excludeDirs != 0, context, ast); + } + } + }; + } + + /** + * Adds a glob to the package, reporting any errors it finds. + * + * @param includes the list of includes which must be non-null + * @param excludes the list of excludes which must be non-null + * @param context the package context + * @param ast the AST + * @return the list of matches + * @throws EvalException if globbing failed + */ + private static GlobList<String> handleGlob(List<String> includes, List<String> excludes, + boolean excludeDirs, PackageContext context, FuncallExpression ast) + throws EvalException, InterruptedException { + try { + Globber.Token globToken = context.globber.runAsync(includes, excludes, excludeDirs); + List<String> matches = context.globber.fetch(globToken); + return GlobList.captureResults(includes, excludes, matches); + } catch (IOException expected) { + context.eventHandler.handle(Event.error(ast.getLocation(), + "error globbing [" + Joiner.on(", ").join(includes) + "]: " + expected.getMessage())); + context.pkgBuilder.setContainsTemporaryErrors(); + return GlobList.captureResults(includes, excludes, ImmutableList.<String>of()); + } catch (GlobCache.BadGlobException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + } + + /** + * Returns a function value implementing the "mocksubinclude" function, + * emitted by the PythonPreprocessor. We annotate the + * package with additional dependencies. (A 'real' subinclude will never be + * seen by the parser, because the presence of "subinclude" triggers + * preprocessing.) + */ + private static Function newMockSubincludeFunction(final PackageContext context) { + return new MixedModeFunction("mocksubinclude", ImmutableList.of("label", "path"), 2, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) + throws ConversionException { + Label label = Type.LABEL.convert(args[0], "'mocksubinclude' argument", + context.pkgBuilder.getBuildFileLabel()); + String pathString = Type.STRING.convert(args[1], "'mocksubinclude' argument"); + Path path = pathString.isEmpty() + ? null + : context.pkgBuilder.getFilename().getRelative(pathString); + // A subinclude within a package counts as a file declaration. + if (label.getPackageIdentifier().equals(context.pkgBuilder.getPackageIdentifier())) { + Location location = ast.getLocation(); + if (location == null) { + location = Location.fromFile(context.pkgBuilder.getFilename()); + } + context.pkgBuilder.createInputFileMaybe(label, location); + } + + context.pkgBuilder.addSubinclude(label, path); + return Environment.NONE; + } + }; + } + + /** + * Fake function: subinclude calls are ignored + * They will disappear after the Python preprocessing. + */ + private static Function newSubincludeFunction() { + return new MixedModeFunction("subinclude", ImmutableList.of("file"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) { + return Environment.NONE; + } + }; + } + + /** + * Returns a function value implementing "environment_group" in the specified package context. + * Syntax is as follows: + * + * <pre>{@code + * environment_group( + * name = "sample_group", + * environments = [":env1", ":env2", ...], + * defaults = [":env1", ...] + * ) + * }</pre> + * + * <p>Where ":env1", "env2", ... are all environment rules declared in the same package. All + * parameters are mandatory. + */ + private static Function newEnvironmentGroupFunction(final PackageContext context) { + List<String> params = ImmutableList.of("name", "environments", "defaults"); + return new MixedModeFunction("environment_group", params, params.size(), true) { + @Override + public Object call(Object[] namedArgs, FuncallExpression ast) + throws EvalException, ConversionException { + Preconditions.checkState(namedArgs[0] != null); + String name = Type.STRING.convert(namedArgs[0], "'environment_group' argument"); + Preconditions.checkState(namedArgs[1] != null); + List<Label> environments = Type.LABEL_LIST.convert( + namedArgs[1], "'environment_group argument'", context.pkgBuilder.getBuildFileLabel()); + Preconditions.checkState(namedArgs[2] != null); + List<Label> defaults = Type.LABEL_LIST.convert( + namedArgs[2], "'environment_group argument'", context.pkgBuilder.getBuildFileLabel()); + + try { + context.pkgBuilder.addEnvironmentGroup(name, environments, defaults, + context.eventHandler, ast.getLocation()); + return Environment.NONE; + } catch (Label.SyntaxException e) { + throw new EvalException(ast.getLocation(), + "environment group has invalid name: " + name + ": " + e.getMessage()); + } catch (Package.NameConflictException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + } + }; + } + + /** + * Returns a function-value implementing "exports_files" in the specified + * package context. + */ + private static Function newExportsFilesFunction(final PackageContext context) { + final Package.LegacyBuilder pkgBuilder = context.pkgBuilder; + List<String> params = ImmutableList.of("srcs", "visibility", "licenses"); + return new MixedModeFunction("exports_files", params, 1, false) { + @Override + public Object call(Object[] namedArgs, FuncallExpression ast) + throws EvalException, ConversionException { + + List<String> files = Type.STRING_LIST.convert(namedArgs[0], "'exports_files' operand"); + + RuleVisibility visibility = namedArgs[1] == null + ? ConstantRuleVisibility.PUBLIC + : getVisibility(Type.LABEL_LIST.convert( + namedArgs[1], + "'exports_files' operand", + pkgBuilder.getBuildFileLabel())); + License license = namedArgs[2] == null + ? null + : Type.LICENSE.convert(namedArgs[2], "'exports_files' operand"); + + for (String file : files) { + String errorMessage = LabelValidator.validateTargetName(file); + if (errorMessage != null) { + throw new EvalException(ast.getLocation(), errorMessage); + } + try { + InputFile inputFile = pkgBuilder.createInputFile(file, ast.getLocation()); + if (inputFile.isVisibilitySpecified() + && inputFile.getVisibility() != visibility) { + throw new EvalException(ast.getLocation(), + String.format("visibility for exported file '%s' declared twice", + inputFile.getName())); + } + if (license != null && inputFile.isLicenseSpecified()) { + throw new EvalException(ast.getLocation(), + String.format("licenses for exported file '%s' declared twice", + inputFile.getName())); + } + if (license == null && pkgBuilder.getDefaultLicense() == License.NO_LICENSE + && pkgBuilder.getBuildFileLabel().toString().startsWith("//third_party/")) { + throw new EvalException(ast.getLocation(), + "third-party file '" + inputFile.getName() + "' lacks a license declaration " + + "with one of the following types: notice, reciprocal, permissive, " + + "restricted, unencumbered, by_exception_only"); + } + + pkgBuilder.setVisibilityAndLicense(inputFile, visibility, license); + } catch (Package.Builder.GeneratedLabelConflict e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + } + return Environment.NONE; + } + }; + } + + /** + * Returns a function-value implementing "licenses" in the specified package + * context. + * TODO(bazel-team): Remove in favor of package.licenses. + */ + private static Function newLicensesFunction(final PackageContext context) { + return new MixedModeFunction("licenses", ImmutableList.of("object"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) { + try { + License license = Type.LICENSE.convert(args[0], "'licenses' operand"); + context.pkgBuilder.setDefaultLicense(license); + } catch (ConversionException e) { + context.eventHandler.handle(Event.error(ast.getLocation(), e.getMessage())); + context.pkgBuilder.setContainsErrors(); + } + return Environment.NONE; + } + }; + } + + /** + * Returns a function-value implementing "distribs" in the specified package + * context. + * TODO(bazel-team): Remove in favor of package.distribs. + */ + private static Function newDistribsFunction(final PackageContext context) { + return new MixedModeFunction("distribs", ImmutableList.of("object"), 1, false) { + @Override + public Object call(Object[] args, FuncallExpression ast) { + try { + Set<DistributionType> distribs = Type.DISTRIBUTIONS.convert(args[0], + "'distribs' operand"); + context.pkgBuilder.setDefaultDistribs(distribs); + } catch (ConversionException e) { + context.eventHandler.handle(Event.error(ast.getLocation(), e.getMessage())); + context.pkgBuilder.setContainsErrors(); + } + return Environment.NONE; + } + }; + } + + private static Function newPackageGroupFunction(final PackageContext context) { + List<String> params = ImmutableList.of("name", "packages", "includes"); + return new MixedModeFunction("package_group", params, 1, true) { + @Override + public Object call(Object[] namedArgs, FuncallExpression ast) + throws EvalException, ConversionException { + Preconditions.checkState(namedArgs[0] != null); + String name = Type.STRING.convert(namedArgs[0], "'package_group' argument"); + List<String> packages = namedArgs[1] == null + ? Collections.<String>emptyList() + : Type.STRING_LIST.convert(namedArgs[1], "'package_group' argument"); + List<Label> includes = namedArgs[2] == null + ? Collections.<Label>emptyList() + : Type.LABEL_LIST.convert(namedArgs[2], "'package_group argument'", + context.pkgBuilder.getBuildFileLabel()); + + try { + context.pkgBuilder.addPackageGroup(name, packages, includes, context.eventHandler, + ast.getLocation()); + return Environment.NONE; + } catch (Label.SyntaxException e) { + throw new EvalException(ast.getLocation(), + "package group has invalid name: " + name + ": " + e.getMessage()); + } catch (Package.NameConflictException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + } + }; + } + + public static RuleVisibility getVisibility(List<Label> original) { + RuleVisibility result; + + result = ConstantRuleVisibility.tryParse(original); + if (result != null) { + return result; + } + + result = PackageGroupsRuleVisibility.tryParse(original); + return result; + } + + /** + * Returns a function-value implementing "package" in the specified package + * context. + */ + private static Function newPackageFunction( + final Map<String, PackageArgument<?>> packageArguments) { + return new MixedModeFunction("package", packageArguments.keySet(), 0, true) { + @Override + public Object call(Object[] namedArguments, FuncallExpression ast, Environment env) + throws EvalException, ConversionException { + + Package.LegacyBuilder pkgBuilder = getContext(env, ast).pkgBuilder; + + // Validate parameter list + if (pkgBuilder.isPackageFunctionUsed()) { + throw new EvalException(ast.getLocation(), + "'package' can only be used once per BUILD file"); + } + pkgBuilder.setPackageFunctionUsed(); + + // Parse params + boolean foundParameter = false; + + int argNumber = 0; + for (Map.Entry<String, PackageArgument<?>> entry : packageArguments.entrySet()) { + Object arg = namedArguments[argNumber]; + argNumber += 1; + if (arg == null) { + continue; + } + + foundParameter = true; + entry.getValue().convertAndProcess(pkgBuilder, ast.getLocation(), arg); + } + + if (!foundParameter) { + throw new EvalException(ast.getLocation(), + "at least one argument must be given to the 'package' function"); + } + + return Environment.NONE; + } + }; + } + + // Helper function for createRuleFunction. + private static void addRule(RuleFactory ruleFactory, + String ruleClassName, + PackageContext context, + Map<String, Object> kwargs, + FuncallExpression ast) + throws RuleFactory.InvalidRuleException, Package.NameConflictException { + RuleClass ruleClass = getBuiltInRuleClass(ruleClassName, ruleFactory); + RuleFactory.createAndAddRule(context, ruleClass, kwargs, ast); + } + + private static RuleClass getBuiltInRuleClass(String ruleClassName, RuleFactory ruleFactory) { + if (ruleFactory.getRuleClassNames().contains(ruleClassName)) { + return ruleFactory.getRuleClass(ruleClassName); + } + throw new IllegalArgumentException("no such rule class: " + ruleClassName); + } + + /** + * Get the PackageContext by looking up in the environment. + */ + private static PackageContext getContext(Environment env, FuncallExpression ast) + throws EvalException { + try { + return (PackageContext) env.lookup(PKG_CONTEXT); + } catch (NoSuchVariableException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + } + + /** + * Returns a function-value implementing the build rule "ruleClass" (e.g. cc_library) in the + * specified package context. + */ + private static Function newRuleFunction(final RuleFactory ruleFactory, + final String ruleClass) { + return new AbstractFunction(ruleClass) { + @Override + public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast, + Environment env) + throws EvalException { + if (!args.isEmpty()) { + throw new EvalException(ast.getLocation(), + "build rules do not accept positional parameters"); + } + + try { + addRule(ruleFactory, ruleClass, getContext(env, ast), kwargs, ast); + } catch (RuleFactory.InvalidRuleException | Package.NameConflictException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + return Environment.NONE; + } + }; + } + + /** + * Returns a new environment populated with common entries that can be shared + * across packages and that don't require the context. + */ + private static Environment newGlobalEnvironment() { + Environment env = new Environment(); + MethodLibrary.setupMethodEnvironment(env); + return env; + } + + /**************************************************************************** + * Package creation. + */ + + /** + * Loads, scans parses and evaluates the build file at "buildFile", and + * creates and returns a Package builder instance capable of building a package identified by + * "packageId". + * + * <p>This method returns a builder to allow the caller to do additional work, if necessary. + * + * <p>This method assumes "packageId" is a valid package name according to the + * {@link LabelValidator#validatePackageName} heuristic. + * + * <p>See {@link #evaluateBuildFile} for information on AST retention. + * + * <p>Executes {@code globber.onCompletion()} on completion and executes + * {@code globber.onInterrupt()} on an {@link InterruptedException}. + */ + private Package.LegacyBuilder createPackage(PackageIdentifier packageId, Path buildFile, + List<Statement> preludeStatements, ParserInputSource inputSource, + Map<PathFragment, SkylarkEnvironment> imports, ImmutableList<Label> skylarkFileDependencies, + CachingPackageLocator locator, RuleVisibility defaultVisibility, Globber globber) + throws InterruptedException { + StoredEventHandler localReporter = new StoredEventHandler(); + Preprocessor.Result preprocessingResult = preprocess(packageId, buildFile, inputSource, globber, + localReporter); + return createPackageFromPreprocessingResult(packageId, buildFile, preprocessingResult, + localReporter.getEvents(), preludeStatements, imports, skylarkFileDependencies, locator, + defaultVisibility, globber); + } + + /** + * Same as {@link #createPackage}, but using a {@link Preprocessor.Result} from + * {@link #preprocess}. + * + * <p>Executes {@code globber.onCompletion()} on completion and executes + * {@code globber.onInterrupt()} on an {@link InterruptedException}. + */ + // Used outside of bazel! + public Package.LegacyBuilder createPackageFromPreprocessingResult(PackageIdentifier packageId, + Path buildFile, + Preprocessor.Result preprocessingResult, + Iterable<Event> preprocessingEvents, + List<Statement> preludeStatements, + Map<PathFragment, SkylarkEnvironment> imports, + ImmutableList<Label> skylarkFileDependencies, + CachingPackageLocator locator, + RuleVisibility defaultVisibility, + Globber globber) throws InterruptedException { + StoredEventHandler localReporter = new StoredEventHandler(); + // Run the lexer and parser with a local reporter, so that errors from other threads do not + // show up below. Merge the local and global reporters afterwards. + // Logged messages are used as a testability hook tracing the parsing progress + LOG.fine("Starting to parse " + packageId); + BuildFileAST buildFileAST = BuildFileAST.parseBuildFile( + preprocessingResult.result, preludeStatements, localReporter, locator, false); + LOG.fine("Finished parsing of " + packageId); + + MakeEnvironment.Builder makeEnv = new MakeEnvironment.Builder(); + if (platformSetRegexps != null) { + makeEnv.setPlatformSetRegexps(platformSetRegexps); + } + try { + // At this point the package is guaranteed to exist. It may have parse or + // evaluation errors, resulting in a diminished number of rules. + prefetchGlobs(packageId, buildFileAST, preprocessingResult.preprocessed, + buildFile, globber, defaultVisibility, makeEnv); + return evaluateBuildFile( + packageId, buildFileAST, buildFile, globber, + Iterables.concat(preprocessingEvents, localReporter.getEvents()), + defaultVisibility, preprocessingResult.containsErrors, + preprocessingResult.containsTransientErrors, makeEnv, imports, skylarkFileDependencies); + } catch (InterruptedException e) { + globber.onInterrupt(); + throw e; + } finally { + globber.onCompletion(); + } + } + + /** + * Same as {@link #createPackage}, but does the required validation of "packageName" first, + * throwing a {@link NoSuchPackageException} if the name is invalid. + */ + @VisibleForTesting + public Package createPackageForTesting(PackageIdentifier packageId, + Path buildFile, + CachingPackageLocator locator, + EventHandler eventHandler) throws NoSuchPackageException, InterruptedException { + String error = LabelValidator.validatePackageName( + packageId.getPackageFragment().getPathString()); + if (error != null) { + throw new BuildFileNotFoundException(packageId.toString(), + "illegal package name: '" + packageId.toString() + "' (" + error + ")"); + } + ParserInputSource inputSource = maybeGetParserInputSource(buildFile, eventHandler); + if (inputSource == null) { + throw new BuildFileContainsErrorsException(packageId.toString(), "IOException occured"); + } + Package result = createPackage(packageId, buildFile, + ImmutableList.<Statement>of(), inputSource, + ImmutableMap.<PathFragment, SkylarkEnvironment>of(), + ImmutableList.<Label>of(), + locator, ConstantRuleVisibility.PUBLIC, + createLegacyGlobber(buildFile.getParentDirectory(), packageId, locator)).build(); + Event.replayEventsOn(eventHandler, result.getEvents()); + return result; + } + + /** Preprocesses the given BUILD file. */ + // Used outside of bazel! + public Preprocessor.Result preprocess( + PackageIdentifier packageId, + Path buildFile, + CachingPackageLocator locator, + EventHandler eventHandler) throws InterruptedException { + ParserInputSource inputSource = maybeGetParserInputSource(buildFile, eventHandler); + if (inputSource == null) { + return Preprocessor.Result.transientError(buildFile); + } + Globber globber = createLegacyGlobber(buildFile.getParentDirectory(), packageId, locator); + try { + return preprocess(packageId, buildFile, inputSource, globber, eventHandler); + } finally { + globber.onCompletion(); + } + } + + /** + * Preprocesses the given BUILD file, executing {@code globber.onInterrupt()} on an + * {@link InterruptedException}. + */ + // Used outside of bazel! + public Preprocessor.Result preprocess( + PackageIdentifier packageId, + Path buildFile, + ParserInputSource inputSource, + Globber globber, + EventHandler eventHandler) throws InterruptedException { + Preprocessor preprocessor = preprocessorFactory.getPreprocessor(); + if (preprocessor == null) { + return Preprocessor.Result.noPreprocessing(inputSource); + } + try { + return preprocessor.preprocess(inputSource, packageId.toString(), globber, eventHandler, + globalEnv, ruleFactory.getRuleClassNames()); + } catch (IOException e) { + eventHandler.handle(Event.error(Location.fromFile(buildFile), + "preprocessing failed: " + e.getMessage())); + return Preprocessor.Result.transientError(buildFile); + } catch (InterruptedException e) { + globber.onInterrupt(); + throw e; + } + } + + // Used outside of bazel! + public LegacyGlobber createLegacyGlobber(Path packageDirectory, PackageIdentifier packageId, + CachingPackageLocator locator) { + return new LegacyGlobber(new GlobCache(packageDirectory, packageId, locator, syscalls, + threadPool)); + } + + @Nullable + private ParserInputSource maybeGetParserInputSource(Path buildFile, EventHandler eventHandler) { + try { + return ParserInputSource.create(buildFile); + } catch (IOException e) { + eventHandler.handle(Event.error(Location.fromFile(buildFile), e.getMessage())); + return null; + } + } + + /** + * This tuple holds the current package builder, current lexer, etc, for the + * duration of the evaluation of one BUILD file. (We use a PackageContext + * object in preference to storing these values in mutable fields of the + * PackageFactory.) + * + * <p>PLEASE NOTE: references to PackageContext objects are held by many + * Function closures, but should become unreachable once the Environment is + * discarded at the end of evaluation. Please be aware of your memory + * footprint when making changes here! + */ + public static class PackageContext { + + final Package.LegacyBuilder pkgBuilder; + final Globber globber; + final EventHandler eventHandler; + + @VisibleForTesting + public PackageContext(Package.LegacyBuilder pkgBuilder, Globber globber, + EventHandler eventHandler) { + this.pkgBuilder = pkgBuilder; + this.eventHandler = eventHandler; + this.globber = globber; + } + } + + /** + * Returns the list of native rule functions created using the {@link RuleClassProvider} + * of this {@link PackageFactory}. + */ + public ImmutableList<Function> collectNativeRuleFunctions() { + ImmutableList.Builder<Function> builder = ImmutableList.builder(); + for (String ruleClass : ruleFactory.getRuleClassNames()) { + builder.add(newRuleFunction(ruleFactory, ruleClass)); + } + builder.add(newGlobFunction(null, false)); + builder.add(newPackageFunction(packageArguments)); + return builder.build(); + } + + private void buildPkgEnv(Environment pkgEnv, String packageName, + MakeEnvironment.Builder pkgMakeEnv, PackageContext context, RuleFactory ruleFactory) { + pkgEnv.update("distribs", newDistribsFunction(context)); + pkgEnv.update("glob", newGlobFunction(context, /*async=*/false)); + pkgEnv.update("mocksubinclude", newMockSubincludeFunction(context)); + pkgEnv.update("licenses", newLicensesFunction(context)); + pkgEnv.update("exports_files", newExportsFilesFunction(context)); + pkgEnv.update("package_group", newPackageGroupFunction(context)); + pkgEnv.update("package", newPackageFunction(packageArguments)); + pkgEnv.update("subinclude", newSubincludeFunction()); + pkgEnv.update("environment_group", newEnvironmentGroupFunction(context)); + + pkgEnv.update("PACKAGE_NAME", packageName); + + for (String ruleClass : ruleFactory.getRuleClassNames()) { + Function ruleFunction = newRuleFunction(ruleFactory, ruleClass); + pkgEnv.update(ruleClass, ruleFunction); + } + + for (EnvironmentExtension extension : environmentExtensions) { + extension.update(pkgEnv, pkgMakeEnv, context.pkgBuilder.getBuildFileLabel()); + } + } + + /** + * Constructs a Package instance, evaluates the BUILD-file AST inside the + * build environment, and populates the package with Rule instances as it + * goes. As with most programming languages, evaluation stops when an + * exception is encountered: no further rules after the point of failure will + * be constructed. We assume that rules constructed before the point of + * failure are valid; this assumption is not entirely correct, since a + * "vardef" after a rule declaration can affect the behavior of that rule. + * + * <p>Rule attribute checking is performed during evaluation. Each attribute + * must conform to the type specified for that <i>(rule class, attribute + * name)</i> pair. Errors reported at this stage include: missing value for + * mandatory attribute, value of wrong type. Such error cause Rule + * construction to be aborted, so the resulting package will have missing + * members. + * + * @see PackageFactory#PackageFactory + */ + @VisibleForTesting // used by PackageFactoryApparatus + public Package.LegacyBuilder evaluateBuildFile(PackageIdentifier packageId, + BuildFileAST buildFileAST, Path buildFilePath, Globber globber, + Iterable<Event> pastEvents, RuleVisibility defaultVisibility, boolean containsError, + boolean containsTransientError, MakeEnvironment.Builder pkgMakeEnv, + Map<PathFragment, SkylarkEnvironment> imports, + ImmutableList<Label> skylarkFileDependencies) throws InterruptedException { + // Important: Environment should be unreachable by the end of this method! + StoredEventHandler eventHandler = new StoredEventHandler(); + Environment pkgEnv = new Environment(globalEnv, eventHandler); + + Package.LegacyBuilder pkgBuilder = + new Package.LegacyBuilder(packageId) + .setGlobber(globber) + .setFilename(buildFilePath) + .setMakeEnv(pkgMakeEnv) + .setDefaultVisibility(defaultVisibility) + // "defaultVisibility" comes from the command line. Let's give the BUILD file a chance to + // set default_visibility once, be reseting the PackageBuilder.defaultVisibilitySet flag. + .setDefaultVisibilitySet(false) + .setSkylarkFileDependencies(skylarkFileDependencies); + + Event.replayEventsOn(eventHandler, pastEvents); + + // Stuff that closes over the package context: + PackageContext context = new PackageContext(pkgBuilder, globber, eventHandler); + buildPkgEnv(pkgEnv, packageId.toString(), pkgMakeEnv, context, ruleFactory); + + if (containsError) { + pkgBuilder.setContainsErrors(); + } + + if (containsTransientError) { + pkgBuilder.setContainsTemporaryErrors(); + } + + if (!validatePackageIdentifier(packageId, buildFileAST.getLocation(), eventHandler)) { + pkgBuilder.setContainsErrors(); + } + + pkgEnv.setImportedExtensions(imports); + pkgEnv.updateAndPropagate(PKG_CONTEXT, context); + pkgEnv.updateAndPropagate(Environment.PKG_NAME, packageId.toString()); + + if (!validateAssignmentStatements(pkgEnv, buildFileAST, eventHandler)) { + pkgBuilder.setContainsErrors(); + } + + if (buildFileAST.containsErrors()) { + pkgBuilder.setContainsErrors(); + } + + // TODO(bazel-team): (2009) the invariant "if errors are reported, mark the package + // as containing errors" is strewn all over this class. Refactor to use an + // event sensor--and see if we can simplify the calling code in + // createPackage(). + if (!buildFileAST.exec(pkgEnv, eventHandler)) { + pkgBuilder.setContainsErrors(); + } + + pkgBuilder.addEvents(eventHandler.getEvents()); + return pkgBuilder; + } + + /** + * Visit all targets and expand the globs in parallel. + */ + private void prefetchGlobs(PackageIdentifier packageId, BuildFileAST buildFileAST, + boolean wasPreprocessed, Path buildFilePath, Globber globber, + RuleVisibility defaultVisibility, MakeEnvironment.Builder pkgMakeEnv) + throws InterruptedException { + if (wasPreprocessed) { + // No point in prefetching globs here: preprocessing implies eager evaluation + // of all globs. + return; + } + // Important: Environment should be unreachable by the end of this method! + Environment pkgEnv = new Environment(); + + Package.LegacyBuilder pkgBuilder = + new Package.LegacyBuilder(packageId) + .setFilename(buildFilePath) + .setMakeEnv(pkgMakeEnv) + .setDefaultVisibility(defaultVisibility) + // "defaultVisibility" comes from the command line. Let's give the BUILD file a chance to + // set default_visibility once, be reseting the PackageBuilder.defaultVisibilitySet flag. + .setDefaultVisibilitySet(false); + + // Stuff that closes over the package context: + PackageContext context = new PackageContext(pkgBuilder, globber, NullEventHandler.INSTANCE); + buildPkgEnv(pkgEnv, packageId.toString(), pkgMakeEnv, context, ruleFactory); + pkgEnv.update("glob", newGlobFunction(context, /*async=*/true)); + // The Fileset function is heavyweight in that it can run glob(). Avoid this during the + // preloading phase. + pkgEnv.remove("FilesetEntry"); + + buildFileAST.exec(pkgEnv, NullEventHandler.INSTANCE); + } + + + /** + * Tests a build AST to ensure that it contains no assignment statements that + * redefine built-in build rules. + * + * @param pkgEnv a package environment initialized with all of the built-in + * build rules + * @param ast the build file AST to be tested + * @param eventHandler a eventHandler where any errors should be logged + * @return true if the build file contains no redefinitions of built-in + * functions + */ + private static boolean validateAssignmentStatements(Environment pkgEnv, + BuildFileAST ast, + EventHandler eventHandler) { + for (Statement stmt : ast.getStatements()) { + if (stmt instanceof AssignmentStatement) { + Expression lvalue = ((AssignmentStatement) stmt).getLValue(); + if (!(lvalue instanceof Ident)) { + continue; + } + String target = ((Ident) lvalue).getName(); + if (pkgEnv.lookup(target, null) != null) { + eventHandler.handle(Event.error(stmt.getLocation(), "Reassignment of builtin build " + + "function '" + target + "' not permitted")); + return false; + } + } + } + return true; + } + + // Reports an error and returns false iff package identifier was illegal. + private static boolean validatePackageIdentifier(PackageIdentifier packageId, Location location, + EventHandler eventHandler) { + String error = LabelValidator.validatePackageName(packageId.getPackageFragment().toString()); + if (error != null) { + eventHandler.handle(Event.error(location, error)); + return false; // Invalid package name 'foo' + } + return true; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageGroup.java b/src/main/java/com/google/devtools/build/lib/packages/PackageGroup.java new file mode 100644 index 0000000..17ba29c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/PackageGroup.java
@@ -0,0 +1,152 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.License.DistributionType; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * This class represents a package group. It has a name and a set of packages + * and can be asked if a specific package is included in it. The package set is + * represented as a list of PathFragments. + */ +public class PackageGroup implements Target { + private boolean containsErrors; + private final Label label; + private final Location location; + private final Package containingPackage; + private final List<PackageSpecification> packageSpecifications; + private final List<Label> includes; + + public PackageGroup(Label label, Package pkg, Collection<String> packages, + Collection<Label> includes, EventHandler eventHandler, Location location) { + this.label = label; + this.location = location; + this.containingPackage = pkg; + this.includes = ImmutableList.copyOf(includes); + + ImmutableList.Builder<PackageSpecification> packagesBuilder = ImmutableList.builder(); + for (String containedPackage : packages) { + PackageSpecification specification = null; + try { + specification = PackageSpecification.fromString(containedPackage); + } catch (PackageSpecification.InvalidPackageSpecificationException e) { + containsErrors = true; + eventHandler.handle(Event.error(location, e.getMessage())); + } + + if (specification != null) { + packagesBuilder.add(specification); + } + } + this.packageSpecifications = packagesBuilder.build(); + } + + public boolean containsErrors() { + return containsErrors; + } + + public Iterable<PackageSpecification> getPackageSpecifications() { + return packageSpecifications; + } + + public boolean contains(Package pkg) { + for (PackageSpecification specification : packageSpecifications) { + if (specification.containsPackage(pkg.getNameFragment())) { + return true; + } + } + + return false; + } + + public List<Label> getIncludes() { + return includes; + } + + public List<String> getContainedPackages() { + List<String> result = Lists.newArrayListWithCapacity(packageSpecifications.size()); + for (PackageSpecification specification : packageSpecifications) { + result.add(specification.toString()); + } + return result; + } + + @Override + public Rule getAssociatedRule() { + return null; + } + + @Override + public Set<DistributionType> getDistributions() { + return Collections.emptySet(); + } + + @Override + public Label getLabel() { + return label; + } + + @Override public String getName() { + return label.getName(); + } + + @Override + public License getLicense() { + return License.NO_LICENSE; + } + + @Override + public Package getPackage() { + return containingPackage; + } + + @Override + public String getTargetKind() { + return targetKind(); + } + + @Override + public Location getLocation() { + return location; + } + + @Override + public String toString() { + return targetKind() + " " + getLabel(); + } + + @Override + public RuleVisibility getVisibility() { + // Package groups are always public to avoid a PackageGroupConfiguredTarget + // needing itself for the visibility check. It may work, but I did not + // think it over completely. + return ConstantRuleVisibility.PUBLIC; + } + + public static String targetKind() { + return "package group"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageGroupsRuleVisibility.java b/src/main/java/com/google/devtools/build/lib/packages/PackageGroupsRuleVisibility.java new file mode 100644 index 0000000..70ffa11 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/PackageGroupsRuleVisibility.java
@@ -0,0 +1,81 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Collection; +import java.util.List; + +/** + * A rule visibility that allows visibility to a list of package groups. + */ +@Immutable @ThreadSafe +public class PackageGroupsRuleVisibility implements RuleVisibility { + public static final String PACKAGE_LABEL = "__pkg__"; + public static final String SUBTREE_LABEL = "__subpackages__"; + private final List<Label> packageGroups; + private final List<PackageSpecification> directPackages; + private final List<Label> declaredLabels; + + public PackageGroupsRuleVisibility(List<Label> labels) { + declaredLabels = ImmutableList.copyOf(labels); + ImmutableList.Builder<PackageSpecification> directPackageBuilder = ImmutableList.builder(); + ImmutableList.Builder<Label> packageGroupBuilder = ImmutableList.builder(); + + for (Label label : labels) { + PackageSpecification specification = PackageSpecification.fromLabel(label); + if (specification != null) { + directPackageBuilder.add(specification); + } else { + packageGroupBuilder.add(label); + } + } + + packageGroups = packageGroupBuilder.build(); + directPackages = directPackageBuilder.build(); + } + + public Collection<Label> getPackageGroups() { + return packageGroups; + } + + public Collection<PackageSpecification> getDirectPackages() { + return directPackages; + } + + @Override + public List<Label> getDependencyLabels() { + return packageGroups; + } + + @Override + public List<Label> getDeclaredLabels() { + return declaredLabels; + } + + /** + * Tries to parse a list of labels into a {@link PackageGroupsRuleVisibility}. + * + * @param labels the list of labels to parse + * @return The resulting visibility object. A list of labels can always be + * parsed into a PackageGroupsRuleVisibility. + */ + public static PackageGroupsRuleVisibility tryParse(List<Label> labels) { + return new PackageGroupsRuleVisibility(labels); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageIdentifier.java b/src/main/java/com/google/devtools/build/lib/packages/PackageIdentifier.java new file mode 100644 index 0000000..7b16e3c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/PackageIdentifier.java
@@ -0,0 +1,271 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ComparisonChain; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.util.StringCanonicalizer; +import com.google.devtools.build.lib.util.StringUtilities; +import com.google.devtools.build.lib.vfs.Canonicalizer; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.ObjectStreamException; +import java.io.Serializable; +import java.util.Objects; + +import javax.annotation.concurrent.Immutable; + +/** + * Uniquely identifies a package, given a repository name and a package's path fragment. + * + * <p>The repository the build is happening in is the <i>default workspace</i>, and is identified + * by the workspace name "". Other repositories can be named in the WORKSPACE file. These + * workspaces are prefixed by {@literal @}.</p> + */ +@Immutable +public final class PackageIdentifier implements Comparable<PackageIdentifier>, Serializable { + + /** + * A human-readable name for the repository. + */ + public static final class RepositoryName { + private final String name; + + /** + * Makes sure that name is a valid repository name and creates a new RepositoryName using it. + * @throws SyntaxException if the name is invalid. + */ + public static RepositoryName create(String name) throws SyntaxException { + String errorMessage = validate(name); + if (errorMessage != null) { + errorMessage = "invalid repository name '" + + StringUtilities.sanitizeControlChars(name) + "': " + errorMessage; + throw new SyntaxException(errorMessage); + } + return new RepositoryName(StringCanonicalizer.intern(name)); + } + + private RepositoryName(String name) { + this.name = name; + } + + /** + * Performs validity checking. Returns null on success, an error message otherwise. + */ + private static String validate(String name) { + if (name.isEmpty()) { + return null; + } + + if (!name.startsWith("@")) { + return "workspace name must start with '@'"; + } + + // "@" isn't a valid workspace name. + if (name.length() == 1) { + return "empty workspace name"; + } + + // Check for any character outside of [/0-9A-Z_a-z-]. Try to evaluate the + // conditional quickly (by looking in decreasing order of character class + // likelihood). + for (int i = name.length() - 1; i >= 1; --i) { + char c = name.charAt(i); + if ((c < 'a' || c > 'z') && c != '_' && c != '-' + && (c < '0' || c > '9') && (c < 'A' || c > 'Z')) { + return "workspace names may contain only A-Z, a-z, 0-9, '-' and '_'"; + } + } + return null; + } + + /** + * Returns the repository name without the leading "{@literal @}". For the default repository, + * returns "". + */ + public String strippedName() { + if (name.isEmpty()) { + return name; + } + return name.substring(1); + } + + /** + * Returns if this is the default repository, that is, {@link #name} is "". + */ + public boolean isDefault() { + return name.isEmpty(); + } + + /** + * Returns the repository name, with leading "{@literal @}" (or "" for the default repository). + */ + @Override + public String toString() { + return name; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object instanceof RepositoryName) { + return name.equals(((RepositoryName) object).name); + } + return false; + } + + @Override + public int hashCode() { + return name.hashCode(); + } + } + + public static final String DEFAULT_REPOSITORY = ""; + + /** + * Helper for serializing PackageIdentifiers. + * + * <p>PackageIdentifier's field should be final, but then it couldn't be deserialized. This + * allows the fields to be deserialized and copied into a new PackageIdentifier.</p> + */ + private static final class SerializationProxy implements Serializable { + PackageIdentifier packageId; + + public SerializationProxy(PackageIdentifier packageId) { + this.packageId = packageId; + } + + private void writeObject(ObjectOutputStream out) throws IOException { + out.writeObject(packageId.repository.toString()); + out.writeObject(packageId.pkgName); + } + + private void readObject(ObjectInputStream in) + throws IOException, ClassNotFoundException { + try { + packageId = new PackageIdentifier((String) in.readObject(), (PathFragment) in.readObject()); + } catch (SyntaxException e) { + throw new IOException("Error serializing package identifier: " + e.getMessage()); + } + } + + @SuppressWarnings("unused") + private void readObjectNoData() throws ObjectStreamException { + } + + private Object readResolve() { + return packageId; + } + } + + // Temporary factory for identifiers without explicit repositories. + // TODO(bazel-team): remove all usages of this. + public static PackageIdentifier createInDefaultRepo(String name) { + return createInDefaultRepo(new PathFragment(name)); + } + + public static PackageIdentifier createInDefaultRepo(PathFragment name) { + try { + return new PackageIdentifier(DEFAULT_REPOSITORY, name); + } catch (SyntaxException e) { + throw new IllegalArgumentException("could not create package identifier for " + name + + ": " + e.getMessage()); + } + } + + /** + * The identifier for this repository. This is either "" or prefixed with an "@", + * e.g., "@myrepo". + */ + private final RepositoryName repository; + + /** The name of the package. Canonical (i.e. x.equals(y) <=> x==y). */ + private final PathFragment pkgName; + + public PackageIdentifier(String repository, PathFragment pkgName) throws SyntaxException { + this(RepositoryName.create(repository), pkgName); + } + + public PackageIdentifier(RepositoryName repository, PathFragment pkgName) { + Preconditions.checkNotNull(repository); + Preconditions.checkNotNull(pkgName); + this.repository = repository; + this.pkgName = Canonicalizer.fragments().intern(pkgName); + } + + private Object writeReplace() throws ObjectStreamException { + return new SerializationProxy(this); + } + + private void readObject(ObjectInputStream in) + throws IOException, ClassNotFoundException { + throw new IOException("Serialization is allowed only by proxy"); + } + + @SuppressWarnings("unused") + private void readObjectNoData() throws ObjectStreamException { + } + + public RepositoryName getRepository() { + return repository; + } + + public PathFragment getPackageFragment() { + return pkgName; + } + + /** + * Returns the name of this package. + * + * <p>There are certain places that expect the path fragment as the package name ('foo/bar') as a + * package identifier. This isn't specific enough for packages in other repositories, so their + * stringified version is '@baz//foo/bar'.</p> + */ + @Override + public String toString() { + return (repository.isDefault() ? "" : repository + "//") + pkgName; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object instanceof PackageIdentifier) { + PackageIdentifier that = (PackageIdentifier) object; + return repository.equals(that.repository) && pkgName.equals(that.pkgName); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(repository, pkgName); + } + + @Override + public int compareTo(PackageIdentifier that) { + return ComparisonChain.start() + .compare(repository.toString(), that.repository.toString()) + .compare(pkgName, that.pkgName) + .result(); + } +} \ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageLoadedEvent.java b/src/main/java/com/google/devtools/build/lib/packages/PackageLoadedEvent.java new file mode 100644 index 0000000..3dbf3c2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/PackageLoadedEvent.java
@@ -0,0 +1,60 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +/** + * An event that is fired after a package is loaded. + */ +public final class PackageLoadedEvent { + private final String packageName; + private final long timeInMillis; + private final boolean reloading; + private final boolean successful; + + public PackageLoadedEvent(String packageName, long timeInMillis, boolean reloading, + boolean successful) { + this.packageName = packageName; + this.timeInMillis = timeInMillis; + this.reloading = reloading; + this.successful = successful; + } + + /** + * Returns the package name. + */ + public String getPackageName() { + return packageName; + } + + /** + * Returns time which was spent to load a package. + */ + public long getTimeInMillis() { + return timeInMillis; + } + + /** + * Returns true if package had been loaded before. + */ + public boolean isReloading() { + return reloading; + } + + /** + * Returns true if package was loaded successfully. + */ + public boolean isSuccessful() { + return successful; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageNotInCacheException.java b/src/main/java/com/google/devtools/build/lib/packages/PackageNotInCacheException.java new file mode 100644 index 0000000..f191a1c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/PackageNotInCacheException.java
@@ -0,0 +1,24 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +/** + * Exception indicating that a package is not in the cache, although it should + * have been loaded. + */ +public class PackageNotInCacheException extends NoSuchPackageException { + public PackageNotInCacheException(String packageName) { + super(packageName, "package '" + packageName + "' not in cache"); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageSerializer.java b/src/main/java/com/google/devtools/build/lib/packages/PackageSerializer.java new file mode 100644 index 0000000..8971cf2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/PackageSerializer.java
@@ -0,0 +1,462 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.DISTRIBUTIONS; +import static com.google.devtools.build.lib.packages.Type.FILESET_ENTRY_LIST; +import static com.google.devtools.build.lib.packages.Type.INTEGER; +import static com.google.devtools.build.lib.packages.Type.INTEGER_LIST; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST_DICT; +import static com.google.devtools.build.lib.packages.Type.LICENSE; +import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL; +import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.OUTPUT; +import static com.google.devtools.build.lib.packages.Type.OUTPUT_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING; +import static com.google.devtools.build.lib.packages.Type.STRING_DICT; +import static com.google.devtools.build.lib.packages.Type.STRING_DICT_UNARY; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST_DICT; +import static com.google.devtools.build.lib.packages.Type.TRISTATE; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.License.DistributionType; +import com.google.devtools.build.lib.packages.MakeEnvironment.Binding; +import com.google.devtools.build.lib.query2.proto.proto2api.Build; +import com.google.devtools.build.lib.syntax.FilesetEntry; +import com.google.devtools.build.lib.syntax.GlobCriteria; +import com.google.devtools.build.lib.syntax.GlobList; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * Functionality to serialize loaded packages. + */ +public class PackageSerializer { + private static Build.SourceFile serializeInputFile(InputFile inputFile) { + Build.SourceFile.Builder result = Build.SourceFile.newBuilder(); + result.setName(inputFile.getLabel().toString()); + if (inputFile.isVisibilitySpecified()) { + for (Label visibilityLabel : inputFile.getVisibility().getDeclaredLabels()) { + result.addVisibilityLabel(visibilityLabel.toString()); + } + } + if (inputFile.isLicenseSpecified()) { + result.setLicense(serializeLicense(inputFile.getLicense())); + } + + result.setParseableLocation(serializeLocation(inputFile.getLocation())); + return result.build(); + } + + private static Build.Location serializeLocation(Location location) { + Build.Location.Builder result = Build.Location.newBuilder(); + + result.setStartOffset(location.getStartOffset()); + if (location.getStartLineAndColumn() != null) { + result.setStartLine(location.getStartLineAndColumn().getLine()); + result.setStartColumn(location.getStartLineAndColumn().getColumn()); + } + + result.setEndOffset(location.getEndOffset()); + if (location.getEndLineAndColumn() != null) { + result.setEndLine(location.getEndLineAndColumn().getLine()); + result.setEndColumn(location.getEndLineAndColumn().getColumn()); + } + + return result.build(); + } + + private static Build.PackageGroup serializePackageGroup(PackageGroup packageGroup) { + Build.PackageGroup.Builder result = Build.PackageGroup.newBuilder(); + + result.setName(packageGroup.getLabel().toString()); + result.setParseableLocation(serializeLocation(packageGroup.getLocation())); + + for (PackageSpecification packageSpecification : packageGroup.getPackageSpecifications()) { + result.addContainedPackage(packageSpecification.toString()); + } + + for (Label include : packageGroup.getIncludes()) { + result.addIncludedPackageGroup(include.toString()); + } + + return result.build(); + } + + private static Build.Rule serializeRule(Rule rule) { + Build.Rule.Builder result = Build.Rule.newBuilder(); + result.setName(rule.getLabel().toString()); + result.setRuleClass(rule.getRuleClass()); + result.setParseableLocation(serializeLocation(rule.getLocation())); + for (Attribute attribute : rule.getAttributes()) { + Object value = attribute.getName().equals("visibility") + ? rule.getVisibility().getDeclaredLabels() + // TODO(bazel-team): support configurable attributes. AggregatingAttributeMapper + // may make more sense here. Computed defaults may add complications. + : RawAttributeMapper.of(rule).get(attribute.getName(), attribute.getType()); + if (value != null) { + PackageSerializer.addAttributeToProto(result, attribute, value, + rule.getAttributeLocation(attribute.getName()), + rule.isAttributeValueExplicitlySpecified(attribute), + true); + } + } + + return result.build(); + } + + private static List<Build.MakeVar> serializeMakeEnvironment(MakeEnvironment makeEnv) { + List<Build.MakeVar> result = new ArrayList<>(); + + for (Map.Entry<String, ImmutableList<Binding>> var : makeEnv.getBindings().entrySet()) { + Build.MakeVar.Builder varPb = Build.MakeVar.newBuilder(); + varPb.setName(var.getKey()); + for (Binding binding : var.getValue()) { + Build.MakeVarBinding.Builder bindingPb = Build.MakeVarBinding.newBuilder(); + bindingPb.setValue(binding.getValue()); + bindingPb.setPlatformSetRegexp(binding.getPlatformSetRegexp()); + varPb.addBinding(bindingPb); + } + + result.add(varPb.build()); + } + + return result; + } + + private static Build.License serializeLicense(License license) { + Build.License.Builder result = Build.License.newBuilder(); + + for (License.LicenseType licenseType : license.getLicenseTypes()) { + result.addLicenseType(licenseType.toString()); + } + + for (Label exception : license.getExceptions()) { + result.addException(exception.toString()); + } + return result.build(); + } + + private static Build.Event serializeEvent(Event event) { + Build.Event.Builder result = Build.Event.newBuilder(); + result.setMessage(event.getMessage()); + if (event.getLocation() != null) { + result.setLocation(serializeLocation(event.getLocation())); + } + + Build.Event.EventKind kind; + + switch (event.getKind()) { + case ERROR: + kind = Build.Event.EventKind.ERROR; + break; + case WARNING: + kind = Build.Event.EventKind.WARNING; + break; + case INFO: + kind = Build.Event.EventKind.INFO; + break; + case PROGRESS: + kind = Build.Event.EventKind.PROGRESS; + break; + default: throw new IllegalStateException(); + } + + result.setKind(kind); + return result.build(); + } + + private static void serializePackageInternal(Package pkg, Build.Package.Builder builder) { + builder.setName(pkg.getName()); + builder.setRepository(pkg.getPackageIdentifier().getRepository().toString()); + builder.setBuildFilePath(pkg.getFilename().getPathString()); + // The extra bit is needed to handle the corner case when the default visibility is [], i.e. + // zero labels. + builder.setDefaultVisibilitySet(pkg.isDefaultVisibilitySet()); + if (pkg.isDefaultVisibilitySet()) { + for (Label visibilityLabel : pkg.getDefaultVisibility().getDeclaredLabels()) { + builder.addDefaultVisibilityLabel(visibilityLabel.toString()); + } + } + + builder.setDefaultObsolete(pkg.getDefaultObsolete()); + builder.setDefaultTestonly(pkg.getDefaultTestOnly()); + if (pkg.getDefaultDeprecation() != null) { + builder.setDefaultDeprecation(pkg.getDefaultDeprecation()); + } + + for (String defaultCopt : pkg.getDefaultCopts()) { + builder.addDefaultCopt(defaultCopt); + } + + if (pkg.isDefaultHdrsCheckSet()) { + builder.setDefaultHdrsCheck(pkg.getDefaultHdrsCheck()); + } + + builder.setDefaultLicense(serializeLicense(pkg.getDefaultLicense())); + + for (DistributionType distributionType : pkg.getDefaultDistribs()) { + builder.addDefaultDistrib(distributionType.toString()); + } + + for (String feature : pkg.getFeatures()) { + builder.addDefaultSetting(feature); + } + + for (Label subincludeLabel : pkg.getSubincludeLabels()) { + builder.addSubincludeLabel(subincludeLabel.toString()); + } + + for (Label skylarkLabel : pkg.getSkylarkFileDependencies()) { + builder.addSkylarkLabel(skylarkLabel.toString()); + } + + for (Build.MakeVar makeVar : + serializeMakeEnvironment(pkg.getMakeEnvironment())) { + builder.addMakeVariable(makeVar); + } + + for (Target target : pkg.getTargets()) { + if (target instanceof InputFile) { + builder.addSourceFile(serializeInputFile((InputFile) target)); + } else if (target instanceof OutputFile) { + // Output files are ignored; they are recorded in rules. + } else if (target instanceof PackageGroup) { + builder.addPackageGroup(serializePackageGroup((PackageGroup) target)); + } else if (target instanceof Rule) { + builder.addRule(serializeRule((Rule) target)); + } + } + + for (Event event : pkg.getEvents()) { + builder.addEvent(serializeEvent(event)); + } + + builder.setContainsErrors(pkg.containsErrors()); + builder.setContainsTemporaryErrors(pkg.containsTemporaryErrors()); + } + + /** + * Serialize a package to a protocol message. The inverse of + * {@link PackageDeserializer#deserialize}. + */ + public static Build.Package serializePackage(Package pkg) { + Build.Package.Builder builder = Build.Package.newBuilder(); + serializePackageInternal(pkg, builder); + return builder.build(); + } + + /** + * Adds the serialized version of the specified attribute to the specified message. + * + * @param result the message to amend + * @param attr the attribute to add + * @param value the value of the attribute + * @param location the location of the attribute in the source file + * @param explicitlySpecified whether the attribute was explicitly specified or not + * @param includeGlobs add glob expression for attributes that contain them + */ + // TODO(bazel-team): This is a copy of the code in ProtoOutputFormatter. + @SuppressWarnings("unchecked") + public static void addAttributeToProto( + Build.Rule.Builder result, Attribute attr, Object value, Location location, + Boolean explicitlySpecified, boolean includeGlobs) { + // Get the attribute type. We need to convert and add appropriately + com.google.devtools.build.lib.packages.Type<?> type = attr.getType(); + + Build.Attribute.Builder attrPb = Build.Attribute.newBuilder(); + + // Set the type, name and source + Build.Attribute.Discriminator attributeProtoType = ProtoUtils.getDiscriminatorFromType(type); + attrPb.setName(attr.getName()); + attrPb.setType(attributeProtoType); + + if (location != null) { + attrPb.setParseableLocation(serializeLocation(location)); + } + + if (explicitlySpecified != null) { + attrPb.setExplicitlySpecified(explicitlySpecified); + } + + /* + * Set the appropriate type and value. Since string and string list store + * values for multiple types, use the toString() method on the objects + * instead of casting them. Note that Boolean and TriState attributes have + * both an integer and string representation. + */ + if (type == INTEGER) { + attrPb.setIntValue((Integer) value); + } else if (type == STRING || type == LABEL || type == NODEP_LABEL || type == OUTPUT) { + attrPb.setStringValue(value.toString()); + } else if (type == STRING_LIST || type == LABEL_LIST || type == NODEP_LABEL_LIST + || type == OUTPUT_LIST || type == DISTRIBUTIONS) { + Collection<?> values = (Collection<?>) value; + for (Object entry : values) { + attrPb.addStringListValue(entry.toString()); + } + } else if (type == INTEGER_LIST) { + Collection<Integer> values = (Collection<Integer>) value; + for (Integer entry : values) { + attrPb.addIntListValue(entry); + } + } else if (type == BOOLEAN) { + if ((Boolean) value) { + attrPb.setStringValue("true"); + attrPb.setBooleanValue(true); + } else { + attrPb.setStringValue("false"); + attrPb.setBooleanValue(false); + } + // This maintains partial backward compatibility for external users of the + // protobuf that were expecting an integer field and not a true boolean. + attrPb.setIntValue((Boolean) value ? 1 : 0); + } else if (type == TRISTATE) { + switch ((TriState) value) { + case AUTO: + attrPb.setIntValue(-1); + attrPb.setStringValue("auto"); + attrPb.setTristateValue(Build.Attribute.Tristate.AUTO); + break; + case NO: + attrPb.setIntValue(0); + attrPb.setStringValue("no"); + attrPb.setTristateValue(Build.Attribute.Tristate.NO); + break; + case YES: + attrPb.setIntValue(1); + attrPb.setStringValue("yes"); + attrPb.setTristateValue(Build.Attribute.Tristate.YES); + break; + } + } else if (type == LICENSE) { + License license = (License) value; + Build.License.Builder licensePb = Build.License.newBuilder(); + for (License.LicenseType licenseType : license.getLicenseTypes()) { + licensePb.addLicenseType(licenseType.toString()); + } + for (Label exception : license.getExceptions()) { + licensePb.addException(exception.toString()); + } + attrPb.setLicense(licensePb); + } else if (type == STRING_DICT) { + Map<String, String> dict = (Map<String, String>) value; + for (Map.Entry<String, String> dictEntry : dict.entrySet()) { + Build.StringDictEntry entry = Build.StringDictEntry.newBuilder() + .setKey(dictEntry.getKey()) + .setValue(dictEntry.getValue()) + .build(); + attrPb.addStringDictValue(entry); + } + } else if (type == STRING_DICT_UNARY) { + Map<String, String> dict = (Map<String, String>) value; + for (Map.Entry<String, String> dictEntry : dict.entrySet()) { + Build.StringDictUnaryEntry entry = Build.StringDictUnaryEntry.newBuilder() + .setKey(dictEntry.getKey()) + .setValue(dictEntry.getValue()) + .build(); + attrPb.addStringDictUnaryValue(entry); + } + } else if (type == STRING_LIST_DICT) { + Map<String, List<String>> dict = (Map<String, List<String>>) value; + for (Map.Entry<String, List<String>> dictEntry : dict.entrySet()) { + Build.StringListDictEntry.Builder entry = Build.StringListDictEntry.newBuilder() + .setKey(dictEntry.getKey()); + for (Object dictEntryValue : dictEntry.getValue()) { + entry.addValue(dictEntryValue.toString()); + } + attrPb.addStringListDictValue(entry); + } + } else if (type == LABEL_LIST_DICT) { + Map<String, List<Label>> dict = (Map<String, List<Label>>) value; + for (Map.Entry<String, List<Label>> dictEntry : dict.entrySet()) { + Build.LabelListDictEntry.Builder entry = Build.LabelListDictEntry.newBuilder() + .setKey(dictEntry.getKey()); + for (Object dictEntryValue : dictEntry.getValue()) { + entry.addValue(dictEntryValue.toString()); + } + attrPb.addLabelListDictValue(entry); + } + } else if (type == FILESET_ENTRY_LIST) { + List<FilesetEntry> filesetEntries = (List<FilesetEntry>) value; + for (FilesetEntry filesetEntry : filesetEntries) { + Build.FilesetEntry.Builder filesetEntryPb = Build.FilesetEntry.newBuilder() + .setSource(filesetEntry.getSrcLabel().toString()) + .setDestinationDirectory(filesetEntry.getDestDir().getPathString()) + .setSymlinkBehavior(symlinkBehaviorToPb(filesetEntry.getSymlinkBehavior())) + .setStripPrefix(filesetEntry.getStripPrefix()) + .setFilesPresent(filesetEntry.getFiles() != null); + + if (filesetEntry.getFiles() != null) { + for (Label file : filesetEntry.getFiles()) { + filesetEntryPb.addFile(file.toString()); + } + } + + if (filesetEntry.getExcludes() != null) { + for (String exclude : filesetEntry.getExcludes()) { + filesetEntryPb.addExclude(exclude); + } + } + + attrPb.addFilesetListValue(filesetEntryPb); + } + } else { + throw new IllegalStateException("Unknown type: " + type); + } + + if (includeGlobs && value instanceof GlobList<?>) { + GlobList<?> globList = (GlobList<?>) value; + + for (GlobCriteria criteria : globList.getCriteria()) { + Build.GlobCriteria.Builder criteriaPb = Build.GlobCriteria.newBuilder(); + criteriaPb.setGlob(criteria.isGlob()); + for (String include : criteria.getIncludePatterns()) { + criteriaPb.addInclude(include); + } + for (String exclude : criteria.getExcludePatterns()) { + criteriaPb.addExclude(exclude); + } + + attrPb.addGlobCriteria(criteriaPb); + } + } + result.addAttribute(attrPb); + } + + // This is needed because I do not want to use the SymlinkBehavior from the + // protocol buffer all over the place, so there are two classes that do + // essentially the same thing. + private static Build.FilesetEntry.SymlinkBehavior symlinkBehaviorToPb( + FilesetEntry.SymlinkBehavior symlinkBehavior) { + switch (symlinkBehavior) { + case COPY: + return Build.FilesetEntry.SymlinkBehavior.COPY; + case DEREFERENCE: + return Build.FilesetEntry.SymlinkBehavior.DEREFERENCE; + default: + throw new AssertionError("Unhandled FilesetEntry.SymlinkBehavior"); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageSpecification.java b/src/main/java/com/google/devtools/build/lib/packages/PackageSpecification.java new file mode 100644 index 0000000..20524aa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/PackageSpecification.java
@@ -0,0 +1,150 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.devtools.build.lib.cmdline.LabelValidator; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * A class that represents some packages that are included in the visibility list of a rule. + */ +public abstract class PackageSpecification { + private static final String PACKAGE_LABEL = "__pkg__"; + private static final String SUBTREE_LABEL = "__subpackages__"; + private static final String ALL_BENEATH_SUFFIX = "/..."; + public static final PackageSpecification EVERYTHING = + new AllPackagesBeneath(new PathFragment("")); + + public abstract boolean containsPackage(PathFragment packageName); + + @Override + public int hashCode() { + return toString().hashCode(); + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof PackageSpecification)) { + return false; + } + + return this.toString().equals(that.toString()); + } + + /** + * Parses a string as a visibility specification. + * Throws {@link InvalidPackageSpecificationException} if the label cannot be parsed. + * + * <p>Note that these strings are different from what {@link #fromLabel} understands. + */ + public static PackageSpecification fromString(final String spec) + throws InvalidPackageSpecificationException { + String result = spec; + boolean allBeneath = false; + + if (result.startsWith("//")) { + result = spec.substring(2); + } else { + throw new InvalidPackageSpecificationException("invalid package label: " + spec); + } + + if (result.indexOf(':') >= 0) { + throw new InvalidPackageSpecificationException("invalid package label: " + spec); + } + + if (result.equals("...")) { + // Special case: //... will not end in /... + return EVERYTHING; + } + + if (result.endsWith(ALL_BENEATH_SUFFIX)) { + allBeneath = true; + result = result.substring(0, result.length() - ALL_BENEATH_SUFFIX.length()); + } + + String errorMessage = LabelValidator.validatePackageName(result); + if (errorMessage == null) { + return allBeneath ? + new AllPackagesBeneath(new PathFragment(result)) : + new SinglePackage(new PathFragment(result)); + } else { + throw new InvalidPackageSpecificationException(errorMessage); + } + } + + /** + * Parses a label as a visibility specification. returns null if the label cannot be parsed. + * + * <p>Note that these strings are different from what {@link #fromString} understands. + */ + public static PackageSpecification fromLabel(Label label) { + if (label.getName().equals(PACKAGE_LABEL)) { + return new SinglePackage(label.getPackageFragment()); + } else if (label.getName().equals(SUBTREE_LABEL)) { + return new AllPackagesBeneath(label.getPackageFragment()); + } else { + return null; + } + } + + private static class SinglePackage extends PackageSpecification { + private PathFragment singlePackageName; + + public SinglePackage(PathFragment packageName) { + this.singlePackageName = packageName; + } + + @Override + public boolean containsPackage(PathFragment packageName) { + return this.singlePackageName.equals(packageName); + } + + @Override + public String toString() { + return singlePackageName.toString(); + } + } + + private static class AllPackagesBeneath extends PackageSpecification { + private PathFragment prefix; + + public AllPackagesBeneath(PathFragment prefix) { + this.prefix = prefix; + } + + @Override + public boolean containsPackage(PathFragment packageName) { + return packageName.startsWith(prefix); + } + + @Override + public String toString() { + return prefix.equals(new PathFragment("")) ? "..." : prefix.toString() + "/..."; + } + } + + /** + * Exception class to be thrown when a specification cannot be parsed. + */ + public static class InvalidPackageSpecificationException extends Exception { + public InvalidPackageSpecificationException(String message) { + super(message); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PredicateWithMessage.java b/src/main/java/com/google/devtools/build/lib/packages/PredicateWithMessage.java new file mode 100644 index 0000000..4316f02 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/PredicateWithMessage.java
@@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Predicate; + +/** + * A predicate which supports error messages. + * @param <T> - the predicate is applied on T objects + */ +public interface PredicateWithMessage<T> extends Predicate<T> { + + /** + * The error message to display when predicate checks param. Only makes sense to + * call this method is apply(param) returns false. + */ + public String getErrorReason(T param); + +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PredicatesWithMessage.java b/src/main/java/com/google/devtools/build/lib/packages/PredicatesWithMessage.java new file mode 100644 index 0000000..a7f04bf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/PredicatesWithMessage.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +/** + * A helper class for PredicateWithMessage with default predicates. + */ +public abstract class PredicatesWithMessage implements PredicateWithMessage<Object> { + + private static final PredicateWithMessage<?> ALWAYS_TRUE = new PredicateWithMessage<Object>() { + @Override + public boolean apply(Object input) { + return true; + } + + @Override + public String getErrorReason(Object param) { + throw new UnsupportedOperationException(); + } + }; + + @SuppressWarnings("unchecked") + public static <T> PredicateWithMessage<T> alwaysTrue() { + return (PredicateWithMessage<T>) ALWAYS_TRUE; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/Preprocessor.java b/src/main/java/com/google/devtools/build/lib/packages/Preprocessor.java new file mode 100644 index 0000000..8eda1e9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/Preprocessor.java
@@ -0,0 +1,155 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.PackageFactory.Globber; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.ParserInputSource; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; +import java.util.Set; + +import javax.annotation.Nullable; + +/** A Preprocessor is an interface to implement generic text-based preprocessing of BUILD files. */ +public interface Preprocessor { + /** Factory for {@link Preprocessor} instances. */ + interface Factory { + /** Supplier for {@link Factory} instances. */ + interface Supplier { + /** + * Returns a Preprocessor factory to use for getting Preprocessor instances. + * + * <p>The CachingPackageLocator is provided so the constructed preprocessors can look up + * other BUILD files. + */ + Factory getFactory(CachingPackageLocator loc); + + /** Supplier that always returns {@code NullFactory.INSTANCE}. */ + static class NullSupplier implements Supplier { + + public static final NullSupplier INSTANCE = new NullSupplier(); + + private NullSupplier() { + } + + @Override + public Factory getFactory(CachingPackageLocator loc) { + return NullFactory.INSTANCE; + } + } + } + + /** + * Returns whether this {@link Factory} is still suitable for providing {@link Preprocessor}s. + * If not, all previous preprocessing results should be assumed to be invalid and a new + * {@link Factory} should be created via {@link Supplier#getFactory}. + */ + boolean isStillValid(); + + /** + * Returns a Preprocessor instance capable of preprocessing a BUILD file independently (e.g. it + * ought to be fine to call {@link #getPreprocessor} for each BUILD file). + */ + @Nullable + Preprocessor getPreprocessor(); + + /** Factory that always returns {@code null} {@link Preprocessor}s. */ + static class NullFactory implements Factory { + public static final NullFactory INSTANCE = new NullFactory(); + + private NullFactory() { + } + + @Override + public boolean isStillValid() { + return true; + } + + @Override + public Preprocessor getPreprocessor() { + return null; + } + } + } + + /** + * A (result, success) tuple indicating the outcome of preprocessing. + */ + static class Result { + public final ParserInputSource result; + public final boolean preprocessed; + public final boolean containsErrors; + public final boolean containsTransientErrors; + + private Result(ParserInputSource result, + boolean preprocessed, boolean containsPersistentErrors, boolean containsTransientErrors) { + this.result = result; + this.preprocessed = preprocessed; + this.containsErrors = containsPersistentErrors || containsTransientErrors; + this.containsTransientErrors = containsTransientErrors; + } + + /** Convenience factory for a {@link Result} wrapping non-preprocessed BUILD file contents. */ + public static Result noPreprocessing(ParserInputSource buildFileSource) { + return new Result(buildFileSource, /*preprocessed=*/false, /*containsErrors=*/false, + /*containsTransientErrors=*/false); + } + + /** + * Factory for a successful preprocessing result, meaning that the BUILD file was able to be + * read and has valid syntax and was preprocessed. But note that there may have been be errors + * during preprocessing. + */ + public static Result success(ParserInputSource result, boolean containsErrors) { + return new Result(result, /*preprocessed=*/true, /*containsPersistentErrors=*/containsErrors, + /*containsTransientErrors=*/false); + } + + public static Result invalidSyntax(Path buildFile) { + return new Result(ParserInputSource.create("", buildFile), /*preprocessed=*/true, + /*containsPersistentErrors=*/true, /*containsTransientErrors=*/false); + } + + public static Result transientError(Path buildFile) { + return new Result(ParserInputSource.create("", buildFile), /*preprocessed=*/false, + /*containsPersistentErrors=*/false, /*containsTransientErrors=*/true); + } + } + + /** + * Returns a Result resulting from applying Python preprocessing to the contents of "in". If + * errors happen, they must be reported both as an event on eventHandler and in the function + * return value. + * + * @param in the BUILD file to be preprocessed. + * @param packageName the BUILD file's package. + * @param globber a globber for evaluating globs. + * @param eventHandler a eventHandler on which to report warnings/errors. + * @param globalEnv the GLOBALS Python environment. + * @param ruleNames the set of names of all rules in the build language. + * @throws IOException if there was an I/O problem during preprocessing. + * @return a pair of the ParserInputSource and a map of subincludes seen during the evaluation + */ + Result preprocess( + ParserInputSource in, + String packageName, + Globber globber, + EventHandler eventHandler, + Environment globalEnv, + Set<String> ruleNames) + throws IOException, InterruptedException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/ProtoUtils.java b/src/main/java/com/google/devtools/build/lib/packages/ProtoUtils.java new file mode 100644 index 0000000..7b6eaf1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/ProtoUtils.java
@@ -0,0 +1,83 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.DISTRIBUTIONS; +import static com.google.devtools.build.lib.packages.Type.FILESET_ENTRY_LIST; +import static com.google.devtools.build.lib.packages.Type.INTEGER; +import static com.google.devtools.build.lib.packages.Type.INTEGER_LIST; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST_DICT; +import static com.google.devtools.build.lib.packages.Type.LICENSE; +import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL; +import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.OUTPUT; +import static com.google.devtools.build.lib.packages.Type.OUTPUT_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING; +import static com.google.devtools.build.lib.packages.Type.STRING_DICT; +import static com.google.devtools.build.lib.packages.Type.STRING_DICT_UNARY; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST_DICT; +import static com.google.devtools.build.lib.packages.Type.TRISTATE; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.Attribute.Discriminator; + +import java.util.Map; + +/** + * Shared code used in proto buffer output for rules and rule classes. + */ +public class ProtoUtils { + /** + * This map contains all attribute types that are recognized by the protocol + * output formatter. + */ + private static final Map<Type<?>, Discriminator> TYPE_MAP + = new ImmutableMap.Builder<Type<?>, Discriminator>() + .put(INTEGER, Discriminator.INTEGER) + .put(DISTRIBUTIONS, Discriminator.DISTRIBUTION_SET) + .put(LABEL, Discriminator.LABEL) + // NODEP_LABEL attributes are not really strings. This is implemented + // this way for the sake of backward compatibility. + .put(NODEP_LABEL, Discriminator.STRING) + .put(LABEL_LIST, Discriminator.LABEL_LIST) + .put(NODEP_LABEL_LIST, Discriminator.STRING_LIST) + .put(STRING, Discriminator.STRING) + .put(STRING_LIST, Discriminator.STRING_LIST) + .put(OUTPUT, Discriminator.OUTPUT) + .put(OUTPUT_LIST, Discriminator.OUTPUT_LIST) + .put(LICENSE, Discriminator.LICENSE) + .put(STRING_DICT, Discriminator.STRING_DICT) + .put(FILESET_ENTRY_LIST, Discriminator.FILESET_ENTRY_LIST) + .put(LABEL_LIST_DICT, Discriminator.LABEL_LIST_DICT) + .put(STRING_LIST_DICT, Discriminator.STRING_LIST_DICT) + .put(BOOLEAN, Discriminator.BOOLEAN) + .put(TRISTATE, Discriminator.TRISTATE) + .put(INTEGER_LIST, Discriminator.INTEGER_LIST) + .put(STRING_DICT_UNARY, Discriminator.STRING_DICT_UNARY) + .build(); + + /** + * Returns the appropriate Attribute.Discriminator value from an internal attribute type. + */ + public static Discriminator getDiscriminatorFromType(Type<?> type) { + Preconditions.checkArgument(TYPE_MAP.containsKey(type)); + return TYPE_MAP.get(type); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RawAttributeMapper.java b/src/main/java/com/google/devtools/build/lib/packages/RawAttributeMapper.java new file mode 100644 index 0000000..a174193 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/RawAttributeMapper.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.syntax.Label; + +/** + * {@link AttributeMap} implementation that returns raw attribute information as contained + * within a {@link Rule}. In particular, configurable attributes of the form + * { config1: "value1", config2: "value2" } are passed through without being resolved to a + * final value. + */ +public class RawAttributeMapper extends AbstractAttributeMapper { + RawAttributeMapper(Package pkg, RuleClass ruleClass, Label ruleLabel, + AttributeContainer attributes) { + super(pkg, ruleClass, ruleLabel, attributes); + } + + public static RawAttributeMapper of(Rule rule) { + return new RawAttributeMapper(rule.getPackage(), rule.getRuleClassObject(), rule.getLabel(), + rule.getAttributeContainer()); + } + + @Override + protected <T> Iterable<T> visitAttribute(String attributeName, Type<T> type) { + T value = get(attributeName, type); + return value == null ? ImmutableList.<T>of() : ImmutableList.of(value); + } + + /** + * Returns true if the given attribute is configurable for this rule instance, false + * otherwise. + */ + public <T> boolean isConfigurable(String attributeName, Type<T> type) { + return getSelector(attributeName, type) != null; + } + + /** + * If the attribute is configurable for this rule instance, returns its configuration + * keys. Else returns an empty list. + */ + public <T> Iterable<Label> getConfigurabilityKeys(String attributeName, Type<T> type) { + Type.Selector<T> selector = getSelector(attributeName, type); + return selector == null ? ImmutableList.<Label>of() : selector.getEntries().keySet(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RelativePackageNameResolver.java b/src/main/java/com/google/devtools/build/lib/packages/RelativePackageNameResolver.java new file mode 100644 index 0000000..ade196b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/RelativePackageNameResolver.java
@@ -0,0 +1,81 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Resolves relative package names to absolute ones. Handles the absolute + * package path marker ("//") and uplevel references (".."). + */ +public class RelativePackageNameResolver { + private final PathFragment offset; + private final boolean discardBuild; + + /** + * @param offset the base package path used to resolve relative paths + * @param discardBuild if true, discards the last package path segment if + * it is called "BUILD" + */ + public RelativePackageNameResolver(PathFragment offset, boolean discardBuild) { + Preconditions.checkArgument(!offset.containsUplevelReferences(), + "offset should not contain uplevel references"); + + this.offset = offset; + this.discardBuild = discardBuild; + } + + /** + * Resolves the given package name with respect to the offset given in the + * constructor. + * + * @param pkg the relative package name to be resolved + * @return the absolute package name + * @throws InvalidPackageNameException if the package name cannot be resolved + * (only syntactic checks are done -- it is not checked if the package + * really exists or not) + */ + public String resolve(String pkg) throws InvalidPackageNameException { + boolean isAbsolute; + String relativePkg; + + if (pkg.startsWith("//")) { + isAbsolute = true; + relativePkg = pkg.substring(2); + } else if (pkg.startsWith("/")) { + throw new InvalidPackageNameException(pkg, + "package name cannot start with a single slash"); + } else { + isAbsolute = false; + relativePkg = pkg; + } + + PathFragment relative = new PathFragment(relativePkg); + + if (discardBuild && relative.getBaseName().equals("BUILD")) { + relative = relative.getParentDirectory(); + } + + PathFragment result = isAbsolute ? relative : offset.getRelative(relative); + result = result.normalize(); + if (result.containsUplevelReferences()) { + throw new InvalidPackageNameException(pkg, + "package name contains too many '..' segments"); + } + + return result.getPathString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/Rule.java b/src/main/java/com/google/devtools/build/lib/packages/Rule.java new file mode 100644 index 0000000..cff91f6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/Rule.java
@@ -0,0 +1,666 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition; +import com.google.devtools.build.lib.packages.License.DistributionType; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.GlobList; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.util.BinaryPredicate; + +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * An instance of a build rule in the build language. A rule has a name, a + * package to which it belongs, a class such as <code>cc_library</code>, and + * set of typed attributes. The set of attribute names and types is a property + * of the rule's class. The use of the term "class" here has nothing to do + * with Java classes. All rules are implemented by the same Java classes, Rule + * and RuleClass. + * + * <p>Here is a typical rule as it appears in a BUILD file: + * <pre> + * cc_library(name = 'foo', + * defines = ['-Dkey=value'], + * srcs = ['foo.cc'], + * deps = ['bar']) + * </pre> + */ +public final class Rule implements Target { + /** Dependency predicate that includes all dependencies */ + public static final BinaryPredicate<Rule, Attribute> ALL_DEPS = + new BinaryPredicate<Rule, Attribute>() { + @Override + public boolean apply(Rule x, Attribute y) { + return true; + } + }; + + /** Dependency predicate that excludes host dependencies */ + public static final BinaryPredicate<Rule, Attribute> NO_HOST_DEPS = + new BinaryPredicate<Rule, Attribute>() { + @Override + public boolean apply(Rule rule, Attribute attribute) { + // isHostConfiguration() is only defined for labels and label lists. + if (attribute.getType() != Type.LABEL && attribute.getType() != Type.LABEL_LIST) { + return true; + } + + return attribute.getConfigurationTransition() != ConfigurationTransition.HOST; + } + }; + + /** Dependency predicate that excludes implicit dependencies */ + public static final BinaryPredicate<Rule, Attribute> NO_IMPLICIT_DEPS = + new BinaryPredicate<Rule, Attribute>() { + @Override + public boolean apply(Rule rule, Attribute attribute) { + return rule.isAttributeValueExplicitlySpecified(attribute); + } + }; + + /** + * Dependency predicate that excludes those edges that are not present in the + * configured target graph. + */ + public static final BinaryPredicate<Rule, Attribute> NO_NODEP_ATTRIBUTES = + new BinaryPredicate<Rule, Attribute>() { + @Override + public boolean apply(Rule rule, Attribute attribute) { + return attribute.getType() != Type.NODEP_LABEL && + attribute.getType() != Type.NODEP_LABEL_LIST; + } + }; + + /** Label predicate that allows every label. */ + public static final Predicate<Label> ALL_LABELS = Predicates.alwaysTrue(); + + /** + * Checks to see if the attribute has the isDirectCompileTimeInput property. + */ + public static final BinaryPredicate<Rule, Attribute> DIRECT_COMPILE_TIME_INPUT = + new BinaryPredicate<Rule, Attribute>() { + @Override + public boolean apply(Rule rule, Attribute attribute) { + return attribute.isDirectCompileTimeInput(); + } + }; + + /** + * Returns a predicate that computes the logical and of the two given predicates. + */ + public static <X, Y> BinaryPredicate<X, Y> and( + final BinaryPredicate<X, Y> a, final BinaryPredicate<X, Y> b) { + return new BinaryPredicate<X, Y>() { + @Override + public boolean apply(X x, Y y) { + return a.apply(x, y) && b.apply(x, y); + } + }; + } + + private final Label label; + + private final Package pkg; + + private final RuleClass ruleClass; + + private final AttributeContainer attributes; + private final RawAttributeMapper attributeMap; + + private RuleVisibility visibility; + + private boolean containsErrors; + + private final Location location; + + private final FuncallExpression ast; // may be null + + // Initialized in the call to populateOutputFiles. + private List<OutputFile> outputFiles; + private ListMultimap<String, OutputFile> outputFileMap; + + Rule(Package pkg, Label label, RuleClass ruleClass, FuncallExpression ast, Location location) { + this.pkg = Preconditions.checkNotNull(pkg); + this.label = label; + this.ruleClass = Preconditions.checkNotNull(ruleClass); + this.location = Preconditions.checkNotNull(location); + this.attributes = new AttributeContainer(ruleClass); + this.attributeMap = new RawAttributeMapper(pkg, ruleClass, label, attributes); + this.containsErrors = false; + this.ast = ast; + } + + void setVisibility(RuleVisibility visibility) { + this.visibility = visibility; + } + + void setAttributeValue(Attribute attribute, Object value, boolean explicit) { + attributes.setAttributeValue(attribute, value, explicit); + } + + void setAttributeValueByName(String attrName, Object value) { + attributes.setAttributeValueByName(attrName, value); + } + + void setAttributeLocation(int attrIndex, Location location) { + attributes.setAttributeLocation(attrIndex, location); + } + + void setAttributeLocation(Attribute attribute, Location location) { + attributes.setAttributeLocation(attribute, location); + } + + void setContainsErrors() { + this.containsErrors = true; + } + + @Override + public Label getLabel() { + return attributeMap.getLabel(); + } + + @Override + public String getName() { + return attributeMap.getName(); + } + + @Override + public Package getPackage() { + return pkg; + } + + public RuleClass getRuleClassObject() { + return ruleClass; + } + + @Override + public String getTargetKind() { + return ruleClass.getTargetKind(); + } + + /** + * Returns the class of this rule. (e.g. "cc_library") + */ + public String getRuleClass() { + return ruleClass.getName(); + } + + /** + * Returns the build features that apply to this rule. + */ + public ImmutableSet<String> getFeatures() { + return pkg.getFeatures(); + } + + /** + * Returns true iff the outputs of this rule should be created beneath the + * bin directory, false if beneath genfiles. For most rule + * classes, this is a constant, but for genrule, it is a property of the + * individual rule instance, derived from the 'output_to_bindir' attribute. + */ + public boolean hasBinaryOutput() { + return ruleClass.getName().equals("genrule") // this is unfortunate... + ? NonconfigurableAttributeMapper.of(this).get("output_to_bindir", Type.BOOLEAN) + : ruleClass.hasBinaryOutput(); + } + + /** + * Returns the AST for this rule. Returns null if the package factory chose + * not to retain the AST when evaluateBuildFile was called for this rule's + * package. + */ + public FuncallExpression getSyntaxTree() { + return ast; + } + + /** + * Returns true iff there were errors while constructing this rule, such as + * attributes with missing values or values of the wrong type. + */ + public boolean containsErrors() { + return containsErrors; + } + + /** + * Returns an (unmodifiable, unordered) collection containing all the + * Attribute definitions for this kind of rule. (Note, this doesn't include + * the <i>values</i> of the attributes, merely the schema. Call + * get[Type]Attr() methods to access the actual values.) + */ + public Collection<Attribute> getAttributes() { + return ruleClass.getAttributes(); + } + + /** + * Returns true if this rule has any attributes that are configurable. + * + * <p>Note this is *not* the same as having attribute *types* that are configurable. For example, + * "deps" is configurable, in that one can write a rule that sets "deps" to a configuration + * dictionary. But if *this* rule's instance of "deps" doesn't do that, its instance + * of "deps" is not considered configurable. + * + * <p>In other words, this method signals which rules might have their attribute values + * influenced by the configuration. + */ + public boolean hasConfigurableAttributes() { + for (Attribute attribute : getAttributes()) { + if (attributeMap.isConfigurable(attribute.getName(), attribute.getType())) { + return true; + } + } + return false; + } + + /** + * Returns the attribute definition whose name is {@code attrName}, or null + * if not found. (Use get[X]Attr for the actual value.) + * + * @deprecated use {@link AbstractAttributeMapper#getAttributeDefinition} instead + */ + @Deprecated + public Attribute getAttributeDefinition(String attrName) { + return attributeMap.getAttributeDefinition(attrName); + } + + /** + * Returns an (unmodifiable, ordered) collection containing all the declared output files of this + * rule. + * + * <p>All implicit output files (declared in the {@link RuleClass}) are + * listed first, followed by any explicit files (declared via the 'outs' attribute). Additionally + * both implicit and explicit outputs will retain the relative order in which they were declared. + * + * <p>This ordering is useful because it is propagated through to the list of targets returned by + * getOuts() and allows targets to access their implicit outputs easily via + * {@code getOuts().get(N)} (providing that N is less than the number of implicit outputs). + * + * <p>The fact that the relative order of the explicit outputs is also retained is less obviously + * useful but is still well defined. + */ + public Collection<OutputFile> getOutputFiles() { + return outputFiles; + } + + /** + * Returns an (unmodifiable, ordered) map containing the list of output files for every + * output type attribute. + */ + public ListMultimap<String, OutputFile> getOutputFileMap() { + return outputFileMap; + } + + @Override + public Location getLocation() { + return location; + } + + @Override + public Rule getAssociatedRule() { + return this; + } + + /** + * Returns this rule's raw attribute info, suitable for being fed into an + * {@link AttributeMap} for user-level attribute access. Don't use this method + * for direct attribute access. + */ + public AttributeContainer getAttributeContainer() { + return attributes; + } + + /******************************************************************** + * Attribute accessor functions. + * + * The below provide access to attribute definitions and other generic + * metadata. + * + * For access to attribute *values* (e.g. "What's the value of attribute + * X for Rule Y?"), go through {@link RuleContext#attributes}. If no + * RuleContext is available, create a localized {@link AbstractAttributeMapper} + * instance instead. + ********************************************************************/ + + /** + * Returns the default value for the attribute {@code attrName}, which may be + * of any type, but must exist (an exception is thrown otherwise). + */ + public Object getAttrDefaultValue(String attrName) { + Object defaultValue = ruleClass.getAttributeByName(attrName).getDefaultValue(this); + // Computed defaults not expected here. + Preconditions.checkState(!(defaultValue instanceof Attribute.ComputedDefault)); + return defaultValue; + } + + /** + * Returns true iff the rule class has an attribute with the given name and type. + */ + public boolean isAttrDefined(String attrName, Type<?> type) { + return ruleClass.hasAttr(attrName, type); + } + + /** + * Returns true iff the value of the specified attribute is explicitly set in + * the BUILD file (as opposed to its default value). This also returns true if + * the value from the BUILD file is the same as the default value. + */ + public boolean isAttributeValueExplicitlySpecified(Attribute attribute) { + return attributes.isAttributeValueExplicitlySpecified(attribute); + } + + /** + * Returns true iff the value of the specified attribute is explicitly set in the BUILD file (as + * opposed to its default value). This also returns true if the value from the BUILD file is the + * same as the default value. In addition, this method return false if the rule has no attribute + * with the given name. + */ + public boolean isAttributeValueExplicitlySpecified(String attrName) { + return attributeMap.isAttributeValueExplicitlySpecified(attrName); + } + + /** + * Returns the location of the attribute definition for this rule, if known; + * or the location of the whole rule otherwise. "attrName" need not be a + * valid attribute name for this rule. + */ + public Location getAttributeLocation(String attrName) { + Location attrLocation = null; + if (!attrName.equals("name")) { + attrLocation = attributes.getAttributeLocation(attrName); + } + return attrLocation != null ? attrLocation : getLocation(); + } + + /** + * Returns a new List instance containing all direct dependencies (all types). + */ + public Collection<Label> getLabels() { + return getLabels(Rule.ALL_DEPS); + } + + /** + * Returns a new Collection containing all Labels that match a given Predicate, + * not including outputs. + * + * @param predicate A binary predicate that determines if a label should be + * included in the result. The predicate is evaluated with this rule and + * the attribute that contains the label. The label will be contained in the + * result iff (the predicate returned {@code true} and the labels are not outputs) + */ + public Collection<Label> getLabels(final BinaryPredicate<Rule, Attribute> predicate) { + final Set<Label> labels = new HashSet<>(); + // TODO(bazel-team): move this to AttributeMap, too. Just like visitLabels, which labels should + // be visited may depend on the calling context. We shouldn't implicitly decide this for + // the caller. + AggregatingAttributeMapper.of(this).visitLabels(new AttributeMap.AcceptsLabelAttribute() { + @Override + public void acceptLabelAttribute(Label label, Attribute attribute) { + if (predicate.apply(Rule.this, attribute)) { + labels.add(label); + } + } + }); + return labels; + } + + /** + * Check if this rule is valid according to the validityPredicate of its RuleClass. + */ + void checkValidityPredicate(EventHandler eventHandler) { + PredicateWithMessage<Rule> predicate = getRuleClassObject().getValidityPredicate(); + if (!predicate.apply(this)) { + reportError(predicate.getErrorReason(this), eventHandler); + } + } + + /** + * Collects the output files (both implicit and explicit). All the implicit output files are added + * first, followed by any explicit files. Additionally both implicit and explicit output files + * will retain the relative order in which they were declared. + */ + void populateOutputFiles(EventHandler eventHandler, + Package.AbstractBuilder<?, ?> pkgBuilder) throws SyntaxException { + Preconditions.checkState(outputFiles == null); + // Order is important here: implicit before explicit + outputFiles = Lists.newArrayList(); + outputFileMap = LinkedListMultimap.create(); + populateImplicitOutputFiles(eventHandler, pkgBuilder); + populateExplicitOutputFiles(eventHandler); + outputFiles = ImmutableList.copyOf(outputFiles); + outputFileMap = ImmutableListMultimap.copyOf(outputFileMap); + } + + // Explicit output files are user-specified attributes of type OUTPUT. + private void populateExplicitOutputFiles(EventHandler eventHandler) throws SyntaxException { + NonconfigurableAttributeMapper nonConfigurableAttributes = + NonconfigurableAttributeMapper.of(this); + for (Attribute attribute : ruleClass.getAttributes()) { + String name = attribute.getName(); + Type<?> type = attribute.getType(); + if (type == Type.OUTPUT) { + Label outputLabel = nonConfigurableAttributes.get(name, Type.OUTPUT); + if (outputLabel != null) { + addLabelOutput(attribute, outputLabel, eventHandler); + } + } else if (type == Type.OUTPUT_LIST) { + for (Label label : nonConfigurableAttributes.get(name, Type.OUTPUT_LIST)) { + addLabelOutput(attribute, label, eventHandler); + } + } + } + } + + /** + * Implicit output files come from rule-specific patterns, and are a function + * of the rule's "name", "srcs", and other attributes. + */ + private void populateImplicitOutputFiles(EventHandler eventHandler, + Package.AbstractBuilder<?, ?> pkgBuilder) { + try { + for (String out : ruleClass.getImplicitOutputsFunction().getImplicitOutputs(attributeMap)) { + try { + addOutputFile(pkgBuilder.createLabel(out), eventHandler); + } catch (SyntaxException e) { + reportError("illegal output file name '" + out + "' in rule " + + getLabel(), eventHandler); + } + } + } catch (EvalException e) { + reportError(e.print(), eventHandler); + } + } + + private void addLabelOutput(Attribute attribute, Label label, EventHandler eventHandler) + throws SyntaxException { + if (!label.getPackageIdentifier().equals(pkg.getPackageIdentifier())) { + throw new IllegalStateException("Label for attribute " + attribute + + " should refer to '" + pkg.getName() + + "' but instead refers to '" + label.getPackageFragment() + + "' (label '" + label.getName() + "')"); + } + if (label.getName().equals(".")) { + throw new SyntaxException("output file name can't be equal '.'"); + } + OutputFile outputFile = addOutputFile(label, eventHandler); + outputFileMap.put(attribute.getName(), outputFile); + } + + private OutputFile addOutputFile(Label label, EventHandler eventHandler) { + if (label.getName().equals(getName())) { + // TODO(bazel-team): for now (23 Apr 2008) this is just a warning. After + // June 1st we should make it an error. + reportWarning("target '" + getName() + "' is both a rule and a file; please choose " + + "another name for the rule", eventHandler); + } + OutputFile outputFile = new OutputFile(pkg, label, this); + outputFiles.add(outputFile); + return outputFile; + } + + void reportError(String message, EventHandler eventHandler) { + eventHandler.handle(Event.error(location, message)); + this.containsErrors = true; + } + + void reportWarning(String message, EventHandler eventHandler) { + eventHandler.handle(Event.warn(location, message)); + } + + @Override + public int hashCode() { + return label.hashCode(); + } + + /** + * Returns a string of the form "cc_binary rule //foo:foo" + * + * @return a string of the form "cc_binary rule //foo:foo" + */ + @Override + public String toString() { + return getRuleClass() + " rule " + getLabel(); + } + + /** + * Returns the effective visibility of this Rule. Visibility is computed from + * these sources in this order of preference: + * - 'visibility' attribute + * - 'default_visibility;' attribute of package() declaration + * - public. + */ + @Override + public RuleVisibility getVisibility() { + if (visibility != null) { + return visibility; + } + + if (getRuleClassObject().isPublicByDefault()) { + return ConstantRuleVisibility.PUBLIC; + } + + return pkg.getDefaultVisibility(); + } + + public boolean isVisibilitySpecified() { + return visibility != null; + } + + @Override + @SuppressWarnings("unchecked") + public Set<DistributionType> getDistributions() { + if (isAttrDefined("distribs", Type.DISTRIBUTIONS) + && isAttributeValueExplicitlySpecified("distribs")) { + return NonconfigurableAttributeMapper.of(this).get("distribs", Type.DISTRIBUTIONS); + } else { + return getPackage().getDefaultDistribs(); + } + } + + @Override + public License getLicense() { + if (isAttrDefined("licenses", Type.LICENSE) + && isAttributeValueExplicitlySpecified("licenses")) { + return NonconfigurableAttributeMapper.of(this).get("licenses", Type.LICENSE); + } else { + return getPackage().getDefaultLicense(); + } + } + + /** + * Returns the license of the output of the binary created by this rule, or + * null if it is not specified. + */ + public License getToolOutputLicense(AttributeMap attributes) { + if (isAttrDefined("output_licenses", Type.LICENSE) + && attributes.isAttributeValueExplicitlySpecified("output_licenses")) { + return attributes.get("output_licenses", Type.LICENSE); + } else { + return null; + } + } + + /** + * Returns the globs that were expanded to create an attribute value, or + * null if unknown or not applicable. + */ + public static GlobList<?> getGlobInfo(Object attributeValue) { + if (attributeValue instanceof GlobList<?>) { + return (GlobList<?>) attributeValue; + } else { + return null; + } + } + + private void checkForNullLabel(Label labelToCheck, String where) { + if (labelToCheck == null) { + throw new IllegalStateException(String.format( + "null label in rule %s, %s", getLabel().toString(), where)); + } + } + + // Consistency check: check if this label contains any weird labels (i.e. + // null-valued, with a packageFragment that is null...). The bug that prompted + // the introduction of this code is #2210848 (NullPointerException in + // Package.checkForConflicts() ). + void checkForNullLabels() { + AggregatingAttributeMapper.of(this).visitLabels( + new AttributeMap.AcceptsLabelAttribute() { + @Override + public void acceptLabelAttribute(Label labelToCheck, Attribute attribute) { + checkForNullLabel(labelToCheck, "attribute " + attribute.getName()); + } + }); + for (OutputFile outputFile : getOutputFiles()) { + checkForNullLabel(outputFile.getLabel(), "output file"); + } + } + + /** + * Returns the Set of all tags exhibited by this target. May be empty. + */ + public Set<String> getRuleTags() { + Set<String> ruleTags = new LinkedHashSet<>(); + for (Attribute attribute : getRuleClassObject().getAttributes()) { + if (attribute.isTaggable()) { + Type<?> attrType = attribute.getType(); + String name = attribute.getName(); + // This enforces the expectation that taggable attributes are non-configurable. + Object value = NonconfigurableAttributeMapper.of(this).get(name, attrType); + Set<String> tags = attrType.toTagSet(value, name); + ruleTags.addAll(tags); + } + } + return ruleTags; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java b/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java new file mode 100644 index 0000000..f495502 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java
@@ -0,0 +1,1511 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Ordering; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Argument; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.GlobList; +import com.google.devtools.build.lib.syntax.Ident; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.UserDefinedFunction; +import com.google.devtools.build.lib.util.StringUtil; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * Instances of RuleClass encapsulate the set of attributes of a given "class" of rule, such as + * <code>cc_binary</code>. + * + * <p>This is an instance of the "meta-class" pattern for Rules: we achieve using <i>values</i> + * what subclasses achieve using <i>types</i>. (The "Design Patterns" book doesn't include this + * pattern, so think of it as something like a cross between a Flyweight and a State pattern. Like + * Flyweight, we avoid repeatedly storing data that belongs to many instances. Like State, we + * delegate from Rule to RuleClass for the specific behavior of that rule (though unlike state, a + * Rule object never changes its RuleClass). This avoids the need to declare one Java class per + * class of Rule, yet achieves the same behavior.) + * + * <p>The use of a metaclass also allows us to compute a mapping from Attributes to small integers + * and share this between all rules of the same metaclass. This means we can save the attribute + * dictionary for each rule instance using an array, which is much more compact than a hashtable. + * + * <p>Rule classes whose names start with "$" are considered "abstract"; since they are not valid + * identifiers, they cannot be named in the build language. However, they are useful for grouping + * related attributes which are inherited. + * + * <p>The exact values in this class are important. In particular: + * <ul> + * <li>Changing an attribute from MANDATORY to OPTIONAL creates the potential for null-pointer + * exceptions in code that expects a value. + * <li>Attributes whose names are preceded by a "$" or a ":" are "hidden", and cannot be redefined + * in a BUILD file. They are a useful way of adding a special dependency. By convention, + * attributes starting with "$" are implicit dependencies, and those starting with a ":" are + * late-bound implicit dependencies, i.e. dependencies that can only be resolved when the + * configuration is known. + * <li>Attributes should not be introduced into the hierarchy higher then necessary. + * <li>The 'deps' and 'data' attributes are treated specially by the code that builds the runfiles + * tree. All targets appearing in these attributes appears beneath the ".runfiles" tree; in + * addition, "deps" may have rule-specific semantics. + * </ul> + */ +@Immutable +public final class RuleClass { + /** + * A constraint for the package name of the Rule instances. + */ + public static class PackageNameConstraint implements PredicateWithMessage<Rule> { + + public static final int ANY_SEGMENT = 0; + + private final int pathSegment; + + private final Set<String> values; + + /** + * The pathSegment-th segment of the package must be one of the specified values. + * The path segment indexing starts from 1. + */ + public PackageNameConstraint(int pathSegment, String... values) { + this.values = ImmutableSet.copyOf(values); + this.pathSegment = pathSegment; + } + + @Override + public boolean apply(Rule input) { + PathFragment path = input.getLabel().getPackageFragment(); + if (pathSegment == ANY_SEGMENT) { + return path.getFirstSegment(values) != PathFragment.INVALID_SEGMENT; + } else { + return path.segmentCount() >= pathSegment + && values.contains(path.getSegment(pathSegment - 1)); + } + } + + @Override + public String getErrorReason(Rule param) { + if (pathSegment == ANY_SEGMENT) { + return param.getRuleClass() + " rules have to be under a " + + StringUtil.joinEnglishList(values, "or", "'") + " directory"; + } else if (pathSegment == 1) { + return param.getRuleClass() + " rules are only allowed in " + + StringUtil.joinEnglishList(StringUtil.append(values, "//", ""), "or"); + } else { + return param.getRuleClass() + " rules are only allowed in packages which " + + StringUtil.ordinal(pathSegment) + " is " + StringUtil.joinEnglishList(values, "or"); + } + } + + @VisibleForTesting + public int getPathSegment() { + return pathSegment; + } + + @VisibleForTesting + public Collection<String> getValues() { + return values; + } + } + + /** + * Using this callback function, rules can override their own configuration during the + * analysis phase. + */ + public interface Configurator<TConfig, TRule> { + TConfig apply(TRule rule, TConfig configuration); + } + + /** + * A factory or builder class for rule implementations. + */ + public interface ConfiguredTargetFactory<TConfiguredTarget, TContext> { + /** + * Returns a fully initialized configured target instance using the given context. + */ + TConfiguredTarget create(TContext ruleContext) throws InterruptedException; + } + + /** + * Default rule configurator, it doesn't change the assigned configuration. + */ + public static final RuleClass.Configurator<Object, Object> NO_CHANGE = + new RuleClass.Configurator<Object, Object>() { + @Override + public Object apply(Object rule, Object configuration) { + return configuration; + } + }; + + /** + * For Bazel's constraint system: the attribute that declares the set of environments a rule + * supports, overriding the defaults for their respective groups. + */ + public static final String RESTRICTED_ENVIRONMENT_ATTR = "restricted_to"; + + /** + * For Bazel's constraint system: the attribute that declares the set of environments a rule + * supports, appending them to the defaults for their respective groups. + */ + public static final String COMPATIBLE_ENVIRONMENT_ATTR = "compatible_with"; + + /** + * For Bazel's constraint system: the implicit attribute used to store rule class restriction + * defaults as specified by {@link Builder#restrictedTo}. + */ + public static final String DEFAULT_RESTRICTED_ENVIRONMENT_ATTR = + "$" + RESTRICTED_ENVIRONMENT_ATTR; + + /** + * For Bazel's constraint system: the implicit attribute used to store rule class compatibility + * defaults as specified by {@link Builder#compatibleWith}. + */ + public static final String DEFAULT_COMPATIBLE_ENVIRONMENT_ATTR = + "$" + COMPATIBLE_ENVIRONMENT_ATTR; + + /** + * Checks if an attribute is part of the constraint system. + */ + public static boolean isConstraintAttribute(String attr) { + return RESTRICTED_ENVIRONMENT_ATTR.equals(attr) + || COMPATIBLE_ENVIRONMENT_ATTR.equals(attr) + || DEFAULT_RESTRICTED_ENVIRONMENT_ATTR.equals(attr) + || DEFAULT_COMPATIBLE_ENVIRONMENT_ATTR.equals(attr); + } + + /** + * A support class to make it easier to create {@code RuleClass} instances. + * This class follows the 'fluent builder' pattern. + * + * <p>The {@link #addAttribute} method will throw an exception if an attribute + * of that name already exists. Use {@link #overrideAttribute} in that case. + */ + public static final class Builder { + private static final Pattern RULE_NAME_PATTERN = Pattern.compile("[A-Za-z][A-Za-z0-9_]*"); + + /** + * The type of the rule class, which determines valid names and required + * attributes. + */ + public enum RuleClassType { + /** + * Abstract rules are intended for rule classes that are just used to + * factor out common attributes, and for rule classes that are used only + * internally. These rules cannot be instantiated by a BUILD file. + * + * <p>The rule name must contain a '$' and {@link + * TargetUtils#isTestRuleName} must return false for the name. + */ + ABSTRACT { + @Override + public void checkName(String name) { + Preconditions.checkArgument( + (name.contains("$") && !TargetUtils.isTestRuleName(name)) || name.equals("")); + } + + @Override + public void checkAttributes(Map<String, Attribute> attributes) { + // No required attributes. + } + }, + + /** + * Invisible rule classes should contain a dollar sign so that they cannot be instantiated + * by the user. They are different from abstract rules in that they can be instantiated + * at will. + */ + INVISIBLE { + @Override + public void checkName(String name) { + Preconditions.checkArgument(name.contains("$")); + } + + @Override + public void checkAttributes(Map<String, Attribute> attributes) { + // No required attributes. + } + }, + + /** + * Normal rules are instantiable by BUILD files. Their names must therefore + * obey the rules for identifiers in the BUILD language. In addition, + * {@link TargetUtils#isTestRuleName} must return false for the name. + */ + NORMAL { + @Override + public void checkName(String name) { + Preconditions.checkArgument(!TargetUtils.isTestRuleName(name) + && RULE_NAME_PATTERN.matcher(name).matches(), "Invalid rule name: " + name); + } + + @Override + public void checkAttributes(Map<String, Attribute> attributes) { + for (Attribute attribute : REQUIRED_ATTRIBUTES_FOR_NORMAL_RULES) { + Attribute presentAttribute = attributes.get(attribute.getName()); + Preconditions.checkState(presentAttribute != null, + "Missing mandatory '%s' attribute in normal rule class.", attribute.getName()); + Preconditions.checkState(presentAttribute.getType().equals(attribute.getType()), + "Mandatory attribute '%s' in normal rule class has incorrect type (expcected" + + " %s).", attribute.getName(), attribute.getType()); + } + } + }, + + /** + * Workspace rules can only be instantiated from a WORKSPACE file. Their names obey the + * rule for identifiers. + */ + WORKSPACE { + @Override + public void checkName(String name) { + Preconditions.checkArgument(RULE_NAME_PATTERN.matcher(name).matches()); + } + + @Override + public void checkAttributes(Map<String, Attribute> attributes) { + // No required attributes. + } + }, + + /** + * Test rules are instantiable by BUILD files and are handled specially + * when run with the 'test' command. Their names must obey the rules + * for identifiers in the BUILD language and {@link + * TargetUtils#isTestRuleName} must return true for the name. + * + * <p>In addition, test rules must contain certain attributes. See {@link + * Builder#REQUIRED_ATTRIBUTES_FOR_TESTS}. + */ + TEST { + @Override + public void checkName(String name) { + Preconditions.checkArgument(TargetUtils.isTestRuleName(name) + && RULE_NAME_PATTERN.matcher(name).matches()); + } + + @Override + public void checkAttributes(Map<String, Attribute> attributes) { + for (Attribute attribute : REQUIRED_ATTRIBUTES_FOR_TESTS) { + Attribute presentAttribute = attributes.get(attribute.getName()); + Preconditions.checkState(presentAttribute != null, + "Missing mandatory '%s' attribute in test rule class.", attribute.getName()); + Preconditions.checkState(presentAttribute.getType().equals(attribute.getType()), + "Mandatory attribute '%s' in test rule class has incorrect type (expcected %s).", + attribute.getName(), attribute.getType()); + } + } + }; + + /** + * Checks whether the given name is valid for the current rule class type. + * + * @throws IllegalArgumentException if the name is not valid + */ + public abstract void checkName(String name); + + /** + * Checks whether the given set of attributes contains all the required + * attributes for the current rule class type. + * + * @throws IllegalArgumentException if a required attribute is missing + */ + public abstract void checkAttributes(Map<String, Attribute> attributes); + } + + /** + * A predicate that filters rule classes based on their names. + */ + public static class RuleClassNamePredicate implements Predicate<RuleClass> { + + private final Set<String> ruleClasses; + + public RuleClassNamePredicate(Iterable<String> ruleClasses) { + this.ruleClasses = ImmutableSet.copyOf(ruleClasses); + } + + public RuleClassNamePredicate(String... ruleClasses) { + this.ruleClasses = ImmutableSet.copyOf(ruleClasses); + } + + public RuleClassNamePredicate() { + this(ImmutableSet.<String>of()); + } + + @Override + public boolean apply(RuleClass ruleClass) { + return ruleClasses.contains(ruleClass.getName()); + } + + @Override + public int hashCode() { + return ruleClasses.hashCode(); + } + + @Override + public boolean equals(Object o) { + return (o instanceof RuleClassNamePredicate) && + ruleClasses.equals(((RuleClassNamePredicate) o).ruleClasses); + } + + @Override + public String toString() { + return ruleClasses.isEmpty() ? "nothing" : StringUtil.joinEnglishList(ruleClasses); + } + } + + /** + * List of required attributes for normal rules, name and type. + */ + public static final List<Attribute> REQUIRED_ATTRIBUTES_FOR_NORMAL_RULES = ImmutableList.of( + attr("tags", Type.STRING_LIST).build() + ); + + /** + * List of required attributes for test rules, name and type. + */ + public static final List<Attribute> REQUIRED_ATTRIBUTES_FOR_TESTS = ImmutableList.of( + attr("tags", Type.STRING_LIST).build(), + attr("size", Type.STRING).build(), + attr("timeout", Type.STRING).build(), + attr("flaky", Type.BOOLEAN).build(), + attr("shard_count", Type.INTEGER).build(), + attr("local", Type.BOOLEAN).build() + ); + + private String name; + private final RuleClassType type; + private final boolean skylark; + private boolean documented; + private boolean publicByDefault = false; + private boolean binaryOutput = true; + private boolean workspaceOnly = false; + private boolean outputsDefaultExecutable = false; + private ImplicitOutputsFunction implicitOutputsFunction = ImplicitOutputsFunction.NONE; + private Configurator<?, ?> configurator = NO_CHANGE; + private ConfiguredTargetFactory<?, ?> configuredTargetFactory = null; + private PredicateWithMessage<Rule> validityPredicate = + PredicatesWithMessage.<Rule>alwaysTrue(); + private Predicate<String> preferredDependencyPredicate = Predicates.alwaysFalse(); + private List<Class<?>> advertisedProviders = new ArrayList<>(); + private UserDefinedFunction configuredTargetFunction = null; + private SkylarkEnvironment ruleDefinitionEnvironment = null; + private Set<Class<?>> configurationFragments = new LinkedHashSet<>(); + private boolean failIfMissingConfigurationFragment; + + private final Map<String, Attribute> attributes = new LinkedHashMap<>(); + + /** + * Constructs a new {@code RuleClassBuilder} using all attributes from all + * parent rule classes. An attribute cannot exist in more than one parent. + * + * <p>The rule type affects the the allowed names and the required + * attributes (see {@link RuleClassType}). + * + * @throws IllegalArgumentException if an attribute with the same name exists + * in more than one parent + */ + public Builder(String name, RuleClassType type, boolean skylark, RuleClass... parents) { + this.name = name; + this.skylark = skylark; + this.type = type; + this.documented = type != RuleClassType.ABSTRACT; + for (RuleClass parent : parents) { + if (parent.getValidityPredicate() != PredicatesWithMessage.<Rule>alwaysTrue()) { + setValidityPredicate(parent.getValidityPredicate()); + } + if (parent.preferredDependencyPredicate != Predicates.<String>alwaysFalse()) { + setPreferredDependencyPredicate(parent.preferredDependencyPredicate); + } + configurationFragments.addAll(parent.requiredConfigurationFragments); + failIfMissingConfigurationFragment |= parent.failIfMissingConfigurationFragment; + + for (Attribute attribute : parent.getAttributes()) { + String attrName = attribute.getName(); + Preconditions.checkArgument( + !attributes.containsKey(attrName) || attributes.get(attrName) == attribute, + String.format("Attribute %s is inherited multiple times in %s ruleclass", + attrName, name)); + attributes.put(attrName, attribute); + } + } + // TODO(bazel-team): move this testonly attribute setting to somewhere else + // preferably to some base RuleClass implementation. + if (this.type.equals(RuleClassType.TEST)) { + Attribute.Builder<Boolean> testOnlyAttr = attr("testonly", BOOLEAN).value(true) + .nonconfigurable("policy decision: this shouldn't depend on the configuration"); + if (attributes.containsKey("testonly")) { + override(testOnlyAttr); + } else { + add(testOnlyAttr); + } + } + } + + /** + * Checks that required attributes for test rules are present, creates the + * {@link RuleClass} object and returns it. + * + * @throws IllegalStateException if any of the required attributes is missing + */ + public RuleClass build() { + return build(name); + } + + /** + * Same as {@link #build} except with setting the name parameter. + */ + public RuleClass build(String name) { + Preconditions.checkArgument(this.name.isEmpty() || this.name.equals(name)); + type.checkName(name); + type.checkAttributes(attributes); + boolean skylarkExecutable = + skylark && (type == RuleClassType.NORMAL || type == RuleClassType.TEST); + Preconditions.checkState( + (type == RuleClassType.ABSTRACT) + == (configuredTargetFactory == null && configuredTargetFunction == null)); + Preconditions.checkState(skylarkExecutable == (configuredTargetFunction != null)); + Preconditions.checkState(skylarkExecutable == (ruleDefinitionEnvironment != null)); + return new RuleClass(name, skylarkExecutable, documented, publicByDefault, binaryOutput, + workspaceOnly, outputsDefaultExecutable, implicitOutputsFunction, configurator, + configuredTargetFactory, validityPredicate, preferredDependencyPredicate, + ImmutableSet.copyOf(advertisedProviders), configuredTargetFunction, + ruleDefinitionEnvironment, configurationFragments, failIfMissingConfigurationFragment, + attributes.values().toArray(new Attribute[0])); + } + + /** + * Declares that the implementation of this rule class requires the given configuration + * fragments to be present in the configuration. The value is inherited by subclasses. + * + * <p>For backwards compatibility, if the set is empty, all fragments may be accessed. But note + * that this is only enforced in the {@link com.google.devtools.build.lib.analysis.RuleContext} + * class. + */ + public Builder requiresConfigurationFragments(Class<?>... configurationFragment) { + Collections.addAll(configurationFragments, configurationFragment); + return this; + } + + public Builder failIfMissingConfigurationFragment() { + this.failIfMissingConfigurationFragment = true; + return this; + } + + public Builder setUndocumented() { + documented = false; + return this; + } + + public Builder publicByDefault() { + publicByDefault = true; + return this; + } + + public Builder setWorkspaceOnly() { + workspaceOnly = true; + return this; + } + + /** + * Determines the outputs of this rule to be created beneath the {@code + * genfiles} directory. By default, files are created beneath the {@code bin} + * directory. + * + * <p>This property is not inherited and this method should not be called by + * builder of {@link RuleClassType#ABSTRACT} rule class. + * + * @throws IllegalStateException if called for abstract rule class builder + */ + public Builder setOutputToGenfiles() { + Preconditions.checkState(type != RuleClassType.ABSTRACT, + "Setting not inherited property (output to genrules) of abstract rule class '%s'", name); + this.binaryOutput = false; + return this; + } + + /** + * Sets the implicit outputs function of the rule class. The default implicit + * outputs function is {@link ImplicitOutputsFunction#NONE}. + * + * <p>This property is not inherited and this method should not be called by + * builder of {@link RuleClassType#ABSTRACT} rule class. + * + * @throws IllegalStateException if called for abstract rule class builder + */ + public Builder setImplicitOutputsFunction( + ImplicitOutputsFunction implicitOutputsFunction) { + Preconditions.checkState(type != RuleClassType.ABSTRACT, + "Setting not inherited property (implicit output function) of abstract rule class '%s'", + name); + this.implicitOutputsFunction = implicitOutputsFunction; + return this; + } + + public Builder cfg(Configurator<?, ?> configurator) { + Preconditions.checkState(type != RuleClassType.ABSTRACT, + "Setting not inherited property (cfg) of abstract rule class '%s'", name); + this.configurator = configurator; + return this; + } + + public Builder factory(ConfiguredTargetFactory<?, ?> factory) { + this.configuredTargetFactory = factory; + return this; + } + + public Builder setValidityPredicate(PredicateWithMessage<Rule> predicate) { + this.validityPredicate = predicate; + return this; + } + + public Builder setPreferredDependencyPredicate(Predicate<String> predicate) { + this.preferredDependencyPredicate = predicate; + return this; + } + + /** + * State that the rule class being built possibly supplies the specified provider to its direct + * dependencies. + * + * <p>When computing the set of aspects required for a rule, only the providers listed here are + * considered. The presence of a provider here does not mean that the rule <b>must</b> implement + * said provider, merely that it <b>can</b>. After the configured target is constructed from + * this rule, aspects will be filtered according to the set of actual providers. + * + * <p>This is here so that we can do the loading phase overestimation required for + * "blaze query", which does not have the configured targets available. + * + * <p>It's okay for the rule class eventually not to supply it (possibly based on analysis phase + * logic), but if a provider is not advertised but is supplied, aspects that require the it will + * not be evaluated for the rule. + */ + public Builder advertiseProvider(Class<?>... providers) { + Collections.addAll(advertisedProviders, providers); + return this; + } + + private void addAttribute(Attribute attribute) { + Preconditions.checkState(!attributes.containsKey(attribute.getName()), + "An attribute with the name '%s' already exists.", attribute.getName()); + attributes.put(attribute.getName(), attribute); + } + + private void overrideAttribute(Attribute attribute) { + String attrName = attribute.getName(); + Preconditions.checkState(attributes.containsKey(attrName), + "No such attribute '%s' to override in ruleclass '%s'.", attrName, name); + Type<?> origType = attributes.get(attrName).getType(); + Type<?> newType = attribute.getType(); + Preconditions.checkState(origType.equals(newType), + "The type of the new attribute '%s' is different from the original one '%s'.", + newType, origType); + attributes.put(attrName, attribute); + } + + /** + * Builds attribute from the attribute builder and adds it to this rule + * class. + * + * @param attr attribute builder + */ + public <TYPE> Builder add(Attribute.Builder<TYPE> attr) { + addAttribute(attr.build()); + return this; + } + + /** + * Builds attribute from the attribute builder and overrides the attribute + * with the same name. + * + * @throws IllegalArgumentException if the attribute does not override one of the same name + */ + public <TYPE> Builder override(Attribute.Builder<TYPE> attr) { + overrideAttribute(attr.build()); + return this; + } + + /** + * Adds or overrides the attribute in the rule class. Meant for Skylark usage. + */ + public void addOrOverrideAttribute(Attribute attribute) { + if (attributes.containsKey(attribute.getName())) { + overrideAttribute(attribute); + } else { + addAttribute(attribute); + } + } + + /** + * Sets the rule implementation function. Meant for Skylark usage. + */ + public Builder setConfiguredTargetFunction(UserDefinedFunction func) { + this.configuredTargetFunction = func; + return this; + } + + /** + * Sets the rule definition environment. Meant for Skylark usage. + */ + public Builder setRuleDefinitionEnvironment(SkylarkEnvironment env) { + this.ruleDefinitionEnvironment = env; + return this; + } + + /** + * Removes an attribute with the same name from this rule class. + * + * @throws IllegalArgumentException if the attribute with this name does + * not exist + */ + public <TYPE> Builder removeAttribute(String name) { + Preconditions.checkState(attributes.containsKey(name), "No such attribute '%s' to remove.", + name); + attributes.remove(name); + return this; + } + + /** + * This rule class outputs a default executable for every rule with the same name as + * the rules's. Only works for Skylark. + */ + public <TYPE> Builder setOutputsDefaultExecutable() { + this.outputsDefaultExecutable = true; + return this; + } + + /** + * Declares that instances of this rule are compatible with the specified environments, + * in addition to the defaults declared by their environment groups. This can be overridden + * by rule-specific declarations. See + * {@link com.google.devtools.build.lib.analysis.constraints.ConstraintSemantics} for details. + */ + public <TYPE> Builder compatibleWith(Label... environments) { + add(attr(DEFAULT_COMPATIBLE_ENVIRONMENT_ATTR, LABEL_LIST).cfg(HOST) + .value(ImmutableList.copyOf(environments))); + return this; + } + + /** + * Declares that instances of this rule are restricted to the specified environments, i.e. + * these override the defaults declared by their environment groups. This can be overridden + * by rule-specific declarations. See + * {@link com.google.devtools.build.lib.analysis.constraints.ConstraintSemantics} for details. + * + * <p>The input list cannot be empty. + */ + public <TYPE> Builder restrictedTo(Label firstEnvironment, Label... otherEnvironments) { + ImmutableList<Label> environments = ImmutableList.<Label>builder().add(firstEnvironment) + .add(otherEnvironments).build(); + add(attr(DEFAULT_RESTRICTED_ENVIRONMENT_ATTR, LABEL_LIST).cfg(HOST).value(environments)); + return this; + + } + + /** + * Returns an Attribute.Builder object which contains a replica of the + * same attribute in the parent rule if exists. + * + * @param name the name of the attribute + */ + public Attribute.Builder<?> copy(String name) { + Preconditions.checkArgument(attributes.containsKey(name), + "Attribute %s does not exist in parent rule class.", name); + return attributes.get(name).cloneBuilder(); + } + } + + private final String name; // e.g. "cc_library" + + /** + * The kind of target represented by this RuleClass (e.g. "cc_library rule"). + * Note: Even though there is partial duplication with the {@link RuleClass#name} field, + * we want to store this as a separate field instead of generating it on demand in order to + * avoid string duplication. + */ + private final String targetKind; + + private final boolean skylarkExecutable; + private final boolean documented; + private final boolean publicByDefault; + private final boolean binaryOutput; + private final boolean workspaceOnly; + private final boolean outputsDefaultExecutable; + + /** + * A (unordered) mapping from attribute names to small integers indexing into + * the {@code attributes} array. + */ + private final Map<String, Integer> attributeIndex = new HashMap<>(); + + /** + * All attributes of this rule class (including inherited ones) ordered by + * attributeIndex value. + */ + private final Attribute[] attributes; + + /** + * The set of implicit outputs generated by a rule, expressed as a function + * of that rule. + */ + private final ImplicitOutputsFunction implicitOutputsFunction; + + /** + * The set of implicit outputs generated by a rule, expressed as a function + * of that rule. + */ + private final Configurator<?, ?> configurator; + + /** + * The factory that creates configured targets from this rule. + */ + private final ConfiguredTargetFactory<?, ?> configuredTargetFactory; + + /** + * The constraint the package name of the rule instance must fulfill + */ + private final PredicateWithMessage<Rule> validityPredicate; + + /** + * See {@link #isPreferredDependency}. + */ + private final Predicate<String> preferredDependencyPredicate; + + /** + * The list of transitive info providers this class advertises to aspects. + */ + private final ImmutableSet<Class<?>> advertisedProviders; + + /** + * The Skylark rule implementation of this RuleClass. Null for non Skylark executable RuleClasses. + */ + @Nullable private final UserDefinedFunction configuredTargetFunction; + + /** + * The Skylark rule definition environment of this RuleClass. + * Null for non Skylark executable RuleClasses. + */ + @Nullable private final SkylarkEnvironment ruleDefinitionEnvironment; + + /** + * The set of required configuration fragments; this should list all fragments that can be + * accessed by the rule implementation. If empty, all fragments are allowed to be accessed for + * backwards compatibility. + */ + private final ImmutableSet<Class<?>> requiredConfigurationFragments; + + /** + * Whether to fail during analysis if a configuration fragment is missing. The default behavior is + * to create fail actions for all declared outputs, i.e., to fail during execution, if any of the + * outputs is actually attempted to be built. + */ + private final boolean failIfMissingConfigurationFragment; + + /** + * Constructs an instance of RuleClass whose name is 'name', attributes + * are 'attributes'. The {@code srcsAllowedFiles} determines which types of + * files are allowed as parameters to the "srcs" attribute; rules are always + * allowed. For the "deps" attribute, there are four cases: + * <ul> + * <li>if the parameter is a file, it is allowed if its file type is given + * in {@code depsAllowedFiles}, + * <li>if the parameter is a rule and the rule class is accepted by + * {@code depsAllowedRules}, then it is allowed, + * <li>if the parameter is a rule and the rule class is not accepted by + * {@code depsAllowedRules}, but accepted by + * {@code depsAllowedRulesWithWarning}, then it is allowed, but + * triggers a warning; + * <li>all other parameters trigger an error. + * </ul> + * + * <p>The {@code depsAllowedRules} predicate should have a {@code toString} + * method which returns a plain English enumeration of the allowed rule class + * names, if it does not allow all rule classes. + * @param workspaceOnly + */ + @VisibleForTesting + RuleClass(String name, + boolean skylarkExecutable, boolean documented, boolean publicByDefault, + boolean binaryOutput, boolean workspaceOnly, boolean outputsDefaultExecutable, + ImplicitOutputsFunction implicitOutputsFunction, + Configurator<?, ?> configurator, + ConfiguredTargetFactory<?, ?> configuredTargetFactory, + PredicateWithMessage<Rule> validityPredicate, Predicate<String> preferredDependencyPredicate, + ImmutableSet<Class<?>> advertisedProviders, + @Nullable UserDefinedFunction configuredTargetFunction, + @Nullable SkylarkEnvironment ruleDefinitionEnvironment, + Set<Class<?>> allowedConfigurationFragments, boolean failIfMissingConfigurationFragment, + Attribute... attributes) { + this.name = name; + this.targetKind = name + " rule"; + this.skylarkExecutable = skylarkExecutable; + this.documented = documented; + this.publicByDefault = publicByDefault; + this.binaryOutput = binaryOutput; + this.implicitOutputsFunction = implicitOutputsFunction; + this.configurator = Preconditions.checkNotNull(configurator); + this.configuredTargetFactory = configuredTargetFactory; + this.validityPredicate = validityPredicate; + this.preferredDependencyPredicate = preferredDependencyPredicate; + this.advertisedProviders = advertisedProviders; + this.configuredTargetFunction = configuredTargetFunction; + this.ruleDefinitionEnvironment = ruleDefinitionEnvironment; + // Do not make a defensive copy as builder does that already + this.attributes = attributes; + this.workspaceOnly = workspaceOnly; + this.outputsDefaultExecutable = outputsDefaultExecutable; + this.requiredConfigurationFragments = ImmutableSet.copyOf(allowedConfigurationFragments); + this.failIfMissingConfigurationFragment = failIfMissingConfigurationFragment; + + // create the index: + int index = 0; + for (Attribute attribute : attributes) { + attributeIndex.put(attribute.getName(), index++); + } + } + + /** + * Returns the function which determines the set of implicit outputs + * generated by a given rule. + * + * <p>An implicit output is an OutputFile that automatically comes into + * existence when a rule of this class is declared, and whose name is derived + * from the name of the rule. + * + * <p>Implicit outputs are a widely-relied upon. All ".so", + * and "_deploy.jar" targets referenced in BUILD files are examples. + */ + @VisibleForTesting + public ImplicitOutputsFunction getImplicitOutputsFunction() { + return implicitOutputsFunction; + } + + @SuppressWarnings("unchecked") + public <C, R> Configurator<C, R> getConfigurator() { + return (Configurator<C, R>) configurator; + } + + @SuppressWarnings("unchecked") + public <CT, RC> ConfiguredTargetFactory<CT, RC> getConfiguredTargetFactory() { + return (ConfiguredTargetFactory<CT, RC>) configuredTargetFactory; + } + + /** + * Returns the class of rule that this RuleClass represents (e.g. "cc_library"). + */ + public String getName() { + return name; + } + + /** + * Returns the target kind of this class of rule (e.g. "cc_library rule"). + */ + String getTargetKind() { + return targetKind; + } + + public boolean getWorkspaceOnly() { + return workspaceOnly; + } + + /** + * Returns true iff the attribute 'attrName' is defined for this rule class, + * and has type 'type'. + */ + public boolean hasAttr(String attrName, Type<?> type) { + Integer index = getAttributeIndex(attrName); + return index != null && getAttribute(index).getType() == type; + } + + /** + * Returns the index of the specified attribute name. Use of indices allows + * space-efficient storage of attribute values in rules, since hashtables are + * not required. (The index mapping is specific to each RuleClass and an + * attribute may have a different index in the parent RuleClass.) + * + * <p>Returns null if the named attribute is not defined for this class of Rule. + */ + Integer getAttributeIndex(String attrName) { + return attributeIndex.get(attrName); + } + + /** + * Returns the attribute whose index is 'attrIndex'. Fails if attrIndex is + * not in range. + */ + Attribute getAttribute(int attrIndex) { + return attributes[attrIndex]; + } + + /** + * Returns the attribute whose name is 'attrName'; fails if not found. + */ + public Attribute getAttributeByName(String attrName) { + return attributes[getAttributeIndex(attrName)]; + } + + /** + * Returns the attribute whose name is {@code attrName}, or null if not + * found. + */ + Attribute getAttributeByNameMaybe(String attrName) { + Integer i = getAttributeIndex(attrName); + return i == null ? null : attributes[i]; + } + + /** + * Returns the number of attributes defined for this rule class. + */ + public int getAttributeCount() { + return attributeIndex.size(); + } + + /** + * Returns an (immutable) list of all Attributes defined for this class of + * rule, ordered by increasing index. + */ + public List<Attribute> getAttributes() { + return ImmutableList.copyOf(attributes); + } + + public PredicateWithMessage<Rule> getValidityPredicate() { + return validityPredicate; + } + + /** + * Returns the set of advertised transitive info providers. + * + * <p>When computing the set of aspects required for a rule, only the providers listed here are + * considered. The presence of a provider here does not mean that the rule <b>must</b> implement + * said provider, merely that it <b>can</b>. After the configured target is constructed from this + * rule, aspects will be filtered according to the set of actual providers. + * + * <p>This is here so that we can do the loading phase overestimation required for "blaze query", + * which does not have the configured targets available. + * + * <p>This should in theory only contain subclasses of + * {@link com.google.devtools.build.lib.analysis.TransitiveInfoProvider}, but our current dependency + * structure does not allow a reference to that class here. + */ + public ImmutableSet<Class<?>> getAdvertisedProviders() { + return advertisedProviders; + } + + /** + * For --compile_one_dependency: if multiple rules consume the specified target, + * should we choose this one over the "unpreferred" options? + */ + public boolean isPreferredDependency(String filename) { + return preferredDependencyPredicate.apply(filename); + } + + /** + * The set of required configuration fragments; this contains all fragments that can be + * accessed by the rule implementation. If empty, all fragments are allowed to be accessed for + * backwards compatibility. + */ + public Set<Class<?>> getRequiredConfigurationFragments() { + return requiredConfigurationFragments; + } + + /** + * Checks if the configuration fragment may be accessed (i.e., if it's declared). If no fragments + * are declared, this allows access to all fragments for backwards compatibility. + */ + public boolean isLegalConfigurationFragment(Class<?> configurationFragment) { + // For now, we allow all rules that don't declare allowed fragments to access any fragment. + // TODO(bazel-team): Declare fragment dependencies for all rules and remove this. + if (requiredConfigurationFragments.isEmpty()) { + return true; + } + return requiredConfigurationFragments.contains(configurationFragment); + } + + /** + * Whether to fail analysis if any of the required configuration fragments are missing. + */ + public boolean failIfMissingConfigurationFragment() { + return failIfMissingConfigurationFragment; + } + + /** + * Helper function for {@link RuleFactory#createRule}. + */ + Rule createRuleWithLabel(Package.AbstractBuilder<?, ?> pkgBuilder, Label ruleLabel, + Map<String, Object> attributeValues, EventHandler eventHandler, FuncallExpression ast, + Location location) throws SyntaxException { + Rule rule = pkgBuilder.newRuleWithLabel(ruleLabel, this, null, location); + createRuleCommon(rule, pkgBuilder, attributeValues, eventHandler, ast); + return rule; + } + + private void createRuleCommon(Rule rule, Package.AbstractBuilder<?, ?> pkgBuilder, + Map<String, Object> attributeValues, EventHandler eventHandler, FuncallExpression ast) + throws SyntaxException { + populateRuleAttributeValues( + rule, pkgBuilder, attributeValues, eventHandler, ast); + rule.populateOutputFiles(eventHandler, pkgBuilder); + rule.checkForNullLabels(); + rule.checkValidityPredicate(eventHandler); + } + + static class ParsedAttributeValue { + private final boolean explicitlySpecified; + private final Object value; + private final Location location; + + ParsedAttributeValue(boolean explicitlySpecified, Object value, Location location) { + this.explicitlySpecified = explicitlySpecified; + this.value = value; + this.location = location; + } + + public boolean getExplicitlySpecified() { + return explicitlySpecified; + } + + public Object getValue() { + return value; + } + + public Location getLocation() { + return location; + } + } + + /** + * Creates a rule with the attribute values that are already parsed. + * + * <p><b>WARNING:</b> This assumes that the attribute values here have the right type and + * bypasses some sanity checks. If they are of the wrong type, everything will come down burning. + */ + @SuppressWarnings("unchecked") + Rule createRuleWithParsedAttributeValues(Label label, + Package.AbstractBuilder<?, ?> pkgBuilder, Location ruleLocation, + Map<String, ParsedAttributeValue> attributeValues, EventHandler eventHandler) + throws SyntaxException{ + Rule rule = pkgBuilder.newRuleWithLabel(label, this, null, ruleLocation); + rule.checkValidityPredicate(eventHandler); + + for (Attribute attribute : rule.getRuleClassObject().getAttributes()) { + ParsedAttributeValue value = attributeValues.get(attribute.getName()); + if (attribute.isMandatory()) { + Preconditions.checkState(value != null); + } + + if (value == null) { + continue; + } + + checkAllowedValues(rule, attribute, value.getValue(), eventHandler); + rule.setAttributeValue(attribute, value.getValue(), value.getExplicitlySpecified()); + rule.setAttributeLocation(attribute, value.getLocation()); + + if (attribute.getName().equals("visibility")) { + // TODO(bazel-team): Verify that this cast works + rule.setVisibility(PackageFactory.getVisibility((List<Label>) value.getValue())); + } + } + + rule.populateOutputFiles(eventHandler, pkgBuilder); + Preconditions.checkState(!rule.containsErrors()); + return rule; + } + + /** + * Populates the attributes table of new rule "rule" from the + * "attributeValues" mapping from attribute names to values in the build + * language. Errors are reported on "reporter". "ast" is used to associate + * location information with each rule attribute. + */ + private void populateRuleAttributeValues(Rule rule, + Package.AbstractBuilder<?, ?> pkgBuilder, + Map<String, Object> attributeValues, + EventHandler eventHandler, + FuncallExpression ast) { + BitSet definedAttrs = new BitSet(); // set of attr indices + + for (Map.Entry<String, Object> entry : attributeValues.entrySet()) { + String attributeName = entry.getKey(); + Object attributeValue = entry.getValue(); + if (attributeValue == Environment.NONE) { // Ignore all None values. + continue; + } + Integer attrIndex = setRuleAttributeValue(rule, eventHandler, attributeName, attributeValue); + if (attrIndex != null) { + definedAttrs.set(attrIndex); + checkAttrValNonEmpty(rule, eventHandler, attributeValue, attrIndex); + } + } + + // Save the location of each non-default attribute definition: + if (ast != null) { + for (Argument arg : ast.getArguments()) { + Ident keyword = arg.getName(); + if (keyword != null) { + String name = keyword.getName(); + Integer attrIndex = getAttributeIndex(name); + if (attrIndex != null) { + rule.setAttributeLocation(attrIndex, arg.getValue().getLocation()); + } + } + } + } + + List<Attribute> attrsWithComputedDefaults = new ArrayList<>(); + + // Set defaults; ensure that every mandatory attribute has a value. Use + // the default if none is specified. + int numAttributes = getAttributeCount(); + for (int attrIndex = 0; attrIndex < numAttributes; ++attrIndex) { + if (!definedAttrs.get(attrIndex)) { + Attribute attr = getAttribute(attrIndex); + if (attr.isMandatory()) { + rule.reportError(rule.getLabel() + ": missing value for mandatory " + + "attribute '" + attr.getName() + "' in '" + + name + "' rule", eventHandler); + } + + if (attr.hasComputedDefault()) { + attrsWithComputedDefaults.add(attr); + } else { + Object defaultValue = getAttributeNoncomputedDefaultValue(attr, pkgBuilder); + checkAttrValNonEmpty(rule, eventHandler, defaultValue, attrIndex); + checkAllowedValues(rule, attr, defaultValue, eventHandler); + rule.setAttributeValue(attr, defaultValue, /*explicit=*/false); + } + } + } + + // Evaluate and set any computed defaults now that all non-computed + // TODO(bazel-team): remove this special casing. Thanks to configurable attributes refactoring, + // computed defaults don't get bound to their final values at this point, so we no longer + // have to wait until regular attributes have been initialized. + for (Attribute attr : attrsWithComputedDefaults) { + rule.setAttributeValue(attr, attr.getDefaultValue(rule), /*explicit=*/false); + } + + // Now that all attributes are bound to values, collect and store configurable attribute keys. + populateConfigDependenciesAttribute(rule); + checkForDuplicateLabels(rule, eventHandler); + checkThirdPartyRuleHasLicense(rule, pkgBuilder, eventHandler); + checkForValidSizeAndTimeoutValues(rule, eventHandler); + } + + /** + * Collects all labels used as keys for configurable attributes and places them into + * the special implicit attribute that tracks them. + */ + private static void populateConfigDependenciesAttribute(Rule rule) { + RawAttributeMapper attributes = RawAttributeMapper.of(rule); + Attribute configDepsAttribute = attributes.getAttributeDefinition("$config_dependencies"); + if (configDepsAttribute == null) { + // Not currently compatible with Skylark rules. + return; + } + + Set<Label> configLabels = new LinkedHashSet<>(); + for (Attribute attr : rule.getAttributes()) { + Type.Selector<?> selector = attributes.getSelector(attr.getName(), attr.getType()); + if (selector != null) { + for (Label label : selector.getEntries().keySet()) { + if (!Type.Selector.isReservedLabel(label)) { + configLabels.add(label); + } + } + } + } + + rule.setAttributeValue(configDepsAttribute, ImmutableList.copyOf(configLabels), + /*explicit=*/false); + } + + private void checkAttrValNonEmpty( + Rule rule, EventHandler eventHandler, Object attributeValue, Integer attrIndex) { + if (attributeValue instanceof List<?>) { + Attribute attr = getAttribute(attrIndex); + if (attr.isNonEmpty() && ((List<?>) attributeValue).isEmpty()) { + rule.reportError(rule.getLabel() + ": non empty " + "attribute '" + attr.getName() + + "' in '" + name + "' rule '" + rule.getLabel() + "' has to have at least one value", + eventHandler); + } + } + } + + /** + * Report an error for each label that appears more than once in a LABEL_LIST attribute + * of the given rule. + * + * @param rule The rule. + * @param eventHandler The eventHandler to use to report the duplicated deps. + */ + private static void checkForDuplicateLabels(Rule rule, EventHandler eventHandler) { + for (Attribute attribute : rule.getAttributes()) { + if (attribute.getType() == Type.LABEL_LIST) { + checkForDuplicateLabels(rule, attribute, eventHandler); + } + } + } + + /** + * Reports an error against the specified rule if it's beneath third_party + * but does not have a declared license. + */ + private static void checkThirdPartyRuleHasLicense(Rule rule, + Package.AbstractBuilder<?, ?> pkgBuilder, EventHandler eventHandler) { + if (rule.getLabel().getPackageName().startsWith("third_party/")) { + License license = rule.getLicense(); + if (license == null) { + license = pkgBuilder.getDefaultLicense(); + } + if (license == License.NO_LICENSE) { + rule.reportError("third-party rule '" + rule.getLabel() + "' lacks a license declaration " + + "with one of the following types: notice, reciprocal, permissive, " + + "restricted, unencumbered, by_exception_only", + eventHandler); + } + } + } + + /** + * Report an error for each label that appears more than once in the given attribute + * of the given rule. + * + * @param rule The rule. + * @param attribute The attribute to check. Must exist in rule and be of type LABEL_LIST. + * @param eventHandler The eventHandler to use to report the duplicated deps. + */ + private static void checkForDuplicateLabels(Rule rule, Attribute attribute, + EventHandler eventHandler) { + final String attrName = attribute.getName(); + // This attribute may be selectable, so iterate over each selection possibility in turn. + // TODO(bazel-team): merge '*' condition into all lists when implemented. + AggregatingAttributeMapper attributeMap = AggregatingAttributeMapper.of(rule); + for (List<Label> labels : attributeMap.visitAttribute(attrName, Type.LABEL_LIST)) { + if (!labels.isEmpty()) { + Set<Label> duplicates = CollectionUtils.duplicatedElementsOf(labels); + for (Label label : duplicates) { + rule.reportError( + String.format("Label '%s' is duplicated in the '%s' attribute of rule '%s'", + label, attrName, rule.getName()), eventHandler); + } + } + } + } + + /** + * Report an error if the rule has a timeout or size attribute that is not a + * legal value. These attributes appear on all tests. + * + * @param rule the rule to check + * @param eventHandler the eventHandler to use to report the duplicated deps + */ + private static void checkForValidSizeAndTimeoutValues(Rule rule, EventHandler eventHandler) { + if (rule.getRuleClassObject().hasAttr("size", Type.STRING)) { + String size = NonconfigurableAttributeMapper.of(rule).get("size", Type.STRING); + if (TestSize.getTestSize(size) == null) { + rule.reportError( + String.format("In rule '%s', size '%s' is not a valid size.", rule.getName(), size), + eventHandler); + } + } + if (rule.getRuleClassObject().hasAttr("timeout", Type.STRING)) { + String timeout = NonconfigurableAttributeMapper.of(rule).get("timeout", Type.STRING); + if (TestTimeout.getTestTimeout(timeout) == null) { + rule.reportError( + String.format( + "In rule '%s', timeout '%s' is not a valid timeout.", rule.getName(), timeout), + eventHandler); + } + } + } + + /** + * Returns the default value for the specified rule attribute. + * + * For most rule attributes, the default value is either explicitly specified + * in the attribute, or implicitly based on the type of the attribute, except + * for some special cases (e.g. "licenses", "distribs") where it comes from + * some other source, such as state in the package. + * + * Precondition: {@code !attr.hasComputedDefault()}. (Computed defaults are + * evaluated in second pass.) + */ + private static Object getAttributeNoncomputedDefaultValue(Attribute attr, + Package.AbstractBuilder<?, ?> pkgBuilder) { + if (attr.getName().equals("licenses")) { + return pkgBuilder.getDefaultLicense(); + } + if (attr.getName().equals("distribs")) { + return pkgBuilder.getDefaultDistribs(); + } + return attr.getDefaultValue(null); + } + + /** + * Sets the value of attribute "attrName" in rule "rule", by converting the + * build-language value "attrVal" to the appropriate type for the attribute. + * Returns the attribute index iff successful, null otherwise. + * + * <p>In case of failure, error messages are reported on "handler", and "rule" + * is marked as containing errors. + */ + @SuppressWarnings("unchecked") + private Integer setRuleAttributeValue(Rule rule, + EventHandler eventHandler, + String attrName, + Object attrVal) { + if (attrName.equals("name")) { + return null; // "name" is handled specially + } + + Integer attrIndex = getAttributeIndex(attrName); + if (attrIndex == null) { + rule.reportError(rule.getLabel() + ": no such attribute '" + attrName + + "' in '" + name + "' rule", eventHandler); + return null; + } + + Attribute attr = getAttribute(attrIndex); + Object converted; + try { + String what = "attribute '" + attrName + "' in '" + name + "' rule"; + converted = attr.getType().selectableConvert(attrVal, what, rule.getLabel()); + + if ((converted instanceof Type.Selector<?>) && !attr.isConfigurable()) { + rule.reportError(rule.getLabel() + ": attribute \"" + attr.getName() + + "\" is not configurable", eventHandler); + return null; + } + + if ((converted instanceof List<?>) && !(converted instanceof GlobList<?>)) { + if (attr.isOrderIndependent()) { + converted = Ordering.natural().sortedCopy((List<? extends Comparable<?>>) converted); + } + converted = ImmutableList.copyOf((List<?>) converted); + } + } catch (Type.ConversionException e) { + rule.reportError(rule.getLabel() + ": " + e.getMessage(), eventHandler); + return null; + } + + if (attrName.equals("visibility")) { + List<Label> attrList = (List<Label>) converted; + if (!attrList.isEmpty() && + ConstantRuleVisibility.LEGACY_PUBLIC_LABEL.equals(attrList.get(0))) { + rule.reportError(rule.getLabel() + ": //visibility:legacy_public only allowed in package " + + "declaration", eventHandler); + } + rule.setVisibility(PackageFactory.getVisibility(attrList)); + } + + checkAllowedValues(rule, attr, converted, eventHandler); + rule.setAttributeValue(attr, converted, /*explicit=*/true); + return attrIndex; + } + + private void checkAllowedValues(Rule rule, Attribute attribute, Object value, + EventHandler eventHandler) { + if (attribute.checkAllowedValues()) { + PredicateWithMessage<Object> allowedValues = attribute.getAllowedValues(); + if (!allowedValues.apply(value)) { + rule.reportError(String.format(rule.getLabel() + ": invalid value in '%s' attribute: %s", + attribute.getName(), + allowedValues.getErrorReason(value)), eventHandler); + } + } + } + + @Override + public String toString() { + return name; + } + + public boolean isDocumented() { + return documented; + } + + public boolean isPublicByDefault() { + return publicByDefault; + } + + /** + * Returns true iff the outputs of this rule should be created beneath the + * <i>bin</i> directory, false if beneath <i>genfiles</i>. For most rule + * classes, this is a constant, but for genrule, it is a property of the + * individual rule instance, derived from the 'output_to_bindir' attribute; + * see Rule.hasBinaryOutput(). + */ + boolean hasBinaryOutput() { + return binaryOutput; + } + + /** + * Returns this RuleClass's custom Skylark rule implementation. + */ + @Nullable public UserDefinedFunction getConfiguredTargetFunction() { + return configuredTargetFunction; + } + + /** + * Returns this RuleClass's rule definition environment. + */ + @Nullable public SkylarkEnvironment getRuleDefinitionEnvironment() { + return ruleDefinitionEnvironment; + } + + /** + * Returns true if this RuleClass is an executable Skylark RuleClass (i.e. it is + * Skylark and Normal or Test RuleClass). + */ + public boolean isSkylarkExecutable() { + return skylarkExecutable; + } + + /** + * Returns true if this rule class outputs a default executable for every rule. + */ + public boolean outputsDefaultExecutable() { + return outputsDefaultExecutable; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleClassProvider.java b/src/main/java/com/google/devtools/build/lib/packages/RuleClassProvider.java new file mode 100644 index 0000000..90fdfca --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/RuleClassProvider.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.ValidationEnvironment; + +import java.util.Map; + +/** + * The collection of the supported build rules. Provides an Environment for Skylark rule creation. + */ +public interface RuleClassProvider { + /** + * Returns a map from rule names to rule class objects. + */ + Map<String, RuleClass> getRuleClassMap(); + + /** + * Returns a new Skylark Environment instance for rule creation. Implementations need to be + * thread safe. + */ + SkylarkEnvironment createSkylarkRuleClassEnvironment( + EventHandler eventHandler, String astFileContentHashCode); + + /** + * Returns a validation environment for static analysis of skylark files. + * The environment has to contain all built-in functions and objects. + */ + ValidationEnvironment getSkylarkValidationEnvironment(); + + /** + * Returns the Skylark module to register the native rules with. + */ + Object getNativeModule(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleErrorConsumer.java b/src/main/java/com/google/devtools/build/lib/packages/RuleErrorConsumer.java new file mode 100644 index 0000000..84d00c0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/RuleErrorConsumer.java
@@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +/** + * A thin interface exposing only the warning and error reporting functionality + * of a rule. + * + * <p>When a class or a method needs only this functionality but not the whole + * {@code RuleConfiguredTarget}, it can use this thin interface instead. + * + * <p>This interface should only be implemented by {@code RuleConfiguredTarget} + * and its subclasses. + */ +public interface RuleErrorConsumer { + /** + * Consume a non-attribute-specific warning in a rule. + */ + void ruleWarning(String message); + + /** + * Consume a non-attribute-specific error in a rule. + */ + void ruleError(String message); + + /** + * Consume an attribute-specific warning in a rule. + */ + void attributeWarning(String attrName, String message); + + /** + * Consume an attribute-specific error in a rule. + */ + void attributeError(String attrName, String message); +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleFactory.java b/src/main/java/com/google/devtools/build/lib/packages/RuleFactory.java new file mode 100644 index 0000000..c79bbaa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/RuleFactory.java
@@ -0,0 +1,145 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.Package.NameConflictException; +import com.google.devtools.build.lib.packages.PackageFactory.PackageContext; +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Map; +import java.util.Set; + +/** + * Given a rule class and a set of attributes, returns a Rule instance. Also + * performs a number of checks and associates the rule and the owning package + * with each other. + * + * <p>Note: the code that actually populates the RuleClass map has been moved + * to {@link RuleClassProvider}. + */ +public class RuleFactory { + + /** + * Maps rule class name to the metaclass instance for that rule. + */ + private final ImmutableMap<String, RuleClass> ruleClassMap; + + /** + * Constructs a RuleFactory instance. + */ + public RuleFactory(RuleClassProvider provider) { + this.ruleClassMap = ImmutableMap.copyOf(provider.getRuleClassMap()); + } + + /** + * Returns the (immutable, unordered) set of names of all the known rule classes. + */ + public Set<String> getRuleClassNames() { + return ruleClassMap.keySet(); + } + + /** + * Returns the RuleClass for the specified rule class name. + */ + public RuleClass getRuleClass(String ruleClassName) { + return ruleClassMap.get(ruleClassName); + } + + /** + * Creates and returns a rule instance. + * + * <p>It is the caller's responsibility to add the rule to the package (the + * caller may choose not to do so if, for example, the rule has errors). + * + * @param pkgBuilder the under-construction package to which the rule belongs + * @param ruleClass the class of the rule; this must not be null + * @param attributeValues a map of attribute names to attribute values. Each + * attribute must be defined for this class of rule, and have a value + * of the appropriate type. There must be a map entry for each + * non-optional attribute of this class of rule. + * @param eventHandler a eventHandler on which errors and warnings are reported during + * rule creation + * @param ast the abstract syntax tree of the rule expression (optional) + * @param location the location at which this rule was declared + * @throws InvalidRuleException if the rule could not be constructed for any + * reason (e.g. no <code>name</code> attribute is defined) + * @throws NameConflictException + */ + static Rule createAndAddRule(Package.AbstractBuilder<?, ?> pkgBuilder, + RuleClass ruleClass, + Map<String, Object> attributeValues, + EventHandler eventHandler, + FuncallExpression ast, + Location location) throws InvalidRuleException, NameConflictException { + Preconditions.checkNotNull(ruleClass); + String ruleClassName = ruleClass.getName(); + Object nameObject = attributeValues.get("name"); + if (!(nameObject instanceof String)) { + throw new InvalidRuleException(ruleClassName + " rule has no 'name' attribute"); + } + String name = (String) nameObject; + Label label; + try { + // Test that this would form a valid label name -- in particular, this + // catches cases where Makefile variables $(foo) appear in "name". + label = pkgBuilder.createLabel(name); + } catch (Label.SyntaxException e) { + throw new InvalidRuleException("illegal rule name: " + name + ": " + e.getMessage()); + } + boolean inWorkspaceFile = location.getPath() != null + && location.getPath().endsWith(new PathFragment("WORKSPACE")); + if (ruleClass.getWorkspaceOnly() && !inWorkspaceFile) { + throw new RuleFactory.InvalidRuleException(ruleClass + " must be in the WORKSPACE file " + + "(used by " + label + ")"); + } else if (!ruleClass.getWorkspaceOnly() && inWorkspaceFile) { + throw new RuleFactory.InvalidRuleException(ruleClass + " cannot be in the WORKSPACE file " + + "(used by " + label + ")"); + } + + try { + Rule rule = ruleClass.createRuleWithLabel(pkgBuilder, label, attributeValues, + eventHandler, ast, location); + pkgBuilder.addRule(rule); + return rule; + } catch (SyntaxException e) { + throw new RuleFactory.InvalidRuleException(ruleClass + " " + e.getMessage()); + } + } + + public static Rule createAndAddRule(PackageContext context, + RuleClass ruleClass, + Map<String, Object> attributeValues, + FuncallExpression ast) throws InvalidRuleException, NameConflictException { + return createAndAddRule(context.pkgBuilder, ruleClass, attributeValues, context.eventHandler, + ast, ast.getLocation()); + } + + /** + * InvalidRuleException is thrown by createRule() if the Rule could not be + * constructed. It contains an error message. + */ + public static class InvalidRuleException extends Exception { + private InvalidRuleException(String message) { + super(message); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleVisibility.java b/src/main/java/com/google/devtools/build/lib/packages/RuleVisibility.java new file mode 100644 index 0000000..ef1a126 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/RuleVisibility.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.devtools.build.lib.syntax.Label; + +import java.util.List; + +/** + * A RuleVisibility specifies which other rules can depend on a specified rule. + * Note that the actual method that performs this check is declared in + * RuleConfiguredTargetVisibility. + * + * <p>The conversion to ConfiguredTargetVisibility is handled in an ugly + * if-ladder, because I want to avoid this package depending on build.lib.view. + * + * All implementations of this interface are immutable. + */ +public interface RuleVisibility { + /** + * Returns the list of labels that need to be loaded so that the visibility + * decision can be made during analysis time. E.g. for package group + * visibility, this is the list of package groups referenced. Does not include + * labels that have special meanings in the visibility declaration, e.g. + * "//visibility:*" or "//*:__pkg__". + */ + List<Label> getDependencyLabels(); + + /** + * Returns the list of labels used during the declaration of this visibility. + * These do not necessarily represent loadable labels: for example, for public + * or private visibilities, the special labels "//visibility:*" will be + * returned, and so will be the special "//*:__pkg__" labels indicating a + * single package. + */ + List<Label> getDeclaredLabels(); +} +
diff --git a/src/main/java/com/google/devtools/build/lib/packages/SkylarkFileType.java b/src/main/java/com/google/devtools/build/lib/packages/SkylarkFileType.java new file mode 100644 index 0000000..f6098cf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/SkylarkFileType.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileType.HasFilename; +import com.google.devtools.build.lib.util.FileTypeSet; + +/** + * A wrapper class for FileType and FileTypeSet functionality in Skylark. + */ +@SkylarkModule(name = "FileType", doc = "File type for file filtering.") +public class SkylarkFileType { + + private final FileType fileType; + + private SkylarkFileType(FileType fileType) { + this.fileType = fileType; + } + + public static SkylarkFileType of(Iterable<String> extensions) { + return new SkylarkFileType(FileType.of(extensions)); + } + + public FileTypeSet getFileTypeSet() { + return FileTypeSet.of(fileType); + } + + @SkylarkCallable(doc = "") + public ImmutableList<HasFilename> filter(Iterable<HasFilename> files) { + return ImmutableList.copyOf(FileType.filter(files, fileType)); + } + + @SkylarkCallable(doc = "") + public boolean matches(String fileName) { + return fileType.apply(fileName); + } + + @VisibleForTesting + public Object getExtensions() { + return fileType.getExtensions(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/Target.java b/src/main/java/com/google/devtools/build/lib/packages/Target.java new file mode 100644 index 0000000..ec5bc86 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/Target.java
@@ -0,0 +1,81 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.License.DistributionType; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; + +import java.util.Set; + +/** + * A node in the build dependency graph, identified by a Label. + */ +@SkylarkModule(name = "target", doc = "A BUILD target.") +public interface Target { + + /** + * Returns the label of this target. (e.g. "//foo:bar") + */ + @SkylarkCallable(name = "label", doc = "") + Label getLabel(); + + /** + * Returns the name of this rule (relative to its owning package). + */ + @SkylarkCallable(name = "name", doc = "") + String getName(); + + /** + * Returns the Package to which this rule belongs. + */ + Package getPackage(); + + /** + * Returns a string describing this kind of target: e.g. "cc_library rule", + * "source file", "generated file". + */ + String getTargetKind(); + + /** + * Returns the rule associated with this target, if any. + * + * If this is a Rule, returns itself; it this is an OutputFile, returns its + * generating rule; if this is an input file, returns null. + */ + Rule getAssociatedRule(); + + /** + * Returns the license associated with this target. + */ + License getLicense(); + + /** + * Returns the place where the target was defined. + */ + Location getLocation(); + + /** + * Returns the set of distribution types associated with this target. + */ + Set<DistributionType> getDistributions(); + + /** + * Returns the visibility of this target. + */ + RuleVisibility getVisibility(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/TargetUtils.java b/src/main/java/com/google/devtools/build/lib/packages/TargetUtils.java new file mode 100644 index 0000000..3710eeb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/TargetUtils.java
@@ -0,0 +1,265 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Utility functions over Targets that don't really belong in the base {@link + * Target} interface. + */ +public final class TargetUtils { + + // *_test / test_suite attribute that used to specify constraint keywords. + private static final String CONSTRAINTS_ATTR = "tags"; + + private TargetUtils() {} // Uninstantiable. + + public static boolean isTestRuleName(String name) { + return name.endsWith("_test"); + } + + public static boolean isTestSuiteRuleName(String name) { + return name.equals("test_suite"); + } + + /** + * Returns true iff {@code target} is a {@code *_test} rule; excludes {@code + * test_suite}. + */ + public static boolean isTestRule(Target target) { + return (target instanceof Rule) && isTestRuleName(((Rule) target).getRuleClass()); + } + + /** + * Returns true iff {@code target} is a {@code test_suite} rule. + */ + public static boolean isTestSuiteRule(Target target) { + return target instanceof Rule && + isTestSuiteRuleName(((Rule) target).getRuleClass()); + } + + /** + * Returns true iff {@code target} is a {@code *_test} or {@code test_suite}. + */ + public static boolean isTestOrTestSuiteRule(Target target) { + return isTestRule (target) || isTestSuiteRule(target); + } + + /** + * Returns true if {@code target} has "manual" in the tags attribute and thus should be ignored by + * command-line wildcards or by test_suite $implicit_tests attribute. + */ + public static boolean hasManualTag(Target target) { + return (target instanceof Rule) && hasConstraint((Rule) target, "manual"); + } + + /** + * Returns true if test marked as "exclusive" by the appropriate keyword + * in the tags attribute. + * + * Method assumes that passed target is a test rule, so usually it should be + * used only after isTestRule() or isTestOrTestSuiteRule(). Behavior is + * undefined otherwise. + */ + public static boolean isExclusiveTestRule(Rule rule) { + return hasConstraint(rule, "exclusive"); + } + + /** + * Returns true if test marked as "local" by the appropriate keyword + * in the tags attribute. + * + * Method assumes that passed target is a test rule, so usually it should be + * used only after isTestRule() or isTestOrTestSuiteRule(). Behavior is + * undefined otherwise. + */ + public static boolean isLocalTestRule(Rule rule) { + return hasConstraint(rule, "local") + || NonconfigurableAttributeMapper.of(rule).get("local", Type.BOOLEAN); + } + + /** + * Returns true if the rule is a test or test suite and is local or exclusive. + * Wraps the above calls into one generic check safely applicable to any rule. + */ + public static boolean isTestRuleAndRunsLocally(Rule rule) { + return isTestOrTestSuiteRule(rule) && + (isLocalTestRule(rule) || isExclusiveTestRule(rule)); + } + + /** + * Returns true if test marked as "external" by the appropriate keyword + * in the tags attribute. + * + * Method assumes that passed target is a test rule, so usually it should be + * used only after isTestRule() or isTestOrTestSuiteRule(). Behavior is + * undefined otherwise. + */ + public static boolean isExternalTestRule(Rule rule) { + return hasConstraint(rule, "external"); + } + + /** + * Returns true, iff the given target is a rule and it has the attribute + * <code>obsolete<code/> set to one. + */ + public static boolean isObsolete(Target target) { + if (!(target instanceof Rule)) { + return false; + } + Rule rule = (Rule) target; + return (rule.isAttrDefined("obsolete", Type.BOOLEAN)) + && NonconfigurableAttributeMapper.of(rule).get("obsolete", Type.BOOLEAN); + } + + /** + * If the given target is a rule, returns its <code>deprecation<code/> value, or null if unset. + */ + @Nullable + public static String getDeprecation(Target target) { + if (!(target instanceof Rule)) { + return null; + } + Rule rule = (Rule) target; + return (rule.isAttrDefined("deprecation", Type.STRING)) + ? NonconfigurableAttributeMapper.of(rule).get("deprecation", Type.STRING) + : null; + } + + /** + * Checks whether specified constraint keyword is present in the + * tags attribute of the test or test suite rule. + * + * Method assumes that provided rule is a test or a test suite. Behavior is + * undefined otherwise. + */ + private static boolean hasConstraint(Rule rule, String keyword) { + return NonconfigurableAttributeMapper.of(rule).get(CONSTRAINTS_ATTR, Type.STRING_LIST) + .contains(keyword); + } + + /** + * Returns the execution info. These include execution requirement + * tags ('requires-*' as well as "local") as keys with empty values. + */ + public static Map<String, String> getExecutionInfo(Rule rule) { + // tags may contain duplicate values. + Map<String, String> map = new HashMap<>(); + for (String tag : + NonconfigurableAttributeMapper.of(rule).get(CONSTRAINTS_ATTR, Type.STRING_LIST)) { + if (tag.startsWith("requires-") || tag.equals("local")) { + map.put(tag, ""); + } + } + return ImmutableMap.copyOf(map); + } + + /** + * Returns the language part of the rule name (e.g. "foo" for foo_test or foo_binary). + * + * <p>In practice this is the part before the "_", if any, otherwise the entire rule class name. + * + * <p>Precondition: isTestRule(target) || isRunnableNonTestRule(target). + */ + public static String getRuleLanguage(Target target) { + return getRuleLanguage(((Rule) target).getRuleClass()); + } + + /** + * Returns the language part of the rule name (e.g. "foo" for foo_test or foo_binary). + * + * <p>In practice this is the part before the "_", if any, otherwise the entire rule class name. + */ + public static String getRuleLanguage(String ruleClass) { + int index = ruleClass.lastIndexOf("_"); + // Chop off "_binary" or "_test". + return index != -1 ? ruleClass.substring(0, index) : ruleClass; + } + + private static boolean isExplicitDependency(Rule rule, Label label) { + if (rule.getVisibility().getDependencyLabels().contains(label)) { + return true; + } + + ExplicitEdgeVisitor visitor = new ExplicitEdgeVisitor(rule, label); + AggregatingAttributeMapper.of(rule).visitLabels(visitor); + return visitor.isExplicit(); + } + + private static class ExplicitEdgeVisitor implements AttributeMap.AcceptsLabelAttribute { + private final Label expectedLabel; + private final Rule rule; + private boolean isExplicit = false; + + public ExplicitEdgeVisitor(Rule rule, Label expected) { + this.rule = rule; + this.expectedLabel = expected; + } + + @Override + public void acceptLabelAttribute(Label label, Attribute attr) { + if (isExplicit || !rule.isAttributeValueExplicitlySpecified(attr)) { + // Nothing to do here. + } else if (expectedLabel.equals(label)) { + isExplicit = true; + } + } + + public boolean isExplicit() { + return isExplicit; + } + } + + /** + * Return {@link Location} for {@link Target} target, if it should not be null. + */ + public static Location getLocationMaybe(Target target) { + return (target instanceof Rule) || (target instanceof InputFile) ? target.getLocation() : null; + } + + /** + * Return nicely formatted error message that {@link Label} label that was pointed to by + * {@link Target} target did not exist, due to {@link NoSuchThingException} e. + */ + public static String formatMissingEdge(@Nullable Target target, Label label, + NoSuchThingException e) { + // instanceof returns false if target is null (which is exploited here) + if (target instanceof Rule) { + Rule rule = (Rule) target; + return !isExplicitDependency(rule, label) + ? ("every rule of type " + rule.getRuleClass() + " implicitly depends upon the target '" + + label + "', but this target could not be found. " + + "If this is an integration test, maybe you forgot to add a mock for your new tool?") + : e.getMessage() + " and referenced by '" + target.getLabel() + "'"; + } else if (target instanceof InputFile) { + return e.getMessage() + " (this is usually caused by a missing package group in the" + + " package-level visibility declaration)"; + } else { + if (target != null) { + return "in target '" + target.getLabel() + "', no such label '" + label + "': " + + e.getMessage(); + } + return e.getMessage(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/TestSize.java b/src/main/java/com/google/devtools/build/lib/packages/TestSize.java new file mode 100644 index 0000000..425a343 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/TestSize.java
@@ -0,0 +1,123 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.Set; + +/** + * Possible test sizes. + * + * Test size may affect the way how test is executed - e.g., it will determine + * default timeout value and estimated local resource usage. + */ +public enum TestSize { + + // Small tests use small amount of memory, but CPU intensive. + SMALL(TestTimeout.SHORT, 2), + // Medium tests tend to use larger amount of memory. + MEDIUM(TestTimeout.MODERATE, 10), + // All other tests estimated to use fairly large amount of memory. + LARGE(TestTimeout.LONG, 20), + ENORMOUS(TestTimeout.ETERNAL, 30); + + private final TestTimeout timeout; + private final int defaultShards; + + private TestSize(TestTimeout defaultTimeout, int defaultShards) { + this.timeout = defaultTimeout; + this.defaultShards = defaultShards; + } + + /** + * Returns default timeout in seconds. + */ + public TestTimeout getDefaultTimeout() { + return timeout; + } + + /** + * Returns default number of shards. + */ + public int getDefaultShards() { return defaultShards; } + + /** + * Returns test size of the given test target, or null if the size attribute is unrecognized. + */ + public static TestSize getTestSize(Rule testTarget) { + String attr = NonconfigurableAttributeMapper.of(testTarget).get("size", Type.STRING); + return getTestSize(attr); + } + + /** + * Returns {@link TestSize} matching the given timeout or null if the + * given timeout doesn't match any {@link TestSize}. + * + * @param timeout The timeout associated with the desired TestSize. + */ + public static TestSize getTestSize(TestTimeout timeout) { + for (TestSize size : TestSize.values()) { + if (size.timeout == timeout) { + return size; + } + } + return null; + } + + /** + * Normal practice is to always use size tags as lower case strings. + */ + @Override + public String toString() { + return super.toString().toLowerCase(); + } + + /** + * Returns the enum associated with a test's size or null if the tag is + * not lower case or an unknown size. + */ + public static TestSize getTestSize(String attr) { + if (!attr.equals(attr.toLowerCase())) { + return null; + } + try { + return TestSize.valueOf(attr.toUpperCase()); + } catch (IllegalArgumentException e) { + return null; + } + } + + /** + * Converter for the --test_size_filters option. + */ + public static class TestSizeFilterConverter extends EnumFilterConverter<TestSize> { + public TestSizeFilterConverter() { + super(TestSize.class, "test size"); + } + + /** + * {@inheritDoc} + * + * <p>This override is necessary to prevent OptionsData + * from throwing a "must be assignable from the converter return type" exception. + * OptionsData doesn't recognize the generic type and actual type are the same. + */ + @Override + public final Set<TestSize> convert(String input) throws OptionsParsingException { + return super.convert(input); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/TestTargetUtils.java b/src/main/java/com/google/devtools/build/lib/packages/TestTargetUtils.java new file mode 100644 index 0000000..dbd4dae --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/TestTargetUtils.java
@@ -0,0 +1,404 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.cmdline.ResolvedTargets; +import com.google.devtools.build.lib.cmdline.TargetParsingException; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.pkgcache.TargetProvider; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Utility functions over test Targets that don't really belong in the base {@link Target} + * interface. + */ +public final class TestTargetUtils { + /** + * Returns a predicate to be used for test size filtering, i.e., that only accepts tests of the + * given size. + */ + public static Predicate<Target> testSizeFilter(final Set<TestSize> allowedSizes) { + return new Predicate<Target>() { + @Override + public boolean apply(Target target) { + if (!(target instanceof Rule)) { + return false; + } + return allowedSizes.contains(TestSize.getTestSize((Rule) target)); + } + }; + } + + /** + * Returns a predicate to be used for test timeout filtering, i.e., that only accepts tests of + * the given timeout. + **/ + public static Predicate<Target> testTimeoutFilter(final Set<TestTimeout> allowedTimeouts) { + return new Predicate<Target>() { + @Override + public boolean apply(Target target) { + if (!(target instanceof Rule)) { + return false; + } + return allowedTimeouts.contains(TestTimeout.getTestTimeout((Rule) target)); + } + }; + } + + /** + * Returns a predicate to be used for test language filtering, i.e., that only accepts tests of + * the specified languages. The reporter and the list of rule names are only used to warn about + * unknown languages. + */ + public static Predicate<Target> testLangFilter(List<String> langFilterList, + EventHandler reporter, Set<String> allRuleNames) { + final Set<String> requiredLangs = new HashSet<>(); + final Set<String> excludedLangs = new HashSet<>(); + + for (String lang : langFilterList) { + if (lang.startsWith("-")) { + lang = lang.substring(1); + excludedLangs.add(lang); + } else { + requiredLangs.add(lang); + } + if (!allRuleNames.contains(lang + "_test")) { + reporter.handle( + Event.warn("Unknown language '" + lang + "' in --test_lang_filters option")); + } + } + + return new Predicate<Target>() { + @Override + public boolean apply(Target rule) { + String ruleLang = TargetUtils.getRuleLanguage(rule); + return (requiredLangs.isEmpty() || requiredLangs.contains(ruleLang)) + && !excludedLangs.contains(ruleLang); + } + }; + } + + /** + * Returns whether a test with the specified tags matches a filter (as specified by the set + * of its positive and its negative filters). + */ + public static boolean testMatchesFilters(Collection<String> testTags, + Collection<String> requiredTags, Collection<String> excludedTags, + boolean mustMatchAllPositive) { + + for (String tag : excludedTags) { + if (testTags.contains(tag)) { + return false; + } + } + + // Check required tags, if there are any. + if (!requiredTags.isEmpty()) { + if (mustMatchAllPositive) { + // Require all tags to be present. + for (String tag : requiredTags) { + if (!testTags.contains(tag)) { + return false; + } + } + return true; + } else { + // Require at least one positive tag. + for (String tag : requiredTags) { + if (testTags.contains(tag)) { + return true; + } + } + } + + return false; // No positive tag found. + } + + return true; // No tags are required. + } + + /** + * Returns a predicate to be used for test tag filtering, i.e., that only accepts tests that match + * all of the required tags and none of the excluded tags. + */ + // TODO(bazel-team): This also applies to non-test rules, so should probably be moved to + // TargetUtils. + public static Predicate<Target> tagFilter(List<String> tagFilterList) { + Pair<Collection<String>, Collection<String>> tagLists = sortTagsBySense(tagFilterList); + final Collection<String> requiredTags = tagLists.first; + final Collection<String> excludedTags = tagLists.second; + return new Predicate<Target>() { + @Override + public boolean apply(Target input) { + if (!(input instanceof Rule)) { + return false; + } + // Note that test_tags are those originating from the XX_test rule, + // whereas the requiredTags and excludedTags originate from the command + // line or test_suite rule. + return testMatchesFilters(((Rule) input).getRuleTags(), + requiredTags, excludedTags, false); + } + }; + } + + /** + * Separates a list of text "tags" into a Pair of Collections, where + * the first element are the required or positive tags and the second element + * are the excluded or negative tags. + * This should work on tag list provided from the command line + * --test_tags_filters flag or on tag filters explicitly declared in the + * suite. + * + * @param tagList A collection of text targets to separate. + */ + public static Pair<Collection<String>, Collection<String>> sortTagsBySense( + Iterable<String> tagList) { + Collection<String> requiredTags = new HashSet<>(); + Collection<String> excludedTags = new HashSet<>(); + + for (String tag : tagList) { + if (tag.startsWith("-")) { + excludedTags.add(tag.substring(1)); + } else if (tag.startsWith("+")) { + requiredTags.add(tag.substring(1)); + } else if (tag.equals("manual")) { + // Ignore manual attribute because it is an exception: it is not a filter + // but a property of test_suite + continue; + } else { + requiredTags.add(tag); + } + } + return Pair.of(requiredTags, excludedTags); + } + + /** + * Returns the (new, mutable) set of test rules, expanding all 'test_suite' rules into the + * individual tests they group together and preserving other test target instances. + * + * Method assumes that passed collection contains only *_test and test_suite rules. While, at this + * point it will successfully preserve non-test rules as well, there is no guarantee that this + * behavior will be kept in the future. + * + * @param targetProvider a target provider + * @param eventHandler a failure eventHandler to report loading failures to + * @param targets Collection of the *_test and test_suite configured targets + * @return a duplicate-free iterable of the tests under the specified targets + */ + public static ResolvedTargets<Target> expandTestSuites(TargetProvider targetProvider, + EventHandler eventHandler, Iterable<? extends Target> targets, boolean strict, + boolean keepGoing) + throws TargetParsingException { + Closure closure = new Closure(targetProvider, eventHandler, strict, keepGoing); + ResolvedTargets.Builder<Target> result = ResolvedTargets.builder(); + for (Target target : targets) { + if (TargetUtils.isTestRule(target)) { + result.add(target); + } else if (TargetUtils.isTestSuiteRule(target)) { + result.addAll(closure.getTestsInSuite((Rule) target)); + } else { + result.add(target); + } + } + if (closure.hasError) { + result.setError(); + } + return result.build(); + } + + // TODO(bazel-team): This is a copy of TestsExpression.Closure with some minor changes; this + // should be unified. + private static final class Closure { + private final TargetProvider targetProvider; + + private final EventHandler eventHandler; + + private final boolean keepGoing; + + private final boolean strict; + + private final Map<Target, Set<Target>> testsInSuite = new HashMap<>(); + + private boolean hasError; + + public Closure(TargetProvider targetProvider, EventHandler eventHandler, boolean strict, + boolean keepGoing) { + this.targetProvider = targetProvider; + this.eventHandler = eventHandler; + this.strict = strict; + this.keepGoing = keepGoing; + } + + /** + * Computes and returns the set of test rules in a particular suite. Uses + * dynamic programming---a memoized version of {@link #computeTestsInSuite}. + */ + private Set<Target> getTestsInSuite(Rule testSuite) throws TargetParsingException { + Set<Target> tests = testsInSuite.get(testSuite); + if (tests == null) { + tests = Sets.newHashSet(); + testsInSuite.put(testSuite, tests); // break cycles by inserting empty set early. + computeTestsInSuite(testSuite, tests); + } + return tests; + } + + /** + * Populates 'result' with all the tests associated with the specified + * 'testSuite'. Throws an exception if any target is missing. + * + * CAUTION! Keep this logic consistent with {@code TestsSuiteConfiguredTarget}! + */ + private void computeTestsInSuite(Rule testSuite, Set<Target> result) + throws TargetParsingException { + List<Target> testsAndSuites = new ArrayList<>(); + // Note that testsAndSuites can contain input file targets; the test_suite rule does not + // restrict the set of targets that can appear in tests or suites. + testsAndSuites.addAll(getPrerequisites(testSuite, "tests")); + testsAndSuites.addAll(getPrerequisites(testSuite, "suites")); + + // 1. Add all tests + for (Target test : testsAndSuites) { + if (TargetUtils.isTestRule(test)) { + result.add(test); + } else if (strict && !TargetUtils.isTestSuiteRule(test)) { + // If strict mode is enabled, then give an error for any non-test, non-test-suite targets. + eventHandler.handle(Event.error(testSuite.getLocation(), + "in test_suite rule '" + testSuite.getLabel() + + "': expecting a test or a test_suite rule but '" + test.getLabel() + + "' is not one.")); + hasError = true; + if (!keepGoing) { + throw new TargetParsingException("Test suite expansion failed."); + } + } + } + + // 2. Add implicit dependencies on tests in same package, if any. + for (Target target : getPrerequisites(testSuite, "$implicit_tests")) { + // The Package construction of $implicit_tests ensures that this check never fails, but we + // add it here anyway for compatibility with future code. + if (TargetUtils.isTestRule(target)) { + result.add(target); + } + } + + // 3. Filter based on tags, size, env. + filterTests(testSuite, result); + + // 4. Expand all suites recursively. + for (Target suite : testsAndSuites) { + if (TargetUtils.isTestSuiteRule(suite)) { + result.addAll(getTestsInSuite((Rule) suite)); + } + } + } + + /** + * Returns the set of rules named by the attribute 'attrName' of test_suite rule 'testSuite'. + * The attribute must be a list of labels. If a target cannot be resolved, then an error is + * reported to the environment (which may throw an exception if {@code keep_going} is disabled). + */ + private Collection<Target> getPrerequisites(Rule testSuite, String attrName) + throws TargetParsingException { + try { + List<Target> targets = new ArrayList<>(); + // TODO(bazel-team): This serializes package loading in some cases. We might want to make + // this multi-threaded. + for (Label label : + NonconfigurableAttributeMapper.of(testSuite).get(attrName, Type.LABEL_LIST)) { + targets.add(targetProvider.getTarget(eventHandler, label)); + } + return targets; + } catch (NoSuchThingException e) { + if (keepGoing) { + hasError = true; + eventHandler.handle(Event.error(e.getMessage())); + return ImmutableList.of(); + } + throw new TargetParsingException(e.getMessage(), e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TargetParsingException("interrupted", e); + } + } + + /** + * Filters 'tests' (by mutation) according to the 'tags' attribute, specifically those that + * match ALL of the tags in tagsAttribute. + * + * @precondition {@code env.getAccessor().isTestSuite(testSuite)} + * @precondition {@code env.getAccessor().isTestRule(test)} for all test in tests + */ + private void filterTests(Rule testSuite, Set<Target> tests) { + List<String> tagsAttribute = + NonconfigurableAttributeMapper.of(testSuite).get("tags", Type.STRING_LIST); + // Split the tags list into positive and negative tags + Pair<Collection<String>, Collection<String>> tagLists = sortTagsBySense(tagsAttribute); + Collection<String> positiveTags = tagLists.first; + Collection<String> negativeTags = tagLists.second; + + Iterator<Target> it = tests.iterator(); + while (it.hasNext()) { + Rule test = (Rule) it.next(); + AttributeMap nonConfigurableAttributes = NonconfigurableAttributeMapper.of(test); + List<String> testTags = + new ArrayList<>(nonConfigurableAttributes.get("tags", Type.STRING_LIST)); + testTags.add(nonConfigurableAttributes.get("size", Type.STRING)); + if (!includeTest(testTags, positiveTags, negativeTags)) { + it.remove(); + } + } + } + + /** + * Decides whether to include a test in a test_suite or not. + * @param testTags Collection of all tags exhibited by a given test. + * @param positiveTags Tags declared by the suite. A Test must match ALL of these. + * @param negativeTags Tags declared by the suite. A Test must match NONE of these. + * @return false is the test is to be removed. + */ + private static boolean includeTest(Collection<String> testTags, + Collection<String> positiveTags, Collection<String> negativeTags) { + // Add this test if it matches ALL of the positive tags and NONE of the + // negative tags in the tags attribute. + for (String tag : negativeTags) { + if (testTags.contains(tag)) { + return false; + } + } + for (String tag : positiveTags) { + if (!testTags.contains(tag)) { + return false; + } + } + return true; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/TestTimeout.java b/src/main/java/com/google/devtools/build/lib/packages/TestTimeout.java new file mode 100644 index 0000000..cfa8047 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/TestTimeout.java
@@ -0,0 +1,198 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.base.Splitter; +import com.google.common.collect.Maps; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Symbolic labels of test timeout. Borrows heavily from {@link TestSize}. + */ +public enum TestTimeout { + + // These symbolic labels are used in the build files. + SHORT(0, 60, 60), + MODERATE(30, 300, 300), + LONG(300, 900, 900), + ETERNAL(900, 365 * 24 * 60 /* One year */, 3600); + + /** + * Default --test_timeout flag, used when collecting code coverage. + */ + public static String COVERAGE_CMD_TIMEOUT = "--test_timeout=300,600,1200,3600"; + + private final Integer rangeMin; + private final Integer rangeMax; + private final Integer timeout; + + private TestTimeout(Integer rangeMin, Integer rangeMax, Integer timeout) { + this.rangeMin = rangeMin; + this.rangeMax = rangeMax; + this.timeout = timeout; + } + + /** + * Returns the enum associated with a test's timeout or null if the tag is + * not lower case or an unknown size. + */ + public static TestTimeout getTestTimeout(String attr) { + if (!attr.equals(attr.toLowerCase())) { + return null; + } + try { + return TestTimeout.valueOf(attr.toUpperCase()); + } catch (IllegalArgumentException e) { + return null; + } + } + + @Override + public String toString() { + return super.toString().toLowerCase(); + } + + /** + * We print to upper case to make the test timeout warnings more readable. + */ + public String prettyPrint() { + return super.toString().toUpperCase(); + } + + public Integer getTimeout() { + return timeout; + } + /** + * Returns true iff the given time in seconds is exactly in the range of valid + * execution times for this TestSize. + */ + public boolean isInRangeExact(Integer timeInSeconds) { + return timeInSeconds >= rangeMin && timeInSeconds < rangeMax; + } + + /** + * Returns true iff the given time in seconds is approximately (+/- 75%) in the range of valid + * execution times for this TestSize. + */ + public boolean isInRangeFuzzy(Integer timeInSeconds) { + return timeInSeconds >= rangeMin - (rangeMin * .75) + && (this == ETERNAL || timeInSeconds <= rangeMax + (rangeMax * .75)); + } + + /** + * Returns suggested test size for the given time in seconds. + */ + public static TestTimeout getSuggestedTestTimeout(Integer timeInSeconds) { + for (TestTimeout testTimeout : values()) { + if (testTimeout.isInRangeExact(timeInSeconds)) { + return testTimeout; + } + } + return ETERNAL; + } + + /** + * Returns test timeout of the given test target using explicitly specified timeout + * or default through to the size label's associated default. + */ + public static TestTimeout getTestTimeout(Rule testTarget) { + String attr = NonconfigurableAttributeMapper.of(testTarget).get("timeout", Type.STRING); + if (!attr.equals(attr.toLowerCase())) { + return null; // attribute values must be lowercase + } + try { + return TestTimeout.valueOf(attr.toUpperCase()); + } catch (IllegalArgumentException e) { + return null; + } + } + + /** + * Converter for the --test_timeout option. + */ + public static class TestTimeoutConverter implements Converter<Map<TestTimeout, Integer>> { + public TestTimeoutConverter() {} + + @Override + public Map<TestTimeout, Integer> convert(String input) throws OptionsParsingException { + List<Integer> values = new ArrayList<>(); + for (String token : Splitter.on(',').limit(6).split(input)) { + // Handle the case of "2," which is accepted as legal... Because Splitter.split is lazy, + // there's no way of knowing if an empty string is a trailing or an intermediate one, + // so we can't fully emulate String.split(String, 0). + if (!token.isEmpty() || values.size() > 1) { + try { + values.add(Integer.valueOf(token)); + } catch (NumberFormatException e) { + throw new OptionsParsingException("'" + input + "' is not an int"); + } + } + } + EnumMap<TestTimeout, Integer> timeouts = Maps.newEnumMap(TestTimeout.class); + if (values.size() == 1) { + timeouts.put(SHORT, values.get(0)); + timeouts.put(MODERATE, values.get(0)); + timeouts.put(LONG, values.get(0)); + timeouts.put(ETERNAL, values.get(0)); + } else if (values.size() == 4) { + timeouts.put(SHORT, values.get(0)); + timeouts.put(MODERATE, values.get(1)); + timeouts.put(LONG, values.get(2)); + timeouts.put(ETERNAL, values.get(3)); + } else { + throw new OptionsParsingException("Invalid number of comma-separated entries"); + } + for (TestTimeout label : values()) { + if (!timeouts.containsKey(label) || timeouts.get(label) <= 0) { + timeouts.put(label, label.getTimeout()); + } + } + return timeouts; + } + + @Override + public String getTypeDescription() { + return "a single integer or comma-separated list of 4 integers"; + } + } + + /** + * Converter for the --test_timeout_filters option. + */ + public static class TestTimeoutFilterConverter extends EnumFilterConverter<TestTimeout> { + public TestTimeoutFilterConverter() { + super(TestTimeout.class, "test timeout"); + } + + /** + * {@inheritDoc} + * + * <p>This override is necessary to prevent OptionsData + * from throwing a "must be assignable from the converter return type" exception. + * OptionsData doesn't recognize the generic type and actual type are the same. + */ + @Override + public final Set<TestTimeout> convert(String input) throws OptionsParsingException { + return super.convert(input); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/TriState.java b/src/main/java/com/google/devtools/build/lib/packages/TriState.java new file mode 100644 index 0000000..0cea28a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/TriState.java
@@ -0,0 +1,22 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +/** + * Enum used to represent tri-state parameters in rule attributes (yes/no/auto). + */ +public enum TriState { + YES, NO, AUTO +}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/Type.java b/src/main/java/com/google/devtools/build/lib/packages/Type.java new file mode 100644 index 0000000..9172a1f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/Type.java
@@ -0,0 +1,1025 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.packages.License.DistributionType; +import com.google.devtools.build.lib.packages.License.LicenseParsingException; +import com.google.devtools.build.lib.syntax.EvalUtils; +import com.google.devtools.build.lib.syntax.FilesetEntry; +import com.google.devtools.build.lib.syntax.GlobList; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SelectorValue; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.StringCanonicalizer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.logging.Level; + +import javax.annotation.Nullable; + +/** + * <p>Root of Type symbol hierarchy for values in the build language.</p> + * + * <p>Type symbols are primarily used for their <code>convert</code> method, + * which is a kind of cast operator enabling conversion from untyped (Object) + * references to values in the build language, to typed references.</p> + * + * <p>For example, this code type-converts a value <code>x</code> returned by + * the evaluator, to a list of strings:</p> + * + * <pre> + * Object x = expr.eval(env); + * List<String> s = Type.STRING_LIST.convert(x); + * </pre> + */ +public abstract class Type<T> { + + private Type() {} + + /** + * Converts untyped Object x resulting from the evaluation of an expression in the build language, + * into a typed object of type T. + * + * <p>x must be *directly* convertible to this type. This therefore disqualifies "selector + * expressions" of the form "{ config1: 'value1_of_orig_type', config2: 'value2_of_orig_type; }" + * (which support configurable attributes). To handle those expressions, see + * {@link #selectableConvert}. + * + * @param x the build-interpreter value to convert. + * @param what a string description of what x is for; should be included in + * any exception thrown. Grammatically, must describe a syntactic + * construct, e.g. "attribute 'srcs' of rule foo". + * @param currentRule the label of the current BUILD rule; must be non-null if resolution of + * package-relative label strings is required + * @throws ConversionException if there was a problem performing the type conversion + */ + public abstract T convert(Object x, String what, @Nullable Label currentRule) + throws ConversionException; + // TODO(bazel-team): Check external calls (e.g. in PackageFactory), verify they always want + // this over selectableConvert. + + /** + * Equivalent to <code>convert(x, null)</code>. Useful for converting values to types that do not + * involve the type <code>LABEL</code> and hence do not require the label of the current package. + */ + public final T convert(Object x, String what) throws ConversionException { + return convert(x, what, null); + } + + /** + * Variation of {@link #convert} that supports selector expressions for configurable attributes + * (i.e. "{ config1: 'value1_of_orig_type', config2: 'value2_of_orig_type; }"). If x is a + * selector expression, returns a {@link Selector} instance that contains key-mapped entries + * of the native type. Else, returns the native type directly. + * + * <p>The caller is responsible for casting the returned value appropriately. + */ + public Object selectableConvert(Object x, String what, @Nullable Label currentRule) + throws ConversionException { + if (x instanceof SelectorValue) { + return new Selector<T>(((SelectorValue) x).getDictionary(), what, currentRule, this); + } + return convert(x, what, currentRule); + } + + public abstract T cast(Object value); + + @Override + public abstract String toString(); + + /** + * Returns the default value for this type; may return null iff no default is defined for this + * type. + */ + public abstract T getDefaultValue(); + + /** + * If this type contains labels (e.g. it *is* a label or it's a collection of labels), + * returns a list of those labels for a value of that type. If this type doesn't + * contain labels, returns an empty list. + * + * <p>This is used to support reliable label visitation in + * {@link AbstractAttributeMapper#visitLabels}. To preserve that reliability, every + * type should faithfully define its own instance of this method. In other words, + * be careful about defining default instances in base types that get auto-inherited + * by their children. Keep all definitions as explicit as possible. + */ + public abstract Iterable<Label> getLabels(Object value); + + /** + * {@link #getLabels} return value for types that don't contain labels. + */ + private static final Iterable<Label> NO_LABELS_HERE = ImmutableList.of(); + + /** + * Converts an initialized Type object into a tag set representation. + * This operation is only valid for certain sub-Types which are guaranteed + * to be properly initialized. + * + * @param value the actual value + * @throws UnsupportedOperationException if the concrete type does not support + * tag conversion or if a convertible type has no initialized value. + */ + public Set<String> toTagSet(Object value, String name) { + String msg = "Attribute " + name + " does not support tag conversion."; + throw new UnsupportedOperationException(msg); + } + + /** + * The type of an integer. + */ + public static final Type<Integer> INTEGER = new IntegerType(); + + /** + * The type of a string. + */ + public static final Type<String> STRING = new StringType(); + + /** + * The type of a boolean. + */ + public static final Type<Boolean> BOOLEAN = new BooleanType(); + + /** + * The type of a TriState with values: true (x>0), false (x==0), auto (x<0). + */ + public static final Type<TriState> TRISTATE = new TriStateType(); + + /** + * The type of a label. Labels are not actually a first-class datatype in + * the build language, but they are so frequently used in the definitions of + * attributes that it's worth treating them specially (and providing support + * for resolution of relative-labels in the <code>convert()</code> method). + */ + public static final Type<Label> LABEL = new LabelType(); + + /** + * This is a label type that does not cause dependencies. It is needed because + * certain rules want to verify the type of a target referenced by one of their attributes, but + * if there was a dependency edge there, it would be a circular dependency. + */ + public static final Type<Label> NODEP_LABEL = new LabelType(); + + /** + * The type of a license. Like Label, licenses aren't first-class, but + * they're important enough to justify early syntax error detection. + */ + public static final Type<License> LICENSE = new LicenseType(); + + /** + * The type of a single distribution. Only used internally, as a type + * symbol, not a converter. + */ + public static final Type<DistributionType> DISTRIBUTION = new Type<DistributionType>() { + @Override + public DistributionType cast(Object value) { + return (DistributionType) value; + } + + @Override + public DistributionType convert(Object x, String what, Label currentRule) { + throw new UnsupportedOperationException(); + } + + @Override + public DistributionType getDefaultValue() { + return null; + } + + @Override + public Iterable<Label> getLabels(Object value) { + return NO_LABELS_HERE; + } + + @Override + public String toString() { + return "distribution"; + } + }; + + /** + * The type of a set of distributions. Distributions are not a first-class type, + * but they do warrant early syntax checking. + */ + public static final Type<Set<DistributionType>> DISTRIBUTIONS = new Distributions(); + + /** + * The type of an output file, treated as a {@link #LABEL}. + */ + public static final Type<Label> OUTPUT = new OutputType(); + + /** + * The type of a FilesetEntry attribute inside a Fileset. + */ + public static final Type<FilesetEntry> FILESET_ENTRY = new FilesetEntryType(); + + /** + * The type of a list of not-yet-typed objects. + */ + public static final ObjectListType OBJECT_LIST = new ObjectListType(); + + /** + * The type of a list of {@linkplain #STRING strings}. + */ + public static final ListType<String> STRING_LIST = ListType.create(STRING); + + /** + * The type of a list of {@linkplain #INTEGER strings}. + */ + public static final ListType<Integer> INTEGER_LIST = ListType.create(INTEGER); + + /** + * The type of a dictionary of {@linkplain #STRING strings}. + */ + public static final DictType<String, String> STRING_DICT = DictType.create(STRING, STRING); + + /** + * The type of a list of {@linkplain #OUTPUT outputs}. + */ + public static final ListType<Label> OUTPUT_LIST = ListType.create(OUTPUT); + + /** + * The type of a list of {@linkplain #LABEL labels}. + */ + public static final ListType<Label> LABEL_LIST = ListType.create(LABEL); + + /** + * The type of a list of {@linkplain #NODEP_LABEL labels} that do not cause + * dependencies. + */ + public static final ListType<Label> NODEP_LABEL_LIST = ListType.create(NODEP_LABEL); + + /** + * The type of a dictionary of {@linkplain #STRING_LIST label lists}. + */ + public static final DictType<String, List<String>> STRING_LIST_DICT = + DictType.create(STRING, STRING_LIST); + + /** + * The type of a dictionary of {@linkplain #STRING strings}, where each entry + * maps to a single string value. + */ + public static final DictType<String, String> STRING_DICT_UNARY = DictType.create(STRING, STRING); + + /** + * The type of a dictionary of {@linkplain #LABEL_LIST label lists}. + */ + public static final DictType<String, List<Label>> LABEL_LIST_DICT = + DictType.create(STRING, LABEL_LIST); + + /** + * The type of a list of {@linkplain #FILESET_ENTRY FilesetEntries}. + */ + public static final ListType<FilesetEntry> FILESET_ENTRY_LIST = ListType.create(FILESET_ENTRY); + + /** + * For ListType objects, returns the type of the elements of the list; for + * all other types, returns null. (This non-obvious implementation strategy + * is necessitated by the wildcard capture rules of the Java type system, + * which disallow conversion from Type{List{ELEM}} to Type{List{?}}.) + */ + public Type<?> getListElementType() { + return null; + } + + /** + * ConversionException is thrown when a type-conversion fails; it contains + * an explanatory error message. + */ + public static class ConversionException extends Exception { + private static String message(Type<?> type, Object value, String what) { + StringBuilder builder = new StringBuilder(); + builder.append("expected value of type '").append(type).append("'"); + if (what != null) { + builder.append(" for ").append(what); + } + builder.append(", but got '"); + EvalUtils.printValue(value, builder); + builder.append("' (").append(EvalUtils.getDatatypeName(value)).append(")"); + return builder.toString(); + } + + private ConversionException(Type<?> type, Object value, String what) { + super(message(type, value, what)); + } + + private ConversionException(String message) { + super(message); + } + } + + /******************************************************************** + * * + * Subclasses * + * * + ********************************************************************/ + + private static class ObjectType extends Type<Object> { + @Override + public Object cast(Object value) { + return value; + } + + @Override + public String getDefaultValue() { + throw new UnsupportedOperationException( + "ObjectType has no default value"); + } + + @Override + public Iterable<Label> getLabels(Object value) { + return NO_LABELS_HERE; + } + + @Override + public String toString() { + return "object"; + } + + @Override + public Object convert(Object x, String what, Label currentRule) { + return x; + } + } + + private static class IntegerType extends Type<Integer> { + @Override + public Integer cast(Object value) { + return (Integer) value; + } + + @Override + public Integer getDefaultValue() { + return 0; + } + + @Override + public Iterable<Label> getLabels(Object value) { + return NO_LABELS_HERE; + } + + @Override + public String toString() { + return "int"; + } + + @Override + public Integer convert(Object x, String what, Label currentRule) + throws ConversionException { + if (!(x instanceof Integer)) { + throw new ConversionException(this, x, what); + } + return (Integer) x; + } + } + + private static class BooleanType extends Type<Boolean> { + @Override + public Boolean cast(Object value) { + return (Boolean) value; + } + + @Override + public Boolean getDefaultValue() { + return false; + } + + @Override + public Iterable<Label> getLabels(Object value) { + return NO_LABELS_HERE; + } + + @Override + public String toString() { + return "boolean"; + } + + // Conversion to boolean must also tolerate integers of 0 and 1 only. + @Override + public Boolean convert(Object x, String what, Label currentRule) + throws ConversionException { + if (x instanceof Boolean) { + return (Boolean) x; + } + Integer xAsInteger = INTEGER.convert(x, what, currentRule); + if (xAsInteger == 0) { + return false; + } else if (xAsInteger == 1) { + return true; + } + throw new ConversionException("boolean is not one of [0, 1]"); + } + + /** + * Booleans attributes are converted to tags based on their names. + */ + @Override + public Set<String> toTagSet(Object value, String name) { + if (value == null) { + String msg = "Illegal tag conversion from null on Attribute " + name + "."; + throw new IllegalStateException(msg); + } + String tag = (Boolean) value ? name : "no" + name; + return new ImmutableSet.Builder<String>() + .add(tag) + .build(); + } + } + + /** + * Tristate values are needed for cases where user intent matters. + * + * <p>Tristate values are not explicitly interchangeable with booleans and are + * handled explicitly as TriStates. Prefer Booleans with default values where + * possible. The main use case for TriState values is when a Rule's behavior + * must interact with a Flag value in a complicated way.</p> + */ + private static class TriStateType extends Type<TriState> { + @Override + public TriState cast(Object value) { + return (TriState) value; + } + + @Override + public TriState getDefaultValue() { + return TriState.AUTO; + } + + @Override + public Iterable<Label> getLabels(Object value) { + return NO_LABELS_HERE; + } + + @Override + public String toString() { + return "tristate"; + } + + // Like BooleanType, this must handle integers as well. + @Override + public TriState convert(Object x, String what, Label currentRule) + throws ConversionException { + if (x instanceof TriState) { + return (TriState) x; + } + if (x instanceof Boolean) { + return ((Boolean) x) ? TriState.YES : TriState.NO; + } + Integer xAsInteger = INTEGER.convert(x, what, currentRule); + if (xAsInteger == -1) { + return TriState.AUTO; + } else if (xAsInteger == 1) { + return TriState.YES; + } else if (xAsInteger == 0) { + return TriState.NO; + } + throw new ConversionException(this, x, "TriState values is not one of [-1, 0, 1]"); + } + } + + private static class StringType extends Type<String> { + @Override + public String cast(Object value) { + return (String) value; + } + + @Override + public String getDefaultValue() { + return ""; + } + + @Override + public Iterable<Label> getLabels(Object value) { + return NO_LABELS_HERE; + } + + @Override + public String toString() { + return "string"; + } + + @Override + public String convert(Object x, String what, Label currentRule) + throws ConversionException { + if (!(x instanceof String)) { + throw new ConversionException(this, x, what); + } + return StringCanonicalizer.intern((String) x); + } + + /** + * A String is representable as a set containing its value. + */ + @Override + public Set<String> toTagSet(Object value, String name) { + if (value == null) { + String msg = "Illegal tag conversion from null on Attribute " + name + "."; + throw new IllegalStateException(msg); + } + return new ImmutableSet.Builder<String>() + .add((String) value) + .build(); + } + } + + private static class FilesetEntryType extends Type<FilesetEntry> { + @Override + public FilesetEntry cast(Object value) { + return (FilesetEntry) value; + } + + @Override + public FilesetEntry convert(Object x, String what, Label currentRule) + throws ConversionException { + if (!(x instanceof FilesetEntry)) { + throw new ConversionException(this, x, what); + } + return (FilesetEntry) x; + } + + @Override + public String toString() { + return "FilesetEntry"; + } + + @Override + public FilesetEntry getDefaultValue() { + return null; + } + + @Override + public Iterable<Label> getLabels(Object value) { + return cast(value).getLabels(); + } + } + + private static class LabelType extends Type<Label> { + @Override + public Label cast(Object value) { + return (Label) value; + } + + @Override + public Label getDefaultValue() { + return null; // Labels have no default value + } + + @Override + public Iterable<Label> getLabels(Object value) { + return ImmutableList.of(cast(value)); + } + + @Override + public String toString() { + return "label"; + } + + @Override + public Label convert(Object x, String what, Label currentRule) + throws ConversionException { + if (x instanceof Label) { + return (Label) x; + } + try { + return currentRule.getRelative( + STRING.convert(x, what, currentRule)); + } catch (Label.SyntaxException e) { + throw new ConversionException("invalid label '" + x + "' in " + + what + ": "+ e.getMessage()); + } + } + } + + /** + * Like Label, LicenseType is a derived type, which is declared specially + * in order to allow syntax validation. It represents the licenses, as + * described in {@ref License}. + */ + public static class LicenseType extends Type<License> { + @Override + public License cast(Object value) { + return (License) value; + } + + @Override + public License convert(Object x, String what, Label currentRule) throws ConversionException { + try { + List<String> licenseStrings = STRING_LIST.convert(x, what); + return License.parseLicense(licenseStrings); + } catch (LicenseParsingException e) { + throw new ConversionException(e.getMessage()); + } + } + + @Override + public License getDefaultValue() { + return License.NO_LICENSE; + } + + @Override + public Iterable<Label> getLabels(Object value) { + return NO_LABELS_HERE; + } + + @Override + public String toString() { + return "license"; + } + } + + /** + * Like Label, Distributions is a derived type, which is declared specially + * in order to allow syntax validation. It represents the declared distributions + * of a target, as described in {@ref License}. + */ + private static class Distributions extends Type<Set<DistributionType>> { + @SuppressWarnings("unchecked") + @Override + public Set<DistributionType> cast(Object value) { + return (Set<DistributionType>) value; + } + + @Override + public Set<DistributionType> convert(Object x, String what, Label currentRule) + throws ConversionException { + try { + List<String> distribStrings = STRING_LIST.convert(x, what); + return License.parseDistributions(distribStrings); + } catch (LicenseParsingException e) { + throw new ConversionException(e.getMessage()); + } + } + + @Override + public Set<DistributionType> getDefaultValue() { + return Collections.emptySet(); + } + + @Override + public Iterable<Label> getLabels(Object what) { + return NO_LABELS_HERE; + } + + @Override + public String toString() { + return "distributions"; + } + + @Override + public Type<DistributionType> getListElementType() { + return DISTRIBUTION; + } + } + + private static class OutputType extends Type<Label> { + @Override + public Label cast(Object value) { + return (Label) value; + } + + @Override + public Label getDefaultValue() { + return null; + } + + @Override + public Iterable<Label> getLabels(Object value) { + return ImmutableList.of(cast(value)); + } + + @Override + public String toString() { + return "output"; + } + + @Override + public Label convert(Object x, String what, Label currentRule) + throws ConversionException { + + String value; + try { + value = STRING.convert(x, what, currentRule); + } catch (ConversionException e) { + throw new ConversionException(this, x, what); + } + try { + // Enforce value is relative to the currentRule. + Label result = currentRule.getRelative(value); + if (!result.getPackageName().equals(currentRule.getPackageName())) { + throw new ConversionException("label '" + value + "' is not in the current package"); + } + return result; + } catch (Label.SyntaxException e) { + throw new ConversionException( + "illegal output file name '" + value + "' in rule " + currentRule + ": " + + e.getMessage()); + } + } + } + + /** + * A type to support dictionary attributes. + */ + public static class DictType<KEY, VALUE> extends Type<Map<KEY, VALUE>> { + + private final Type<KEY> keyType; + private final Type<VALUE> valueType; + + private final Map<KEY, VALUE> empty = ImmutableMap.of(); + + private static <KEY, VALUE> DictType<KEY, VALUE> create( + Type<KEY> keyType, Type<VALUE> valueType) { + return new DictType<>(keyType, valueType); + } + + private DictType(Type<KEY> keyType, Type<VALUE> valueType) { + this.keyType = keyType; + this.valueType = valueType; + } + + public Type<KEY> getKeyType() { + return keyType; + } + + public Type<VALUE> getValueType() { + return valueType; + } + + @SuppressWarnings("unchecked") + @Override + public Map<KEY, VALUE> cast(Object value) { + return (Map<KEY, VALUE>) value; + } + + @Override + public String toString() { + return "dict(" + keyType + ", " + valueType + ")"; + } + + @Override + public Map<KEY, VALUE> convert(Object x, String what, Label currentRule) + throws ConversionException { + if (!(x instanceof Map<?, ?>)) { + throw new ConversionException(String.format( + "Expected a map for dictionary but got a %s", x.getClass().getName())); + } + ImmutableMap.Builder<KEY, VALUE> result = ImmutableMap.builder(); + Map<?, ?> o = (Map<?, ?>) x; + for (Entry<?, ?> elem : o.entrySet()) { + result.put( + keyType.convert(elem.getKey(), "dict key element", currentRule), + valueType.convert(elem.getValue(), "dict value element", currentRule)); + } + return result.build(); + } + + @Override + public Map<KEY, VALUE> getDefaultValue() { + return empty; + } + + @Override + public Iterable<Label> getLabels(Object value) { + ImmutableList.Builder<Label> labels = ImmutableList.builder(); + for (Map.Entry<KEY, VALUE> entry : cast(value).entrySet()) { + labels.addAll(keyType.getLabels(entry.getKey())); + labels.addAll(valueType.getLabels(entry.getValue())); + } + return labels.build(); + } + } + + public static class ListType<ELEM> extends Type<List<ELEM>> { + + private final Type<ELEM> elemType; + + private final List<ELEM> empty = ImmutableList.of(); + + private static <ELEM> ListType<ELEM> create(Type<ELEM> elemType) { + return new ListType<>(elemType); + } + + private ListType(Type<ELEM> elemType) { + this.elemType = elemType; + } + + @SuppressWarnings("unchecked") + @Override + public List<ELEM> cast(Object value) { + return (List<ELEM>) value; + } + + @Override + public Type<ELEM> getListElementType() { + return elemType; + } + + @Override + public List<ELEM> getDefaultValue() { + return empty; + } + + @Override + public Iterable<Label> getLabels(Object value) { + ImmutableList.Builder<Label> labels = ImmutableList.builder(); + for (ELEM entry : cast(value)) { + labels.addAll(elemType.getLabels(entry)); + } + return labels.build(); + } + + @Override + public String toString() { + return "list(" + elemType + ")"; + } + + @Override + public List<ELEM> convert(Object x, String what, Label currentRule) + throws ConversionException { + if (!(x instanceof Iterable<?>)) { + throw new ConversionException(this, x, what); + } + List<ELEM> result = new ArrayList<>(); + int index = 0; + for (Object elem : (Iterable<?>) x) { + ELEM converted = elemType.convert(elem, "element " + index + " of " + what, currentRule); + if (converted != null) { + result.add(converted); + } else { + // shouldn't happen but it does, rarely + String message = "Converting a list with a null element: " + + "element " + index + " of " + what + " in " + currentRule; + LoggingUtil.logToRemote(Level.WARNING, message, + new ConversionException(message)); + } + ++index; + } + if (x instanceof GlobList<?>) { + return new GlobList<>(((GlobList<?>) x).getCriteria(), result); + } else { + return result; + } + } + + /** + * A list is representable as a tag set as the contents of itself expressed + * as Strings. So a List<String> is effectively converted to a Set<String>. + */ + @Override + public Set<String> toTagSet(Object items, String name) { + if (items == null) { + String msg = "Illegal tag conversion from null on Attribute" + name + "."; + throw new IllegalStateException(msg); + } + Set<String> tags = new LinkedHashSet<>(); + @SuppressWarnings("unchecked") + List<ELEM> itemsAsListofElem = (List<ELEM>) items; + for (ELEM element : itemsAsListofElem) { + tags.add(element.toString()); + } + return tags; + } + } + + public static class ObjectListType extends ListType<Object> { + + private static final Type<Object> elemType = new ObjectType(); + + private ObjectListType() { + super(elemType); + } + + @Override + @SuppressWarnings("unchecked") + public List<Object> convert(Object x, String what, Label currentRule) + throws ConversionException { + if (x instanceof List) { + return (List<Object>) x; + } else if (x instanceof Iterable) { + return ImmutableList.copyOf((Iterable<?>) x); + } else { + throw new ConversionException(this, x, what); + } + } + } + + /** + * The type of a general list. + */ + public static final ListType<Object> LIST = new ListType<>(new ObjectType()); + + /** + * Returns whether the specified type is a label type or not. + */ + public static boolean isLabelType(Type<?> type) { + return type == LABEL || type == LABEL_LIST + || type == NODEP_LABEL || type == NODEP_LABEL_LIST + || type == LABEL_LIST_DICT || type == FILESET_ENTRY_LIST; + } + + /** + * Special Type that represents a selector expression for configurable attributes. Holds a + * mapping of <Label, T> entries, where keys are configurability patterns and values are + * objects of the attribute's native Type. + */ + public static final class Selector<T> { + + private final Type<T> originalType; + private final Map<Label, T> map; + private final Label defaultConditionLabel; + private final boolean hasDefaultCondition; + + /** + * Value to use when none of an attribute's selection criteria match. + */ + @VisibleForTesting + public static final String DEFAULT_CONDITION_KEY = "//conditions:default"; + + @VisibleForTesting + Selector(Object x, String what, @Nullable Label currentRule, Type<T> originalType) + throws ConversionException { + Preconditions.checkState(x instanceof Map<?, ?>); + + try { + defaultConditionLabel = Label.parseAbsolute(DEFAULT_CONDITION_KEY); + } catch (Label.SyntaxException e) { + throw new IllegalStateException(DEFAULT_CONDITION_KEY + " is not a valid label"); + } + + + this.originalType = originalType; + Map<Label, T> result = Maps.newLinkedHashMap(); + boolean foundDefaultCondition = false; + for (Entry<?, ?> entry : ((Map<?, ?>) x).entrySet()) { + Label key = LABEL.convert(entry.getKey(), what, currentRule); + if (key.equals(defaultConditionLabel)) { + foundDefaultCondition = true; + } + result.put(key, originalType.convert(entry.getValue(), what, currentRule)); + } + map = ImmutableMap.copyOf(result); + hasDefaultCondition = foundDefaultCondition; + } + + /** + * Returns the selector's (configurability pattern --gt; matching values) map. + */ + public Map<Label, T> getEntries() { + return map; + } + + /** + * Returns the value to use when none of the attribute's selection keys match. + */ + public T getDefault() { + return map.get(defaultConditionLabel); + } + + /** + * Returns whether or not this selector has a default condition. + */ + public boolean hasDefault() { + return hasDefaultCondition; + } + + /** + * Returns the native Type for this attribute (i.e. what this would be if it wasn't a + * selector expression). + */ + public Type<T> getOriginalType() { + return originalType; + } + + /** + * Returns true for labels that are "reserved selector key words" and not intended to + * map to actual targets. + */ + public static boolean isReservedLabel(Label label) { + return label.toString().equals(DEFAULT_CONDITION_KEY); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/CompileOneDependencyTransformer.java b/src/main/java/com/google/devtools/build/lib/pkgcache/CompileOneDependencyTransformer.java new file mode 100644 index 0000000..ae14f18 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/CompileOneDependencyTransformer.java
@@ -0,0 +1,186 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.cmdline.ResolvedTargets; +import com.google.devtools.build.lib.cmdline.TargetParsingException; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.FileTarget; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.RawAttributeMapper; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +/** + * Implementation of --compile_one_dependency. + */ +final class CompileOneDependencyTransformer { + + private final PackageManager pkgManager; + + public CompileOneDependencyTransformer(PackageManager pkgManager) { + this.pkgManager = pkgManager; + } + + /** + * For each input file in the original result, returns a rule in the same package which has the + * input file as a source. + */ + public ResolvedTargets<Target> transformCompileOneDependency(EventHandler eventHandler, + ResolvedTargets<Target> original) throws TargetParsingException { + if (original.hasError()) { + return original; + } + ResolvedTargets.Builder<Target> builder = ResolvedTargets.builder(); + for (Target target : original.getTargets()) { + builder.add(transformCompileOneDependency(eventHandler, target)); + } + return builder.build(); + } + + /** + * Returns a list of rules in the given package sorted by BUILD file order. When + * multiple rules depend on a target, we choose the first match in this list (after + * filtering for preferred dependencies - see below). + * + * <p>Rules with configurable attributes are skipped, as this code doesn't know which + * configuration will be applied, so it can't reliably determine what their 'srcs' + * will look like. + */ + private Iterable<Rule> getOrderedRuleList(Package pkg) { + List<Rule> orderedList = Lists.newArrayList(); + for (Rule rule : pkg.getTargets(Rule.class)) { + if (!rule.hasConfigurableAttributes()) { + orderedList.add(rule); + } + } + + Collections.sort(orderedList, new Comparator<Rule>() { + @Override + public int compare(Rule o1, Rule o2) { + return Integer.compare( + o1.getLocation().getStartOffset(), + o2.getLocation().getStartOffset()); + } + }); + return orderedList; + } + + private Target transformCompileOneDependency(EventHandler eventHandler, Target target) + throws TargetParsingException { + if (!(target instanceof FileTarget)) { + throw new TargetParsingException("--compile_one_dependency target '" + + target.getLabel() + "' must be a file"); + } + + Package pkg; + try { + pkg = pkgManager.getLoadedPackage(target.getLabel().getPackageIdentifier()); + } catch (NoSuchPackageException e) { + throw new IllegalStateException(e); + } + + Iterable<Rule> orderedRuleList = getOrderedRuleList(pkg); + // Consuming rule to return if no "preferred" rules have been found. + Rule fallbackRule = null; + + for (Rule rule : orderedRuleList) { + try { + // The call to getSrcTargets here can be removed in favor of the + // rule.getLabels() call below once we update "srcs" for all rules. + if (SrcTargetUtil.getSrcTargets(eventHandler, rule, pkgManager).contains(target)) { + if (rule.getRuleClassObject().isPreferredDependency(target.getName())) { + return rule; + } else if (fallbackRule == null) { + fallbackRule = rule; + } + } + } catch (NoSuchThingException e) { + // Nothing to see here. Move along. + } catch (InterruptedException e) { + throw new TargetParsingException("interrupted"); + } + } + + Rule result = null; + + // For each rule, see if it has directCompileTimeInputAttribute, + // and if so check the targets listed in that attribute match the label. + for (Rule rule : orderedRuleList) { + if (rule.getLabels(Rule.DIRECT_COMPILE_TIME_INPUT).contains(target.getLabel())) { + if (rule.getRuleClassObject().isPreferredDependency(target.getName())) { + result = rule; + } else if (fallbackRule == null) { + fallbackRule = rule; + } + } + } + + if (result == null) { + result = fallbackRule; + } + + if (result == null) { + throw new TargetParsingException( + "Couldn't find dependency on target '" + target.getLabel() + "'"); + } + + try { + // If the rule has source targets, return it. + if (!SrcTargetUtil.getSrcTargets(eventHandler, result, pkgManager).isEmpty()) { + return result; + } + } catch (NoSuchThingException e) { + throw new TargetParsingException( + "Couldn't find dependency on target '" + target.getLabel() + "'"); + } catch (InterruptedException e) { + throw new TargetParsingException("interrupted"); + } + + for (Rule rule : orderedRuleList) { + RawAttributeMapper attributes = RawAttributeMapper.of(rule); + // We don't know what configuration we're using at this point, so we can't be sure + // which deps/srcs apply to this invocation if they're configurable for this rule. + // So exclude such rules for consideration. + if (attributes.isConfigurable("deps", Type.LABEL_LIST) + || attributes.isConfigurable("srcs", Type.LABEL_LIST)) { + continue; + } + RuleClass ruleClass = rule.getRuleClassObject(); + if (ruleClass.hasAttr("deps", Type.LABEL_LIST) && + ruleClass.hasAttr("srcs", Type.LABEL_LIST)) { + for (Label dep : attributes.get("deps", Type.LABEL_LIST)) { + if (dep.equals(result.getLabel())) { + if (!attributes.get("srcs", Type.LABEL_LIST).isEmpty()) { + return rule; + } + } + } + } + } + + throw new TargetParsingException( + "Couldn't find dependency on target '" + target.getLabel() + "'"); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/FilteringPolicies.java b/src/main/java/com/google/devtools/build/lib/pkgcache/FilteringPolicies.java new file mode 100644 index 0000000..df01326 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/FilteringPolicies.java
@@ -0,0 +1,126 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; + +import java.util.Objects; + +/** + * Utility class for predefined filtering policies. + */ +public final class FilteringPolicies { + + private FilteringPolicies() { + } + + /** + * Base class for singleton filtering policies. + */ + private abstract static class AbstractFilteringPolicy implements FilteringPolicy { + @Override + public int hashCode() { + return getClass().getSimpleName().hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj == this) { + return true; + } + return getClass().equals(obj.getClass()); + } + } + + private static class NoFilter extends AbstractFilteringPolicy { + @Override + public boolean shouldRetain(Target target, boolean explicit) { + return true; + } + } + + public static final FilteringPolicy NO_FILTER = new NoFilter(); + + private static class FilterManualAndObsolete extends AbstractFilteringPolicy { + @Override + public boolean shouldRetain(Target target, boolean explicit) { + return explicit || !(TargetUtils.hasManualTag(target) || TargetUtils.isObsolete(target)); + } + } + + public static final FilteringPolicy FILTER_MANUAL_AND_OBSOLETE = new FilterManualAndObsolete(); + + private static class FilterTests extends AbstractFilteringPolicy { + @Override + public boolean shouldRetain(Target target, boolean explicit) { + return TargetUtils.isTestOrTestSuiteRule(target) + && FILTER_MANUAL_AND_OBSOLETE.shouldRetain(target, explicit); + } + } + + public static final FilteringPolicy FILTER_TESTS = new FilterTests(); + + private static class RulesOnly extends AbstractFilteringPolicy { + @Override + public boolean shouldRetain(Target target, boolean explicit) { + return target instanceof Rule; + } + } + + public static final FilteringPolicy RULES_ONLY = new RulesOnly(); + + /** + * Returns the result of applying y, if target passes x. + */ + public static FilteringPolicy and(final FilteringPolicy x, + final FilteringPolicy y) { + return new AndFilteringPolicy(x, y); + } + + private static class AndFilteringPolicy implements FilteringPolicy { + private final FilteringPolicy firstPolicy; + private final FilteringPolicy secondPolicy; + + public AndFilteringPolicy(FilteringPolicy firstPolicy, FilteringPolicy secondPolicy) { + this.firstPolicy = Preconditions.checkNotNull(firstPolicy); + this.secondPolicy = Preconditions.checkNotNull(secondPolicy); + } + + @Override + public boolean shouldRetain(Target target, boolean explicit) { + return firstPolicy.shouldRetain(target, explicit) + && secondPolicy.shouldRetain(target, explicit); + } + + @Override + public int hashCode() { + return Objects.hash(firstPolicy, secondPolicy); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof AndFilteringPolicy)) { + return false; + } + AndFilteringPolicy other = (AndFilteringPolicy) obj; + return other.firstPolicy.equals(firstPolicy) && other.secondPolicy.equals(secondPolicy); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/FilteringPolicy.java b/src/main/java/com/google/devtools/build/lib/pkgcache/FilteringPolicy.java new file mode 100644 index 0000000..ac27fb0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/FilteringPolicy.java
@@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.packages.Target; + +import java.io.Serializable; + +/** + * A filtering policy defines how target patterns are matched. For instance, we may wish to select + * only tests, no tests, or remove obsolete targets. + */ +public interface FilteringPolicy extends Serializable { + + /** + * Returns true if this target should be retained. + * + * @param explicit true iff the label was specified explicitly, as opposed to being discovered by + * a wildcard. + */ + @ThreadSafety.ThreadSafe + boolean shouldRetain(Target target, boolean explicit); +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadedPackage.java b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadedPackage.java new file mode 100644 index 0000000..3c6f70f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadedPackage.java
@@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.devtools.build.lib.events.Event; + +/** + * A loaded package that can verify whether it is still up to date. + */ +interface LoadedPackage { + /** + * Returns the actual loaded {@link Package} object. + */ + Package getPackage(); + + /** + * Returns true iff the entry is still valid. + * + * <p>An entry is valid when the package it denotes has not been moved, deleted, or changed. This + * requires disk I/O to fetch metadata and re-evaluate globs. + */ + boolean isValid() throws InterruptedException; + + /** + * Returns true iff the the contents of the package are guaranteed not to have changed after + * between {@link #isValid()} calls and syncs of the associated package loader. + */ + boolean contentsCouldNotHaveChanged(); + + /** + * Returns the set of events (sorted by the order they were reported) that occurred during the + * loading of the package. + */ + Iterable<Event> getEvents(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadedPackageProvider.java b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadedPackageProvider.java new file mode 100644 index 0000000..1c51810 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadedPackageProvider.java
@@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; + +/** + * Read-only API for retrieving packages, i.e., calling this API should not result in packages being + * loaded. + * + * <p><b>Concurrency</b>: Implementations should be thread-safe. + */ +// TODO(bazel-team): Skyframe doesn't really implement this - can we remove it? +public interface LoadedPackageProvider { + + /** + * Returns a package if it was recently loaded, i.e., since the most recent cache sync. This + * throws an exception if the package was not loaded, even if it exists on disk. + */ + Package getLoadedPackage(PackageIdentifier packageIdentifier) throws NoSuchPackageException; + + /** + * Returns a target if it was recently loaded, i.e., since the most recent cache sync. This + * throws an exception if the target was not loaded or not validated, even if it exists in the + * surrounding package. + */ + Target getLoadedTarget(Label label) throws NoSuchPackageException, NoSuchTargetException; + + /** + * Returns true iff the specified target is current, i.e. a request for its label using {@link + * #getLoadedTarget} would return the same target instance. + */ + boolean isTargetCurrent(Target target); +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingFailedException.java b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingFailedException.java new file mode 100644 index 0000000..537746b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingFailedException.java
@@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +/** + * An exception indicating that there was a problem during the loading phase for one or more + * targets in such a way that the build cannot proceed (for example because keep_going is disabled). + */ +public class LoadingFailedException extends Exception { + + public LoadingFailedException(String message) { + super(message); + } + + public LoadingFailedException(String message, Throwable cause) { + super(message + ": " + cause.getMessage(), cause); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingFailureEvent.java b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingFailureEvent.java new file mode 100644 index 0000000..d7e0256 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingFailureEvent.java
@@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.devtools.build.lib.syntax.Label; + +/** + * This event is fired during the build, when it becomes known that the loading + * of a target cannot be completed because of an error in one of its + * dependencies. + */ +public class LoadingFailureEvent { + private final Label failedTarget; + private final Label failureReason; + + public LoadingFailureEvent(Label failedTarget, Label failureReason) { + this.failedTarget = failedTarget; + this.failureReason = failureReason; + } + + public Label getFailedTarget() { + return failedTarget; + } + + public Label getFailureReason() { + return failureReason; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseCompleteEvent.java new file mode 100644 index 0000000..19c22ec --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseCompleteEvent.java
@@ -0,0 +1,86 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Collection; + +/** + * This event is fired after the loading phase is complete. + */ +public class LoadingPhaseCompleteEvent { + private final Collection<Target> targets; + private final Collection<Target> filteredTargets; + private final PackageManager.PackageManagerStatistics pkgManagerStats; + private final long timeInMs; + + /** + * Construct the event. + * + * @param targets the set of active targets that remain + * @param pkgManagerStats statistics about the package cache + */ + public LoadingPhaseCompleteEvent(Collection<Target> targets, Collection<Target> filteredTargets, + PackageManager.PackageManagerStatistics pkgManagerStats, long timeInMs) { + this.targets = targets; + this.filteredTargets = filteredTargets; + this.pkgManagerStats = pkgManagerStats; + this.timeInMs = timeInMs; + } + + /** + * @return The set of active targets remaining, which is a subset of the + * targets we attempted to load. + */ + public Collection<Target> getTargets() { + return targets; + } + + /** + * @return The set of filtered targets. + */ + public Collection<Target> getFilteredTargets() { + return filteredTargets; + } + + /** + * @return The set of active target labels remaining, which is a subset of the + * targets we attempted to load. + */ + public Iterable<Label> getLabels() { + return Iterables.transform(targets, TO_LABEL); + } + + public long getTimeInMs() { + return timeInMs; + } + + /** + * Returns the PackageCache statistics. + */ + public PackageManager.PackageManagerStatistics getPkgManagerStats() { + return pkgManagerStats; + } + + private static final Function<Target, Label> TO_LABEL = new Function<Target, Label>() { + @Override + public Label apply(Target input) { + return input.getLabel(); + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseRunner.java b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseRunner.java new file mode 100644 index 0000000..4d8eea6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseRunner.java
@@ -0,0 +1,661 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.cmdline.ResolvedTargets; +import com.google.devtools.build.lib.cmdline.TargetParsingException; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.events.DelegatingEventHandler; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TestSize; +import com.google.devtools.build.lib.packages.TestTargetUtils; +import com.google.devtools.build.lib.packages.TestTimeout; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Converters.CommaSeparatedOptionListConverter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +/** + * Implements the loading phase; responsible for: + * <ul> + * <li>target pattern evaluation + * <li>test suite expansion + * <li>loading the labels needed to construct the build configuration + * <li>loading the labels needed for the analysis with the build configuration + * <li>loading the transitive closure of the targets and the configuration labels + * </ul> + * + * <p>In order to ensure correctness of incremental loading and of full cache hits, this class is + * very restrictive about access to its internal state and to its collaborators. In particular, none + * of the collaborators of this class may change in incompatible ways, such as changing the relative + * working directory for the target pattern parser, without notifying this class. + * + * <p>For full caching, this class tracks the exact values of all inputs to the loading phase. To + * maximize caching, it is vital that these change as rarely as possible. + */ +public class LoadingPhaseRunner { + + /** + * Loading phase options. + */ + public static class Options extends OptionsBase { + + @Option(name = "loading_phase_threads", + defaultValue = "200", + category = "undocumented", + help = "Number of parallel threads to use for the loading phase.") + public int loadingPhaseThreads; + + @Option(name = "build_tests_only", + defaultValue = "false", + category = "what", + help = "If specified, only *_test and test_suite rules will be built " + + "and other targets specified on the command line will be ignored. " + + "By default everything that was requested will be built.") + public boolean buildTestsOnly; + + @Option(name = "compile_one_dependency", + defaultValue = "false", + category = "what", + help = "Compile a single dependency of the argument files. " + + "This is useful for syntax checking source files in IDEs, " + + "for example, by rebuilding a single target that depends on " + + "the source file to detect errors as early as possible in the " + + "edit/build/test cycle. This argument affects the way all " + + "non-flag arguments are interpreted; instead of being targets " + + "to build they are source filenames. For each source filename " + + "an arbitrary target that depends on it will be built.") + public boolean compileOneDependency; + + @Option(name = "test_tag_filters", + converter = CommaSeparatedOptionListConverter.class, + defaultValue = "", + category = "what", + help = "Specifies a comma-separated list of test tags. Each tag can be optionally " + + "preceded with '-' to specify excluded tags. Only those test targets will be " + + "found that contain at least one included tag and do not contain any excluded " + + "tags. This option affects --build_tests_only behavior and the test command." + ) + public List<String> testTagFilterList; + + @Option(name = "test_size_filters", + converter = TestSize.TestSizeFilterConverter.class, + defaultValue = "", + category = "what", + help = "Specifies a comma-separated list of test sizes. Each size can be optionally " + + "preceded with '-' to specify excluded sizes. Only those test targets will be " + + "found that contain at least one included size and do not contain any excluded " + + "sizes. This option affects --build_tests_only behavior and the test command." + ) + public Set<TestSize> testSizeFilterSet; + + @Option(name = "test_timeout_filters", + converter = TestTimeout.TestTimeoutFilterConverter.class, + defaultValue = "", + category = "what", + help = "Specifies a comma-separated list of test timeouts. Each timeout can be " + + "optionally preceded with '-' to specify excluded timeouts. Only those test " + + "targets will be found that contain at least one included timeout and do not " + + "contain any excluded timeouts. This option affects --build_tests_only behavior " + + "and the test command." + ) + public Set<TestTimeout> testTimeoutFilterSet; + + @Option(name = "test_lang_filters", + converter = CommaSeparatedOptionListConverter.class, + defaultValue = "", + category = "what", + help = "Specifies a comma-separated list of test languages. Each language can be " + + "optionally preceded with '-' to specify excluded languages. Only those " + + "test targets will be found that are written in the specified languages. " + + "The name used for each language should be the same as the language prefix in the " + + "*_test rule, e.g. one of 'cc', 'java', 'py', etc." + + "This option affects --build_tests_only behavior and the test command." + ) + public List<String> testLangFilterList; + } + + /** + * A callback interface to notify the caller about specific events. + * TODO(bazel-team): maybe we should use the EventBus instead? + */ + public interface Callback { + /** + * Called after the target patterns have been resolved to give the caller a chance to validate + * the list before proceeding. + */ + void notifyTargets(Collection<Target> targets) throws LoadingFailedException; + + /** + * Called after loading has finished, to notify the caller about the visited packages. + * + * <p>The set of visited packages is the set of packages in the transitive closure of the + * union of the top level targets. + */ + void notifyVisitedPackages(Set<PackageIdentifier> visitedPackages); + } + + /** + * The result of the loading phase, i.e., whether there were errors, and which targets were + * successfully loaded, plus some related metadata. + */ + public static final class LoadingResult { + private final boolean hasTargetPatternError; + private final boolean hasLoadingError; + private final ImmutableSet<Target> targetsToAnalyze; + private final ImmutableSet<Target> testsToRun; + private final ImmutableMap<PackageIdentifier, Path> packageRoots; + // TODO(bazel-team): consider moving this to LoadedPackageProvider + private final ImmutableSet<PackageIdentifier> visitedPackages; + + public LoadingResult(boolean hasTargetPatternError, boolean hasLoadingError, + Collection<Target> targetsToAnalyze, Collection<Target> testsToRun, + ImmutableMap<PackageIdentifier, Path> packageRoots, + Set<PackageIdentifier> visitedPackages) { + this.hasTargetPatternError = hasTargetPatternError; + this.hasLoadingError = hasLoadingError; + this.targetsToAnalyze = + targetsToAnalyze == null ? null : ImmutableSet.copyOf(targetsToAnalyze); + this.testsToRun = testsToRun == null ? null : ImmutableSet.copyOf(testsToRun); + this.packageRoots = packageRoots; + this.visitedPackages = ImmutableSet.copyOf(visitedPackages); + } + + /** Whether there were errors during target pattern evaluation. */ + public boolean hasTargetPatternError() { + return hasTargetPatternError; + } + + /** Whether there were errors during the loading phase. */ + public boolean hasLoadingError() { + return hasLoadingError; + } + + /** Successfully loaded targets that should be built. */ + public Collection<Target> getTargets() { + return targetsToAnalyze; + } + + /** Successfully loaded targets that should be run as tests. Must be a subset of the targets. */ + public Collection<Target> getTestsToRun() { + return testsToRun; + } + + /** + * The map from package names to the package root where each package was found; this is used to + * set up the symlink tree. + */ + public ImmutableMap<PackageIdentifier, Path> getPackageRoots() { + return packageRoots; + } + + /** + * Returns all packages that were visited during this loading phase. + * + * <p>We use this to decide when to evict ConfiguredTarget nodes from the graph. + */ + @ThreadCompatible + private ImmutableSet<PackageIdentifier> getVisitedPackages() { + return visitedPackages; + } + } + + private static final class ParseFailureListenerImpl extends DelegatingEventHandler + implements ParseFailureListener { + private final EventBus eventBus; + + private ParseFailureListenerImpl(EventHandler delegate, EventBus eventBus) { + super(delegate); + this.eventBus = eventBus; + } + + @Override + public void parsingError(String targetPattern, String message) { + if (eventBus != null) { + eventBus.post(new ParsingFailedEvent(targetPattern, message)); + } + } + } + + private static final Logger LOG = Logger.getLogger(LoadingPhaseRunner.class.getName()); + + private final PackageManager packageManager; + private final TargetPatternEvaluator targetPatternEvaluator; + private final Set<String> ruleNames; + private final TransitivePackageLoader pkgLoader; + + public LoadingPhaseRunner(PackageManager packageManager, + Set<String> ruleNames) { + this.packageManager = packageManager; + this.targetPatternEvaluator = packageManager.getTargetPatternEvaluator(); + this.ruleNames = ruleNames; + this.pkgLoader = packageManager.newTransitiveLoader(); + } + + public TargetPatternEvaluator getTargetPatternEvaluator() { + return targetPatternEvaluator; + } + + public void updatePatternEvaluator(PathFragment relativeWorkingDirectory) { + targetPatternEvaluator.updateOffset(relativeWorkingDirectory); + } + + /** + * This method only exists for the benefit of InfoCommand, which needs to construct + * a {@code BuildConfigurationCollection} without running a full loading phase. Don't + * add any more clients; instead, we should change info so that it doesn't need the configuration. + */ + public LoadedPackageProvider loadForConfigurations(EventHandler eventHandler, + Set<Label> labelsToLoad, boolean keepGoing) throws InterruptedException { + // Use a new Label Visitor here to avoid erasing the cache on the existing one. + TransitivePackageLoader transitivePackageLoader = packageManager.newTransitiveLoader(); + boolean loadingSuccessful = transitivePackageLoader.sync( + eventHandler, ImmutableSet.<Target>of(), + labelsToLoad, keepGoing, /*parallelThreads=*/10, + /*maxDepth=*/Integer.MAX_VALUE); + return loadingSuccessful ? packageManager : null; + } + + /** + * Performs target pattern evaluation, test suite expansion (if requested), and loads the + * transitive closure of the resulting targets as well as of the targets needed to use the + * given build configuration provider. + */ + public LoadingResult execute(EventHandler eventHandler, EventBus eventBus, + List<String> targetPatterns, Options options, + ListMultimap<String, Label> labelsToLoadUnconditionally, boolean keepGoing, + boolean determineTests, @Nullable Callback callback) + throws TargetParsingException, LoadingFailedException, InterruptedException { + LOG.info("Starting pattern evaluation"); + Stopwatch timer = Stopwatch.createStarted(); + if (options.buildTestsOnly && options.compileOneDependency) { + throw new LoadingFailedException("--compile_one_dependency cannot be used together with " + + "the --build_tests_only option or the 'bazel test' command "); + } + + EventHandler parseFailureListener = new ParseFailureListenerImpl(eventHandler, eventBus); + // Determine targets to build: + ResolvedTargets<Target> targets = getTargetsToBuild(parseFailureListener, + targetPatterns, options.compileOneDependency, keepGoing); + + ImmutableSet<Target> filteredTargets = targets.getFilteredTargets(); + + boolean buildTestsOnly = options.buildTestsOnly; + ImmutableSet<Target> testsToRun = null; + ImmutableSet<Target> testFilteredTargets = ImmutableSet.of(); + + // Now we have a list of targets to build. If the --build_tests_only option was specified or we + // want to run tests, we need to determine the list of targets to test. For that, we remove + // manual tests and apply the command line filters. Also, if --build_tests_only is specified, + // then the list of filtered targets will be set as build list as well. + if (determineTests || buildTestsOnly) { + // Parse the targets to get the tests. + ResolvedTargets<Target> testTargets = determineTests(parseFailureListener, + targetPatterns, options, keepGoing); + if (testTargets.getTargets().isEmpty() && !testTargets.getFilteredTargets().isEmpty()) { + eventHandler.handle(Event.warn("All specified test targets were excluded by filters")); + } + + if (buildTestsOnly) { + // Replace original targets to build with test targets, so that only targets that are + // actually going to be built are loaded in the loading phase. Note that this has a side + // effect that any test_suite target requested to be built is replaced by the set of *_test + // targets it represents; for example, this affects the status and the summary reports. + Set<Target> allFilteredTargets = new HashSet<>(); + allFilteredTargets.addAll(targets.getTargets()); + allFilteredTargets.addAll(targets.getFilteredTargets()); + allFilteredTargets.removeAll(testTargets.getTargets()); + allFilteredTargets.addAll(testTargets.getFilteredTargets()); + testFilteredTargets = ImmutableSet.copyOf(allFilteredTargets); + filteredTargets = ImmutableSet.of(); + + targets = ResolvedTargets.<Target>builder() + .merge(testTargets) + .mergeError(targets.hasError()) + .build(); + if (determineTests) { + testsToRun = testTargets.getTargets(); + } + } else /*if (determineTests)*/ { + testsToRun = testTargets.getTargets(); + targets = ResolvedTargets.<Target>builder() + .merge(targets) + // Avoid merge() here which would remove the filteredTargets from the targets. + .addAll(testsToRun) + .mergeError(testTargets.hasError()) + .build(); + // filteredTargets is correct in this case - it cannot contain tests that got back in + // through test_suite expansion, because the test determination would also filter those out. + // However, that's not obvious, and it might be better to explicitly recompute it. + } + if (testsToRun != null) { + // Note that testsToRun can still be null here, if buildTestsOnly && !shouldRunTests. + Preconditions.checkState(targets.getTargets().containsAll(testsToRun)); + } + } + + eventBus.post(new TargetParsingCompleteEvent(targets.getTargets(), + filteredTargets, testFilteredTargets, + timer.stop().elapsed(TimeUnit.MILLISECONDS))); + + if (targets.hasError()) { + eventHandler.handle(Event.warn("Target pattern parsing failed. Continuing anyway")); + } + + if (callback != null) { + callback.notifyTargets(targets.getTargets()); + } + + maybeReportDeprecation(eventHandler, targets.getTargets()); + + // Load the transitive closure of all targets. + LoadingResult result = doLoadingPhase(eventHandler, eventBus, targets.getTargets(), + testsToRun, labelsToLoadUnconditionally, keepGoing, options.loadingPhaseThreads, + targets.hasError()); + + if (callback != null) { + callback.notifyVisitedPackages(result.getVisitedPackages()); + } + + return result; + } + + /** + * Visit the transitive closure of the targets, populating the package cache + * and ensuring that all labels can be resolved and all rules were free from + * errors. + * + * @param targetsToLoad the list of command-line target patterns specified by the user + * @param testsToRun the tests to run as a subset of the targets to load + * @param labelsToLoadUnconditionally the labels to load unconditionally (presumably for the build + * configuration) + * @param keepGoing if true, don't throw ViewCreationFailedException if some + * targets could not be loaded, just skip thm. + */ + private LoadingResult doLoadingPhase(EventHandler eventHandler, EventBus eventBus, + ImmutableSet<Target> targetsToLoad, Collection<Target> testsToRun, + ListMultimap<String, Label> labelsToLoadUnconditionally, boolean keepGoing, + int loadingPhaseThreads, boolean hasError) + throws InterruptedException, LoadingFailedException { + eventHandler.handle(Event.progress("Loading...")); + Stopwatch timer = Stopwatch.createStarted(); + LOG.info("Starting loading phase"); + + Set<Label> labelsToLoad = ImmutableSet.copyOf(labelsToLoadUnconditionally.values()); + + // For each label in {@code targetsToLoad}, ensure that the target to which + // it refers exists, and also every target in its transitive closure of label + // dependencies. Success guarantees that a call to + // {@code getConfiguredTarget} for the same targets will not fail; the + // configuration process is intolerant of missing packages/targets. Before + // calling getConfiguredTarget(), clients must ensure that all necessary + // packages/targets have been visited since the last sync/clear. + boolean loadingSuccessful = pkgLoader.sync(eventHandler, targetsToLoad, labelsToLoad, + keepGoing, loadingPhaseThreads, Integer.MAX_VALUE); + + ImmutableSet<Target> targetsToAnalyze; + if (loadingSuccessful) { + // Success: all loaded targets will be analyzed. + targetsToAnalyze = targetsToLoad; + } else if (keepGoing) { + // Keep going: filter out the error-free targets and only continue with those. + targetsToAnalyze = filterErrorFreeTargets(eventBus, targetsToLoad, + pkgLoader, labelsToLoadUnconditionally); + + // Tell the user about the subset of successful targets. + int requested = targetsToLoad.size(); + int loaded = targetsToAnalyze.size(); + if (0 < loaded && loaded < requested) { + String message = String.format("Loading succeeded for only %d of %d targets", loaded, + requested); + eventHandler.handle(Event.info(message)); + LOG.info(message); + } + } else { + throw new LoadingFailedException("Loading failed; build aborted"); + } + + Set<Target> filteredTargets = targetsToAnalyze; + try { + // We use strict test_suite expansion here to match the analysis-time checks. + ResolvedTargets<Target> expandedResult = TestTargetUtils.expandTestSuites( + packageManager, eventHandler, targetsToAnalyze, /*strict=*/true, /*keepGoing=*/true); + targetsToAnalyze = expandedResult.getTargets(); + filteredTargets = Sets.difference(filteredTargets, targetsToAnalyze); + if (expandedResult.hasError()) { + if (!keepGoing) { + throw new LoadingFailedException("Could not expand test suite target"); + } + loadingSuccessful = false; + } + } catch (TargetParsingException e) { + // This shouldn't happen, because we've already loaded the targets successfully. + throw (AssertionError) (new AssertionError("Unexpected target failure").initCause(e)); + } + + // Perform some operations on the set of packages containing the collected targets. + ImmutableMap<PackageIdentifier, Path> packageRoots = collectPackageRoots( + pkgLoader.getErrorFreeVisitedPackages()); + + Set<PackageIdentifier> visitedPackageNames = pkgLoader.getVisitedPackageNames(); + + // Clear some targets from the cache to free memory. + packageManager.partiallyClear(); + + eventBus.post(new LoadingPhaseCompleteEvent( + targetsToAnalyze, filteredTargets, packageManager.getStatistics(), + timer.stop().elapsed(TimeUnit.MILLISECONDS))); + LOG.info("Loading phase finished"); + + // testsToRun can contain targets that aren't analyzed, but the BuildView ignores those. + return new LoadingResult(hasError, !loadingSuccessful, targetsToAnalyze, testsToRun, + packageRoots, visitedPackageNames); + } + + private Collection<Target> getTargetsForLabels(Collection<Label> labels) { + Set<Target> result = new HashSet<>(); + + for (Label label : labels) { + try { + result.add(packageManager.getLoadedTarget(label)); + } catch (NoSuchPackageException e) { + Package pkg = Preconditions.checkNotNull(e.getPackage()); + try { + result.add(pkg.getTarget(label.getName())); + } catch (NoSuchTargetException ex) { + throw new IllegalStateException(ex); + } + } catch (NoSuchThingException e) { + throw new IllegalStateException(e); // The target should have been loaded + } + } + + return result; + } + + private ImmutableSet<Target> filterErrorFreeTargets( + EventBus eventBus, Collection<Target> targetsToLoad, + TransitivePackageLoader pkgLoader, + ListMultimap<String, Label> labelsToLoadUnconditionally) throws LoadingFailedException { + // Error out if any of the labels needed for the configuration could not be loaded. + Collection<Label> labelsToLoad = new ArrayList<>(labelsToLoadUnconditionally.values()); + for (Target target : targetsToLoad) { + labelsToLoad.add(target.getLabel()); + } + Multimap<Label, Label> rootCauses = pkgLoader.getRootCauses(labelsToLoad); + for (Map.Entry<String, Label> entry : labelsToLoadUnconditionally.entries()) { + if (rootCauses.containsKey(entry.getValue())) { + throw new LoadingFailedException("Failed to load required " + entry.getKey() + + " target: '" + entry.getValue() + "'"); + } + } + + // Post root causes for command-line targets that could not be loaded. + for (Map.Entry<Label, Label> entry : rootCauses.entries()) { + eventBus.post(new LoadingFailureEvent(entry.getKey(), entry.getValue())); + } + + return ImmutableSet.copyOf(Sets.difference(ImmutableSet.copyOf(targetsToLoad), + ImmutableSet.copyOf(getTargetsForLabels(rootCauses.keySet())))); + } + + /** + * Returns a map of collected package names to root paths. + */ + private static ImmutableMap<PackageIdentifier, Path> collectPackageRoots( + Collection<Package> packages) { + // Make a map of the package names to their root paths. + ImmutableMap.Builder<PackageIdentifier, Path> packageRoots = ImmutableMap.builder(); + for (Package pkg : packages) { + packageRoots.put(pkg.getPackageIdentifier(), pkg.getSourceRoot()); + } + return packageRoots.build(); + } + + /** + * Interpret the command-line arguments. + * + * @param targetPatterns the list of command-line target patterns specified by the user + * @param compileOneDependency if true, enables alternative interpretation of targetPatterns; see + * {@link Options#compileOneDependency} + * @throws TargetParsingException if parsing failed and !keepGoing + */ + private ResolvedTargets<Target> getTargetsToBuild(EventHandler eventHandler, + List<String> targetPatterns, boolean compileOneDependency, + boolean keepGoing) throws TargetParsingException, InterruptedException { + ResolvedTargets<Target> result = + targetPatternEvaluator.parseTargetPatternList(eventHandler, targetPatterns, + FilteringPolicies.FILTER_MANUAL_AND_OBSOLETE, keepGoing); + if (compileOneDependency) { + return new CompileOneDependencyTransformer(packageManager) + .transformCompileOneDependency(eventHandler, result); + } + return result; + } + + /** + * Interpret test target labels from the command-line arguments and return the corresponding set + * of targets, handling the filter flags, and expanding test suites. + * + * @param eventHandler the error event eventHandler + * @param targetPatterns the list of command-line target patterns specified by the user + * @param options the loading phase options + * @param keepGoing value of the --keep_going flag + */ + private ResolvedTargets<Target> determineTests(EventHandler eventHandler, + List<String> targetPatterns, Options options, boolean keepGoing) + throws TargetParsingException, InterruptedException { + // Parse the targets to get the tests. + ResolvedTargets.Builder<Target> testTargetsBuilder = ResolvedTargets.builder(); + for (String targetPattern : targetPatterns) { + if (targetPattern.startsWith("-")) { + ResolvedTargets<Target> someNegativeTargets = targetPatternEvaluator.parseTargetPatternList( + eventHandler, ImmutableList.of(targetPattern.substring(1)), + FilteringPolicies.FILTER_TESTS, keepGoing); + ResolvedTargets<Target> moreNegativeTargets = TestTargetUtils.expandTestSuites( + packageManager, eventHandler, someNegativeTargets.getTargets(), /*strict=*/false, + keepGoing); + testTargetsBuilder.filter(Predicates.not(Predicates.in(moreNegativeTargets.getTargets()))); + testTargetsBuilder.mergeError(moreNegativeTargets.hasError()); + } else { + ResolvedTargets<Target> somePositiveTargets = targetPatternEvaluator.parseTargetPatternList( + eventHandler, ImmutableList.of(targetPattern), + FilteringPolicies.FILTER_TESTS, keepGoing); + ResolvedTargets<Target> morePositiveTargets = TestTargetUtils.expandTestSuites( + packageManager, eventHandler, somePositiveTargets.getTargets(), /*strict=*/false, + keepGoing); + testTargetsBuilder.addAll(morePositiveTargets.getTargets()); + testTargetsBuilder.mergeError(morePositiveTargets.hasError()); + } + } + testTargetsBuilder.filter(getTestFilter(eventHandler, options)); + return testTargetsBuilder.build(); + } + + /** + * Convert the options into a test filter. + */ + private Predicate<Target> getTestFilter(EventHandler eventHandler, Options options) { + Predicate<Target> testFilter = Predicates.alwaysTrue(); + if (!options.testSizeFilterSet.isEmpty()) { + testFilter = Predicates.and(testFilter, + TestTargetUtils.testSizeFilter(options.testSizeFilterSet)); + } + if (!options.testTimeoutFilterSet.isEmpty()) { + testFilter = Predicates.and(testFilter, + TestTargetUtils.testTimeoutFilter(options.testTimeoutFilterSet)); + } + if (!options.testTagFilterList.isEmpty()) { + testFilter = Predicates.and(testFilter, + TestTargetUtils.tagFilter(options.testTagFilterList)); + } + if (!options.testLangFilterList.isEmpty()) { + testFilter = Predicates.and(testFilter, + TestTargetUtils.testLangFilter(options.testLangFilterList, eventHandler, ruleNames)); + } + return testFilter; + } + + /** + * Emit a warning when a deprecated target is mentioned on the command line. + * + * <p>Note that this does not stop us from emitting "target X depends on deprecated target Y" + * style warnings for the same target and it is a good thing; <i>depending</i> on a target and + * <i>wanting</i> to build it are different things. + */ + private void maybeReportDeprecation(EventHandler eventHandler, Collection<Target> targets) { + for (Rule rule : Iterables.filter(targets, Rule.class)) { + if (rule.isAttributeValueExplicitlySpecified("deprecation")) { + eventHandler.handle(Event.warn(rule.getLocation(), String.format( + "target '%s' is deprecated: %s", rule.getLabel(), + NonconfigurableAttributeMapper.of(rule).get("deprecation", Type.STRING)))); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/PackageCacheBackedTargetPatternResolver.java b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageCacheBackedTargetPatternResolver.java new file mode 100644 index 0000000..e4dc010 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageCacheBackedTargetPatternResolver.java
@@ -0,0 +1,263 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.pkgcache; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.cmdline.LabelValidator; +import com.google.devtools.build.lib.cmdline.ResolvedTargets; +import com.google.devtools.build.lib.cmdline.TargetParsingException; +import com.google.devtools.build.lib.cmdline.TargetPatternResolver; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.concurrent.ThreadPoolExecutor; + +/** + * An implementation of the {@link TargetPatternResolver} that uses the {@link + * RecursivePackageProvider} as the backing implementation. + */ +final class PackageCacheBackedTargetPatternResolver implements TargetPatternResolver<Target> { + + private final RecursivePackageProvider packageProvider; + private final EventHandler eventHandler; + private final boolean keepGoing; + private final FilteringPolicy policy; + private final ThreadPoolExecutor packageVisitorPool; + + PackageCacheBackedTargetPatternResolver(RecursivePackageProvider packageProvider, + EventHandler eventHandler, boolean keepGoing, FilteringPolicy policy, + ThreadPoolExecutor packageVisitorPool) { + this.packageProvider = packageProvider; + this.eventHandler = eventHandler; + this.keepGoing = keepGoing; + this.policy = policy; + this.packageVisitorPool = packageVisitorPool; + } + + @Override + public void warn(String msg) { + eventHandler.handle(Event.warn(msg)); + } + + @Override + public Target getTargetOrNull(String targetName) throws InterruptedException { + try { + return packageProvider.getTarget(eventHandler, Label.parseAbsolute(targetName)); + } catch (NoSuchPackageException | NoSuchTargetException | Label.SyntaxException e) { + return null; + } + } + + @Override + public ResolvedTargets<Target> getExplicitTarget(String targetName) + throws TargetParsingException, InterruptedException { + Label label = TargetPatternResolverUtil.label(targetName); + return getExplicitTarget(label, targetName); + } + + private ResolvedTargets<Target> getExplicitTarget(Label label, String originalLabel) + throws TargetParsingException, InterruptedException { + try { + Target target = packageProvider.getTarget(eventHandler, label); + if (policy.shouldRetain(target, true)) { + return ResolvedTargets.of(target); + } + return ResolvedTargets.<Target>empty(); + } catch (BuildFileContainsErrorsException e) { + // We don't need to report an error here because errors + // would have already been reported in this case. + return handleParsingError(eventHandler, originalLabel, + new TargetParsingException(e.getMessage(), e), keepGoing); + } catch (NoSuchThingException e) { + return handleParsingError(eventHandler, originalLabel, + new TargetParsingException(e.getMessage(), e), keepGoing); + } + } + + /** + * Handles an error differently based on the value of keepGoing. + * + * @param badPattern The pattern we were unable to parse. + * @param e The underlying exception. + * @param keepGoing It true, report a warning and return. + * If false, throw the exception. + * @return the empty set. + * @throws TargetParsingException if !keepGoing. + */ + private ResolvedTargets<Target> handleParsingError(EventHandler eventHandler, String badPattern, + TargetParsingException e, boolean keepGoing) throws TargetParsingException { + if (eventHandler instanceof ParseFailureListener) { + ((ParseFailureListener) eventHandler).parsingError(badPattern, e.getMessage()); + } + if (keepGoing) { + eventHandler.handle(Event.error("Skipping '" + badPattern + "': " + e.getMessage())); + return ResolvedTargets.<Target>failed(); + } else { + throw e; + } + } + + @Override + public ResolvedTargets<Target> getTargetsInPackage(String originalPattern, String packageName, + boolean rulesOnly) throws TargetParsingException, InterruptedException { + FilteringPolicy actualPolicy = rulesOnly + ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy) + : policy; + return getTargetsInPackage(originalPattern, packageName, actualPolicy); + } + + private ResolvedTargets<Target> getTargetsInPackage(String originalPattern, String packageName, + FilteringPolicy policy) throws TargetParsingException, InterruptedException { + // Normalise, e.g "foo//bar" -> "foo/bar"; "foo/" -> "foo": + packageName = new PathFragment(packageName).toString(); + + // it's possible for this check to pass, but for Label.validatePackageNameFull to report an + // error because the package name is illegal. That's a little weird, but we can live with + // that for now--see test case: testBadPackageNameButGoodEnoughForALabel. (BTW I tried + // duplicating that validation logic in Label but it was extremely tricky.) + if (LabelValidator.validatePackageName(packageName) != null) { + return handleParsingError(eventHandler, originalPattern, + new TargetParsingException( + "'" + packageName + "' is not a valid package name"), keepGoing); + } + Package pkg; + try { + pkg = packageProvider.getPackage( + eventHandler, PackageIdentifier.createInDefaultRepo(packageName)); + } catch (NoSuchPackageException e) { + return handleParsingError(eventHandler, originalPattern, new TargetParsingException( + TargetPatternResolverUtil.getParsingErrorMessage( + e.getMessage(), originalPattern)), keepGoing); + } + + if (pkg.containsErrors()) { + // Report an error, but continue (and return partial results) if keepGoing is specified. + handleParsingError(eventHandler, originalPattern, new TargetParsingException( + TargetPatternResolverUtil.getParsingErrorMessage( + "package contains errors", originalPattern)), keepGoing); + } + + return TargetPatternResolverUtil.resolvePackageTargets(pkg, policy); + } + + @Override + public ResolvedTargets<Target> findTargetsBeneathDirectory(String originalPattern, + String pathPrefix, boolean rulesOnly) throws TargetParsingException, InterruptedException { + FilteringPolicy actualPolicy = rulesOnly + ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy) + : policy; + return findTargetsBeneathDirectory(eventHandler, originalPattern, pathPrefix, actualPolicy, + keepGoing, pathPrefix.isEmpty()); + } + + private ResolvedTargets<Target> findTargetsBeneathDirectory(final EventHandler eventHandler, + final String originalPattern, String pathPrefix, final FilteringPolicy policy, + final boolean keepGoing, boolean useTopLevelExcludes) + throws TargetParsingException, InterruptedException { + PathFragment directory = new PathFragment(pathPrefix); + if (directory.containsUplevelReferences()) { + throw new TargetParsingException("up-level references are not permitted: '" + + pathPrefix + "'"); + } + if (!pathPrefix.isEmpty() && (LabelValidator.validatePackageName(pathPrefix) != null)) { + return handleParsingError(eventHandler, pathPrefix, new TargetParsingException( + "'" + pathPrefix + "' is not a valid package name"), keepGoing); + } + + final ResolvedTargets.Builder<Target> builder = ResolvedTargets.concurrentBuilder(); + try { + packageProvider.visitPackageNamesRecursively(eventHandler, directory, + useTopLevelExcludes, packageVisitorPool, + new PathPackageLocator.AcceptsPathFragment() { + @Override + public void accept(PathFragment packageName) { + String pkgName = packageName.getPathString(); + try { + // Get the targets without transforming. We'll do that later below. + builder.merge(getTargetsInPackage(originalPattern, pkgName, + FilteringPolicies.NO_FILTER)); + } catch (InterruptedException e) { + throw new RuntimeParsingException(new TargetParsingException("interrupted")); + } catch (TargetParsingException e) { + // We'd like to make visitPackageNamesRecursively() generic + // over some checked exception type (TargetParsingException in + // this case). To do so, we'd have to make AbstractQueueVisitor + // generic over the same exception type. That won't work due to + // type erasure. As a workaround, we wrap the exception here, + // and unwrap it below. + throw new RuntimeParsingException(e); + } + } + }); + } catch (RuntimeParsingException e) { + throw e.unwrap(); + } catch (UnsupportedOperationException e) { + throw new TargetParsingException("recursive target patterns are not permitted: '" + + originalPattern + "'"); + } + + if (builder.isEmpty()) { + return handleParsingError(eventHandler, originalPattern, + new TargetParsingException("no targets found beneath '" + directory + "'"), + keepGoing); + } + + // Apply the transform after the check so we only return the + // error if the tree really contains no targets. + ResolvedTargets<Target> intermediateResult = builder.build(); + ResolvedTargets.Builder<Target> filteredBuilder = ResolvedTargets.builder(); + if (intermediateResult.hasError()) { + filteredBuilder.setError(); + } + for (Target target : intermediateResult.getTargets()) { + if (policy.shouldRetain(target, false)) { + filteredBuilder.add(target); + } + } + return filteredBuilder.build(); + } + + @Override + public boolean isPackage(String packageName) { + return packageProvider.isPackage(packageName); + } + + @Override + public String getTargetKind(Target target) { + return target.getTargetKind(); + } + + private static final class RuntimeParsingException extends RuntimeException { + private TargetParsingException parsingException; + + public RuntimeParsingException(TargetParsingException cause) { + super(cause); + this.parsingException = Preconditions.checkNotNull(cause); + } + + public TargetParsingException unwrap() { + return parsingException; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/PackageCacheOptions.java b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageCacheOptions.java new file mode 100644 index 0000000..f9d3efc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageCacheOptions.java
@@ -0,0 +1,142 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.pkgcache; + +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.packages.ConstantRuleVisibility; +import com.google.devtools.build.lib.packages.RuleVisibility; +import com.google.devtools.build.lib.syntax.CommaSeparatedPackageNameListConverter; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.List; + +/** + * Options for configuring the PackageCache. + */ +public class PackageCacheOptions extends OptionsBase { + /** + * A converter for package path that defaults to {@code Constants.DEFAULT_PACKAGE_PATH} if the + * option is not given. + * + * <p>Required because you cannot specify a non-constant value in annotation attributes. + */ + public static class PackagePathConverter implements Converter<List<String>> { + @Override + public List<String> convert(String input) throws OptionsParsingException { + return input.isEmpty() + ? Constants.DEFAULT_PACKAGE_PATH + : new Converters.ColonSeparatedOptionListConverter().convert(input); + } + + @Override + public String getTypeDescription() { + return "a string"; + } + } + + /** + * Converter for the {@code --default_visibility} option. + */ + public static class DefaultVisibilityConverter implements Converter<RuleVisibility> { + @Override + public RuleVisibility convert(String input) throws OptionsParsingException { + if (input.equals("public")) { + return ConstantRuleVisibility.PUBLIC; + } else if (input.equals("private")) { + return ConstantRuleVisibility.PRIVATE; + } else { + throw new OptionsParsingException("Not a valid default visibility: '" + input + + "' (should be 'public' or 'private'"); + } + } + + @Override + public String getTypeDescription() { + return "default visibility"; + } + } + + @Option(name = "package_path", + defaultValue = "", + category = "package loading", + converter = PackagePathConverter.class, + help = "A colon-separated list of where to look for packages. " + + "Elements beginning with '%workspace%' are relative to the enclosing " + + "workspace. If omitted or empty, the default is the output of " + + "'blaze info default-package-path'.") + public List<String> packagePath; + + @Option(name = "show_package_location", + defaultValue = "false", + category = "verbosity", + deprecationWarning = "This flag is no longer supported and will go away soon.", + help = "If enabled, causes Blaze to print the location on the --package_path " + + "from which each package was loaded.") + public boolean showPackageLocation; + + @Option(name = "show_loading_progress", + defaultValue = "true", + category = "verbosity", + help = "If enabled, causes Blaze to print \"Loading package:\" messages.") + public boolean showLoadingProgress; + + @Option(name = "deleted_packages", + defaultValue = "", + category = "package loading", + converter = CommaSeparatedPackageNameListConverter.class, + help = "A comma-separated list of names of packages which the " + + "build system will consider non-existent, even if they are " + + "visible somewhere on the package path." + + "\n" + + "Use this option when deleting a subpackage 'x/y' of an " + + "existing package 'x'. For example, after deleting x/y/BUILD " + + "in your client, the build system may complain if it " + + "encounters a label '//x:y/z' if that is still provided by another " + + "package_path entry. Specifying --deleted_packages x/y avoids this " + + "problem.") + public List<String> deletedPackages; + + @Option(name = "default_visibility", + defaultValue = "private", + category = "undocumented", + converter = DefaultVisibilityConverter.class, + help = "Default visibility for packages that don't set it explicitly ('public' or " + + "'private').") + public RuleVisibility defaultVisibility; + + @Option(name = "min_pkg_count_for_ct_node_eviction", + defaultValue = "3700", + // Why is the default value 3700? As of December 2013, a medium target loads about this many + // packages, uses ~310MB RAM to only load [1] or ~990MB to load and analyze [2,3]. So we + // can likely load and analyze this many packages without worrying about Blaze OOM'ing. + // + // If the total number of unique packages so far [4] is higher than the value of this flag, + // then we evict CT nodes [5] from the Skyframe graph. + // + // [1] blaze -x build --nobuild --noanalyze //medium:target + // [2] blaze -x build --nobuild //medium:target + // [3] according to "blaze info used-heap-size" + // [4] this means the number of unique packages loaded by builds, including the current one, + // since the last CT node eviction [5] + // [5] "CT node eviction" means clearing those nodes from the Skyframe graph that correspond + // to ConfiguredTargets; this is done using SkyframeExecutor.resetConfiguredTargets + category = "undocumented", + help = "Threshold for number of loaded packages before skyframe-m1 cache eviction kicks in") + public int minLoadedPkgCountForCtNodeEviction; +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/PackageManager.java b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageManager.java new file mode 100644 index 0000000..4d5e687 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageManager.java
@@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.packages.CachingPackageLocator; + +import java.io.PrintStream; + +/** + * A PackageManager keeps state about loaded packages around for quick lookup, and provides + * related functionality: Recursive package finding, loaded package checking, etc. + */ +public interface PackageManager extends PackageProvider, CachingPackageLocator, + LoadedPackageProvider { + + /** + * Returns the package cache statistics. + */ + PackageManagerStatistics getStatistics(); + + /** + * Removes cached data which is not needed anymore after loading is complete, to reduce memory + * consumption between builds. Whether or not this method is called does not affect correctness. + */ + void partiallyClear(); + + /** + * Dump the contents of the package manager in human-readable form. + * Used by 'bazel dump' and the BuildTool's unexpected exception handler. + */ + void dump(PrintStream printStream); + + /** + * Returns the package locator used by this package manager. + * + * <p>If you are tempted to call {@code getPackagePath().getPathEntries().get(0)}, be warned that + * this is probably not the value you are looking for! Look at the methods of {@code + * BazelRuntime} instead. + */ + @ThreadSafety.ThreadSafe + PathPackageLocator getPackagePath(); + + /** + * Collects statistics of the package manager since the last sync. + */ + interface PackageManagerStatistics { + + /** + * Returns the number of packages loaded since the last sync. I.e. the cache + * misses. + */ + int getPackagesLoaded(); + + /** + * Returns the number of packages looked up since the last sync. + */ + int getPackagesLookedUp(); + + /** + * Returns the number of all the packages currently loaded. + * + * <p> + * Note that this method is not affected by sync(), and the packages it + * returns are not guaranteed to be up-to-date. + */ + int getCacheSize(); + } + + /** + * Retrieve a target pattern parser that works with this package manager. + */ + TargetPatternEvaluator getTargetPatternEvaluator(); + + /** + * Construct a new {@link TransitivePackageLoader}. + */ + TransitivePackageLoader newTransitiveLoader(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/PackageProvider.java b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageProvider.java new file mode 100644 index 0000000..0573768e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/PackageProvider.java
@@ -0,0 +1,60 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.pkgcache; + +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageIdentifier; + +/** + * API for retrieving packages. Implementations generally load packages to fulfill requests. + * + * <p><b>Concurrency</b>: Implementations should be thread safe for {@link #getPackage}. + */ +public interface PackageProvider extends TargetProvider { + + /** + * Returns the {@link Package} named "packageName". If there is no such package (e.g. + * {@code isPackage(packageName)} returns false), throws a {@link NoSuchPackageException}. + * + * <p>The returned package may contain lexical/grammatical errors, in which + * case <code>pkg.containsErrors() == true</code>. Such packages may be + * missing some rules. Any rules that are present may soundly be used for + * builds, though. + * + * @param eventHandler the eventHandler on which to report warning and errors; if the package + * has been loaded by another thread, this eventHandler won't see any warnings or errors + * @param packageName a legal package name. + * @throws NoSuchPackageException if the package could not be found. + * @throws InterruptedException if the package loading was interrupted. + */ + Package getPackage(EventHandler eventHandler, PackageIdentifier packageName) + throws NoSuchPackageException, InterruptedException; + + /** + * Returns whether a package with the given name exists. That is, returns whether all the + * following hold + * <ol> + * <li>{@code packageName} is a valid package name</li> + * <li>there is a BUILD file for the package</li> + * <li>the package is not considered deleted via --deleted_packages</li> + * </ol> + * + * <p> If these don't hold, then attempting to read the package with {@link #getPackage} may fail + * or may return a package containing errors. + */ + boolean isPackage(String packageName); +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/ParseFailureListener.java b/src/main/java/com/google/devtools/build/lib/pkgcache/ParseFailureListener.java new file mode 100644 index 0000000..e3cf5ca --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/ParseFailureListener.java
@@ -0,0 +1,28 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.pkgcache; + +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.events.EventHandler; + +/** + * Represents a listener which reports parse errors to the underlying + * {@link EventHandler} and {@link EventBus} (if non-null). + */ +public interface ParseFailureListener { + + /** Reports a parsing failure. */ + void parsingError(String badPattern, String message); +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/ParsingFailedEvent.java b/src/main/java/com/google/devtools/build/lib/pkgcache/ParsingFailedEvent.java new file mode 100644 index 0000000..7eb9169 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/ParsingFailedEvent.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.pkgcache; + +/** + * This event is fired when a target or target pattern fails to parse. + * In some cases (not all) this happens before targets are created, + * and thus in these cases there are no status lines. + * Therefore, the parse failure is reported separately. + */ +public class ParsingFailedEvent { + private final String targetPattern; + private final String message; + + /** + * Creates a new parsing failed event with the given pattern and message. + */ + public ParsingFailedEvent(String targetPattern, String message) { + this.targetPattern = targetPattern; + this.message = message; + } + + public String getPattern() { + return targetPattern; + } + + public String getMessage() { + return message; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/PathPackageLocator.java b/src/main/java/com/google/devtools/build/lib/pkgcache/PathPackageLocator.java new file mode 100644 index 0000000..a2af5f2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/PathPackageLocator.java
@@ -0,0 +1,213 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.BuildFileNotFoundException; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.vfs.FileStatus; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.lib.vfs.UnixGlob; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Logger; + +/** + * A mapping from the name of a package to the location of its BUILD file. + * The implementation composes an ordered sequence of directories according to + * the package-path rules. + * + * <p>All methods are thread-safe, and (assuming no change to the underlying + * filesystem) idempotent. + */ +public class PathPackageLocator { + + public static final Set<String> DEFAULT_TOP_LEVEL_EXCLUDES = + ImmutableSet.of("experimental", "obsolete"); + + /** + * An interface which accepts {@link PathFragment}s. + */ + public interface AcceptsPathFragment { + + /** + * Accept a {@link PathFragment}. + * + * @param fragment The path fragment. + */ + void accept(PathFragment fragment); + } + + private static final Logger LOG = Logger.getLogger(PathPackageLocator.class.getName()); + + private final ImmutableList<Path> pathEntries; + + /** + * Constructs a PathPackageLocator based on the specified list of package root directories. + */ + public PathPackageLocator(List<Path> pathEntries) { + this.pathEntries = ImmutableList.copyOf(pathEntries); + } + + /** + * Constructs a PathPackageLocator based on the specified array of package root directories. + */ + public PathPackageLocator(Path... pathEntries) { + this(Arrays.asList(pathEntries)); + } + + /** + * Returns the path to the build file for this package. + * + * <p>The package's root directory may be computed by calling getParentFile() + * on the result of this function. + * + * <p>Instances of this interface do not attempt to do any caching, nor + * implement checks for package-boundary crossing logic; the PackageCache + * does that. + * + * <p>If the same package exists beneath multiple package path entries, the + * first path that matches always wins. + */ + public Path getPackageBuildFile(String packageName) throws NoSuchPackageException { + Path buildFile = getPackageBuildFileNullable(packageName, UnixGlob.DEFAULT_SYSCALLS_REF); + if (buildFile == null) { + throw new BuildFileNotFoundException(packageName, "BUILD file not found on package path"); + } + return buildFile; + } + + /** + * Like #getPackageBuildFile(), but returns null instead of throwing. + * + * @param packageName the name of the package. + * @param cache a filesystem-level cache of stat() calls. + */ + public Path getPackageBuildFileNullable(String packageName, + AtomicReference<? extends UnixGlob.FilesystemCalls> cache) { + return getFilePath(new PathFragment(packageName).getRelative("BUILD"), cache); + } + + + /** + * Returns an immutable ordered list of the directories on the package path. + */ + public ImmutableList<Path> getPathEntries() { + return pathEntries; + } + + @Override + public String toString() { + return "PathPackageLocator" + pathEntries; + } + + /** + * A factory of PathPackageLocators from a list of path elements. Elements + * may contain "%workspace%", indicating the workspace. + * + * @param pathElements Each element must be an absolute path, relative path, + * or some string "%workspace%" + relative, where relative is itself a + * relative path. The special symbol "%workspace%" means to interpret + * the path relative to the nearest enclosing workspace. Relative + * paths are interpreted relative to the client's working directory, + * which may be below the workspace. + * @param eventHandler The eventHandler. + * @param workspace The nearest enclosing package root directory. + * @param clientWorkingDirectory The client's working directory. + * @return a list of {@link Path}s. + */ + public static PathPackageLocator create(List<String> pathElements, + EventHandler eventHandler, + Path workspace, + Path clientWorkingDirectory) { + List<Path> resolvedPaths = new ArrayList<>(); + final String workspaceWildcard = "%workspace%"; + + for (String pathElement : pathElements) { + // Replace "%workspace%" with the path of the enclosing workspace directory. + pathElement = pathElement.replace(workspaceWildcard, workspace.getPathString()); + + PathFragment pathElementFragment = new PathFragment(pathElement); + + // If the path string started with "%workspace%" or "/", it is already absolute, + // so the following line is a no-op. + Path rootPath = clientWorkingDirectory.getRelative(pathElementFragment); + + if (!pathElementFragment.isAbsolute() && !clientWorkingDirectory.equals(workspace)) { + eventHandler.handle( + Event.warn("The package path element '" + pathElementFragment + "' will be " + + "taken relative to your working directory. You may have intended " + + "to have the path taken relative to your workspace directory. " + + "If so, please use the '" + workspaceWildcard + "' wildcard.")); + } + + if (rootPath.exists()) { + resolvedPaths.add(rootPath); + } else { + LOG.fine("package path element " + rootPath + " does not exist, ignoring"); + } + } + return new PathPackageLocator(resolvedPaths); + } + + /** + * Returns the path to the WORKSPACE file for this build. + * + * <p>If there are WORKSPACE files beneath multiple package path entries, the first one always + * wins. + */ + public Path getWorkspaceFile() { + AtomicReference<? extends UnixGlob.FilesystemCalls> cache = UnixGlob.DEFAULT_SYSCALLS_REF; + // TODO(bazel-team): correctness in the presence of changes to the location of the WORKSPACE + // file. + return getFilePath(new PathFragment("WORKSPACE"), cache); + } + + private Path getFilePath(PathFragment suffix, + AtomicReference<? extends UnixGlob.FilesystemCalls> cache) { + for (Path pathEntry : pathEntries) { + Path buildFile = pathEntry.getRelative(suffix); + FileStatus stat = cache.get().statNullable(buildFile, Symlinks.FOLLOW); + if (stat != null && stat.isFile()) { + return buildFile; + } + } + return null; + } + + @Override + public int hashCode() { + return pathEntries.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof PathPackageLocator)) { + return false; + } + return this.getPathEntries().equals(((PathPackageLocator) other).getPathEntries()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/RecursivePackageProvider.java b/src/main/java/com/google/devtools/build/lib/pkgcache/RecursivePackageProvider.java new file mode 100644 index 0000000..8d7bd1d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/RecursivePackageProvider.java
@@ -0,0 +1,57 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.concurrent.ThreadPoolExecutor; + +import javax.annotation.Nullable; + +/** + * Support for resolving {@code package/...} target patterns. + */ +public interface RecursivePackageProvider extends PackageProvider { + + /** + * <p>Visits the names of all packages beneath the given directory recursively and concurrently. + * + * <p>Note: This operation needs to stat directories recursively. It could be very expensive when + * there is a big tree under the given directory. + * + * <p>Over a single iteration, package names are unique. + * + * <p>This method uses the given thread pool to call the observer method, possibly concurrently + * (depending on the thread pool). When this method terminates, however, all such threads will + * have completed. + * + * <p>To abort the traversal, call {@link Thread#interrupt()} on the calling thread. + * + * <p>This method guarantees that all BUILD files it returns correspond to valid package names + * that are not marked as deleted within the current build. + * + * @param eventHandler an eventHandler which should be used to log any errors that occur while + * scanning directories for BUILD files + * @param directory a relative, canonical path specifying the directory to search + * @param useTopLevelExcludes whether to skip a pre-set list of top level directories + * @param visitorPool the thread pool to use to visit packages in parallel + * @param observer is called for each path fragment found; thread-safe if the thread pool supports + * multiple parallel threads + * @throws InterruptedException if the calling thread was interrupted. + */ + void visitPackageNamesRecursively(EventHandler eventHandler, PathFragment directory, + boolean useTopLevelExcludes, @Nullable ThreadPoolExecutor visitorPool, + PathPackageLocator.AcceptsPathFragment observer) throws InterruptedException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/SrcTargetUtil.java b/src/main/java/com/google/devtools/build/lib/pkgcache/SrcTargetUtil.java new file mode 100644 index 0000000..85d967e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/SrcTargetUtil.java
@@ -0,0 +1,148 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.FileTarget; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.RawAttributeMapper; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * A helper class for getting source and header files from a given {@link Rule}. + */ +public final class SrcTargetUtil { + private SrcTargetUtil() { + } + + /** + * Given a Rule, returns an immutable list of FileTarget for its sources, in the order they appear + * in its "srcs", "src" or "srcjar" attribute, and any filegroups or other rules it references. + * An empty list is returned if no "srcs" or "src" attribute exists for this rule. The list may + * contain OutputFiles if the sources were generated by another rule. + * + * <p>This method should be considered only a heuristic, and should not be used during the + * analysis phase. + * + * <p>(We could remove the throws clauses if we restrict the results to srcs within the same + * package.) + * + * @throws NoSuchTargetException or NoSuchPackageException when a source label cannot be resolved + * to a Target + */ + @ThreadSafety.ThreadSafe + public static List<FileTarget> getSrcTargets(EventHandler eventHandler, Rule rule, + TargetProvider provider) + throws NoSuchTargetException, NoSuchPackageException, InterruptedException { + return getTargets(eventHandler, rule, SOURCE_ATTRIBUTES, Sets.newHashSet(rule), provider); + } + + // Attributes referring to "sources". + private static final ImmutableSet<String> SOURCE_ATTRIBUTES = + ImmutableSet.of("srcs", "src", "srcjar"); + + // Attributes referring to "headers". + private static final ImmutableSet<String> HEADER_ATTRIBUTES = + ImmutableSet.of("hdrs"); + + // The attribute to search in filegroups. + private static final ImmutableSet<String> FILEGROUP_ATTRIBUTES = + ImmutableSet.of("srcs"); + + /** + * Same as {@link #getSrcTargets}, but for both source and headers (i.e. also traversing + * the "hdrs" attribute). + */ + @ThreadSafety.ThreadSafe + public static List<FileTarget> getSrcAndHdrTargets(EventHandler eventHandler, Rule rule, + TargetProvider provider) + throws NoSuchTargetException, NoSuchPackageException, InterruptedException { + ImmutableSet<String> srcAndHdrAttributes = ImmutableSet.<String>builder() + .addAll(SOURCE_ATTRIBUTES) + .addAll(HEADER_ATTRIBUTES) + .build(); + return getTargets(eventHandler, rule, srcAndHdrAttributes, Sets.newHashSet(rule), provider); + } + + @ThreadSafety.ThreadSafe + public static List<FileTarget> getHdrTargets(EventHandler eventHandler, Rule rule, + TargetProvider provider) + throws NoSuchTargetException, NoSuchPackageException, InterruptedException { + ImmutableSet<String> srcAndHdrAttributes = ImmutableSet.<String>builder() + .addAll(HEADER_ATTRIBUTES) + .build(); + return getTargets(eventHandler, rule, srcAndHdrAttributes, Sets.newHashSet(rule), provider); + } + + /** + * @see #getSrcTargets(EventHandler, Rule, TargetProvider) + */ + private static List<FileTarget> getTargets(EventHandler eventHandler, + Rule rule, + ImmutableSet<String> attributes, + Set<Rule> visitedRules, + TargetProvider targetProvider) + throws NoSuchTargetException, NoSuchPackageException, InterruptedException { + Preconditions.checkState(!rule.hasConfigurableAttributes()); // Not currently supported. + List<Label> srcLabels = Lists.newArrayList(); + AttributeMap attributeMap = RawAttributeMapper.of(rule); + for (String attrName : attributes) { + if (rule.isAttrDefined(attrName, Type.LABEL_LIST)) { + srcLabels.addAll(attributeMap.get(attrName, Type.LABEL_LIST)); + } else if (rule.isAttrDefined(attrName, Type.LABEL)) { + Label srcLabel = attributeMap.get(attrName, Type.LABEL); + if (srcLabel != null) { + srcLabels.add(srcLabel); + } + } + } + if (srcLabels.isEmpty()) { + return ImmutableList.of(); + } + List<FileTarget> srcTargets = new ArrayList<>(); + for (Label label : srcLabels) { + Target target = targetProvider.getTarget(eventHandler, label); + if (target instanceof FileTarget) { + srcTargets.add((FileTarget) target); + } else { + Rule srcRule = target.getAssociatedRule(); + if (srcRule != null && !visitedRules.contains(srcRule)) { + visitedRules.add(srcRule); + if ("filegroup".equals(srcRule.getRuleClass())) { + srcTargets.addAll(getTargets(eventHandler, srcRule, FILEGROUP_ATTRIBUTES, visitedRules, + targetProvider)); + } else { + srcTargets.addAll(srcRule.getOutputFiles()); + } + } + } + } + return ImmutableList.copyOf(srcTargets); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/TargetEdgeObserver.java b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetEdgeObserver.java new file mode 100644 index 0000000..acc858f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetEdgeObserver.java
@@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.pkgcache; + +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; + +import javax.annotation.Nullable; + +/** + * An observer of the visitation over a target graph. + */ +public interface TargetEdgeObserver { + + /** + * Called when an edge is discovered. + * May be called more than once for the same + * (from, to) pair. + * + * @param from the originating node. + * @param attribute The attribute which defines the edge. + * Non-null iff (from instanceof Rule). + * @param to the target node. + */ + void edge(Target from, Attribute attribute, Target to); + + /** + * Called when a Target has a reference to a non-existent target. + * + * @param target the target. May be null (e.g. in the case of an implicit + * dependency on a subincluded file). + * @param to a label reference in the rule, which does not correspond + * to a valid target. + * @param e the corresponding exception thrown + */ + void missingEdge(@Nullable Target target, Label to, NoSuchThingException e); + + /** + * Called when a node is discovered. May be called + * more than once for the same node. + * + * @param node the target. + */ + void node(Target node); +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/TargetParsingCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetParsingCompleteEvent.java new file mode 100644 index 0000000..48d8861 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetParsingCompleteEvent.java
@@ -0,0 +1,87 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Collection; + +/** + * This event is fired just after target pattern evaluation is completed. + */ +public class TargetParsingCompleteEvent { + + private final ImmutableSet<Target> targets; + private final ImmutableSet<Target> filteredTargets; + private final ImmutableSet<Target> testFilteredTargets; + private final long timeInMs; + + /** + * Construct the event. + * @param targets The targets that were parsed from the + * command-line pattern. + */ + public TargetParsingCompleteEvent(Collection<Target> targets, + Collection<Target> filteredTargets, Collection<Target> testFilteredTargets, + long timeInMs) { + this.timeInMs = timeInMs; + this.targets = ImmutableSet.copyOf(targets); + this.filteredTargets = ImmutableSet.copyOf(filteredTargets); + this.testFilteredTargets = ImmutableSet.copyOf(testFilteredTargets); + } + + @VisibleForTesting + public TargetParsingCompleteEvent(Collection<Target> targets) { + this(targets, ImmutableSet.<Target>of(), ImmutableSet.<Target>of(), 0); + } + + /** + * @return the parsed targets, which will subsequently be loaded + */ + public ImmutableSet<Target> getTargets() { + return targets; + } + + public Iterable<Label> getLabels() { + return Iterables.transform(targets, new Function<Target, Label>() { + @Override + public Label apply(Target input) { + return input.getLabel(); + } + }); + } + + /** + * @return the filtered targets (i.e., using -//foo:bar on the command-line) + */ + public ImmutableSet<Target> getFilteredTargets() { + return filteredTargets; + } + + /** + * @return the test-filtered targets, if --build_test_only is in effect + */ + public ImmutableSet<Target> getTestFilteredTargets() { + return testFilteredTargets; + } + + public long getTimeInMs() { + return timeInMs; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/TargetPatternEvaluator.java b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetPatternEvaluator.java new file mode 100644 index 0000000..94e956b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetPatternEvaluator.java
@@ -0,0 +1,97 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.pkgcache; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.lib.cmdline.ResolvedTargets; +import com.google.devtools.build.lib.cmdline.TargetParsingException; +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * A parser for target patterns. Target patterns are a generalisation of + * labels to include wildcards for finding all packages recursively + * beneath some root, and for finding all targets within a package. + * + * <p>A list of target patterns implies a union of all the labels of each + * pattern. Each item in a list of target patterns may include a prefix + * negation operator, indicating that the sets of targets for this pattern + * should be subtracted from the set of targets for the preceding patterns (note + * this means that order matters). Thus, the following list of target patterns: + * <pre>foo/... -foo/bar:all</pre> + * means "all targets beneath <tt>foo</tt> except for those targets in + * package <tt>foo/bar</tt>. + */ +@ThreadSafety.ConditionallyThreadSafe // as long as you don't call updateOffset. +public interface TargetPatternEvaluator { + /** + * Attempts to parse an ordered list of target patterns, computing the union + * of the set of targets represented by each pattern, unless it is preceded by + * "-", in which case the set difference is computed. Implements the + * specification described in the class-level comment. + */ + ResolvedTargets<Target> parseTargetPatternList(EventHandler eventHandler, + List<String> targetPatterns, FilteringPolicy policy, boolean keepGoing) + throws TargetParsingException, InterruptedException; + + /** + * Attempts to parse a single target pattern while consulting the package + * cache to check for the existence of packages and directories and the build + * targets in them. Implements the specification described in the + * class-level comment. Returns a {@link ResolvedTargets} object. + * + * <p>If an error is encountered, a {@link TargetParsingException} is thrown, + * unless {@code keepGoing} is set to true. In that case, the returned object + * will have its error bit set. + */ + ResolvedTargets<Target> parseTargetPattern(EventHandler eventHandler, String pattern, + boolean keepGoing) throws TargetParsingException, InterruptedException; + + /** + * Attempts to parse and load the given collection of patterns; the returned map contains the + * results for each pattern successfully parsed. + * + * <p>If an error is encountered, a {@link TargetParsingException} is thrown, unless {@code + * keepGoing} is set to true. In that case, the patterns that failed to load have the error flag + * set. + */ + Map<String, ResolvedTargets<Target>> preloadTargetPatterns(EventHandler eventHandler, + Collection<String> patterns, boolean keepGoing) + throws TargetParsingException, InterruptedException; + + + /** + * Update the parser's offset, given the workspace and working directory. + * + * @param relativeWorkingDirectory the working directory relative to the workspace + */ + @ThreadHostile + void updateOffset(PathFragment relativeWorkingDirectory); + + /** + * @return the offset of this parser from the root of the workspace. + * Non-absolute package-names will be resolved relative + * to this offset. + */ + @VisibleForTesting + String getOffset(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/TargetPatternResolverUtil.java b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetPatternResolverUtil.java new file mode 100644 index 0000000..6bbf55c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetPatternResolverUtil.java
@@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.devtools.build.lib.cmdline.ResolvedTargets; +import com.google.devtools.build.lib.cmdline.TargetParsingException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.StringUtilities; + +/** + * Common utility methods for target pattern resolution. + */ +public final class TargetPatternResolverUtil { + private TargetPatternResolverUtil() { + // Utility class. + } + + // Parse 'label' as a Label, mapping Label.SyntaxException into + // TargetParsingException. + public static Label label(String label) throws TargetParsingException { + try { + return Label.parseAbsolute(label); + } catch (Label.SyntaxException e) { + throw invalidTarget(label, e.getMessage()); + } + } + + /** + * Returns a new exception indicating that a command-line target is invalid. + */ + private static TargetParsingException invalidTarget(String packageName, + String additionalMessage) { + return new TargetParsingException("invalid target format: '" + + StringUtilities.sanitizeControlChars(packageName) + "'; " + + StringUtilities.sanitizeControlChars(additionalMessage)); + } + + public static String getParsingErrorMessage(String message, String originalPattern) { + if (originalPattern == null) { + return message; + } else { + return String.format("while parsing '%s': %s", originalPattern, message); + } + } + + public static ResolvedTargets<Target> resolvePackageTargets(Package pkg, + FilteringPolicy policy) { + ResolvedTargets.Builder<Target> builder = ResolvedTargets.builder(); + for (Target target : pkg.getTargets()) { + if (policy.shouldRetain(target, false)) { + builder.add(target); + } + } + return builder.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/TargetProvider.java b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetProvider.java new file mode 100644 index 0000000..72dc834 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/TargetProvider.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.pkgcache; + +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; + +/** + * API for retrieving targets. + * + * <p><b>Concurrency</b>: Implementations should be thread safe. + */ +public interface TargetProvider { + /** + * Returns the Target identified by "label", loading, parsing and evaluating the package if it is + * not already loaded. + * + * @throws NoSuchPackageException if the package could not be found + * @throws NoSuchTargetException if the package was loaded successfully, but + * the specified {@link Target} was not found in it + * @throws InterruptedException if the package loading was interrupted + */ + Target getTarget(EventHandler eventHandler, Label label) throws NoSuchPackageException, + NoSuchTargetException, InterruptedException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/TransitivePackageLoader.java b/src/main/java/com/google/devtools/build/lib/pkgcache/TransitivePackageLoader.java new file mode 100644 index 0000000..14990b2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/TransitivePackageLoader.java
@@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.pkgcache; + +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Collection; +import java.util.Set; + +/** + * Visits a set of Targets and Labels transitively. + */ +public interface TransitivePackageLoader { + + /** + * Visit the specified labels and follow the transitive closure of their + * outbound dependencies. If the targets have previously been visited, + * may do an up-to-date check which will not trigger any of the observers. + * + * @param eventHandler the error and warnings eventHandler; must be thread-safe + * @param targetsToVisit the targets to visit + * @param labelsToVisit the labels to visit in addition to the targets + * @param keepGoing if false, stop visitation upon first error. + * @param parallelThreads number of threads to use in the visitation. + * @param maxDepth the maximum depth to traverse to. + */ + boolean sync(EventHandler eventHandler, + Set<Target> targetsToVisit, + Set<Label> labelsToVisit, + boolean keepGoing, + int parallelThreads, + int maxDepth) throws InterruptedException; + + /** + * Returns a read-only view of the set of targets visited since this visitor + * was constructed. + * + * <p>Not thread-safe; do not call during visitation. + */ + // TODO(bazel-team): This is only used in legacy non-Skyframe code. + Set<Label> getVisitedTargets(); + + /** + * Returns a read-only view of the set of packages visited since this visitor + * was constructed. + * + * <p>Not thread-safe; do not call during visitation. + */ + Set<PackageIdentifier> getVisitedPackageNames(); + + /** + * Returns a read-only view of the set of the actual packages visited without error since this + * visitor was constructed. + * + * <p>Use {@link #getVisitedPackageNames()} instead when possible. + * + * <p>Not thread-safe; do not call during visitation. + */ + Set<Package> getErrorFreeVisitedPackages(); + + /** + * Return a mapping between the specified top-level targets and root causes. Note that targets in + * the input that are transitively error free will not be in the output map. "Top-level" targets + * are the targetsToVisit and labelsToVisit specified in the last sync. + * + * <p>May only be called once a keep_going visitation is complete, and prior to + * trimErrorTracking(). + * + * @param targetsToLoad the set of targets to be checked. Implementations may choose to only + * return root causes for targets in this set that were requested top-level targets. + * @return a mapping of targets to root causes + */ + Multimap<Label, Label> getRootCauses(Collection<Label> targetsToLoad); +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/Describable.java b/src/main/java/com/google/devtools/build/lib/profiler/Describable.java new file mode 100644 index 0000000..a5cc08a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/Describable.java
@@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler; + +/** + * Allows class to implement profiler-friendly (and user-friendly) + * textual description of the object that would uniquely identify an object in + * the profiler data dump. + */ +public interface Describable { + + /** + * Returns textual description that will uniquely identify an object. + */ + String describe(); + +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/MemoryProfiler.java b/src/main/java/com/google/devtools/build/lib/profiler/MemoryProfiler.java new file mode 100644 index 0000000..25ebd53 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/MemoryProfiler.java
@@ -0,0 +1,80 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler; + +import java.io.OutputStream; +import java.io.PrintStream; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryUsage; + +/** + * Blaze memory profiler. + * + * <p>At each call to {@code profile} performs garbage collection and stores + * heap and non-heap memory usage in an external file. + * + * <p><em>Heap memory</em> is the runtime data area from which memory for all + * class instances and arrays is allocated. <em>Non-heap memory</em> includes + * the method area and memory required for the internal processing or + * optimization of the JVM. It stores per-class structures such as a runtime + * constant pool, field and method data, and the code for methods and + * constructors. The Java Native Interface (JNI) code or the native library of + * an application and the JVM implementation allocate memory from the + * <em>native heap</em>. + * + * <p>The script in /devtools/blaze/scripts/blaze-memchart.sh can be used for post processing. + */ +public final class MemoryProfiler { + + private static final MemoryProfiler INSTANCE = new MemoryProfiler(); + + public static MemoryProfiler instance() { + return INSTANCE; + } + + private PrintStream memoryProfile; + private ProfilePhase currentPhase; + + public synchronized void start(OutputStream out) { + this.memoryProfile = (out == null) ? null : new PrintStream(out); + this.currentPhase = ProfilePhase.INIT; + } + + public synchronized void stop() { + if (memoryProfile != null) { + memoryProfile.close(); + memoryProfile = null; + } + } + + public synchronized void markPhase(ProfilePhase nextPhase) { + if (memoryProfile != null) { + String name = currentPhase.description; + ManagementFactory.getMemoryMXBean().gc(); + MemoryUsage memoryUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + memoryProfile.println(name + ":heap:init:" + memoryUsage.getInit()); + memoryProfile.println(name + ":heap:used:" + memoryUsage.getUsed()); + memoryProfile.println(name + ":heap:commited:" + memoryUsage.getCommitted()); + memoryProfile.println(name + ":heap:max:" + memoryUsage.getMax()); + + memoryUsage = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage(); + memoryProfile.println(name + ":non-heap:init:" + memoryUsage.getInit()); + memoryProfile.println(name + ":non-heap:used:" + memoryUsage.getUsed()); + memoryProfile.println(name + ":non-heap:commited:" + memoryUsage.getCommitted()); + memoryProfile.println(name + ":non-heap:max:" + memoryUsage.getMax()); + currentPhase = nextPhase; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfileInfo.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfileInfo.java new file mode 100644 index 0000000..4ce3a93 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/ProfileInfo.java
@@ -0,0 +1,926 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.profiler; + +import static com.google.devtools.build.lib.profiler.ProfilerTask.CRITICAL_PATH; +import static com.google.devtools.build.lib.profiler.ProfilerTask.CRITICAL_PATH_COMPONENT; +import static com.google.devtools.build.lib.profiler.ProfilerTask.TASK_COUNT; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.util.VarInt; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Deque; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; + +/** + * Holds parsed profile file information and provides various ways of + * accessing it (mostly through different dictionaries or sorted lists). + * + * Class should not be instantiated directly but through the use of the + * ProfileLoader.loadProfile() method. + */ +public class ProfileInfo { + + /** + * Immutable container for the aggregated stats. + */ + public static final class AggregateAttr { + public final int count; + public final long totalTime; + + AggregateAttr(int count, long totalTime) { + this.count = count; + this.totalTime = totalTime; + } + } + + /** + * Immutable compact representation of the Map<ProfilerTask, AggregateAttr>. + */ + static final class CompactStatistics { + final byte[] content; + + CompactStatistics(byte[] content) { + this.content = content; + } + + /** + * Create compact task statistic instance using provided array. + * Array length must exactly match ProfilerTask value space. + * Each statistic is stored in the array according to the ProfilerTask + * value ordinal() number. Absent statistics are represented by null. + */ + CompactStatistics(AggregateAttr[] stats) { + Preconditions.checkArgument(stats.length == TASK_COUNT); + ByteBuffer sink = ByteBuffer.allocate(TASK_COUNT * (1 + 5 + 10)); + for (int i = 0; i < TASK_COUNT; i++) { + if (stats[i] != null && stats[i].count > 0) { + sink.put((byte) i); + VarInt.putVarInt(stats[i].count, sink); + VarInt.putVarLong(stats[i].totalTime, sink); + } + } + content = sink.position() > 0 ? Arrays.copyOfRange(sink.array(), 0, sink.position()) : null; + } + + boolean isEmpty() { return content == null; } + + /** + * Converts instance back into AggregateAttr[TASK_COUNT]. See + * constructor documentation for more information. + */ + AggregateAttr[] toArray() { + AggregateAttr[] stats = new AggregateAttr[TASK_COUNT]; + if (!isEmpty()) { + ByteBuffer source = ByteBuffer.wrap(content); + while (source.hasRemaining()) { + byte id = source.get(); + int count = VarInt.getVarInt(source); + long time = VarInt.getVarLong(source); + stats[id] = new AggregateAttr(count, time); + } + } + return stats; + } + + /** + * Returns AggregateAttr instance for the given ProfilerTask value. + */ + AggregateAttr getAttr(ProfilerTask task) { + if (isEmpty()) { return ZERO; } + ByteBuffer source = ByteBuffer.wrap(content); + byte id = (byte) task.ordinal(); + while (source.hasRemaining()) { + if (id == source.get()) { + int count = VarInt.getVarInt(source); + long time = VarInt.getVarLong(source); + return new AggregateAttr(count, time); + } else { + VarInt.getVarInt(source); + VarInt.getVarLong(source); + } + } + return ZERO; + } + + /** + * Returns cumulative time stored in this instance across whole + * ProfilerTask dimension. + */ + long getTotalTime() { + if (isEmpty()) { return 0; } + ByteBuffer source = ByteBuffer.wrap(content); + long totalTime = 0; + while (source.hasRemaining()) { + source.get(); + VarInt.getVarInt(source); + totalTime += VarInt.getVarLong(source); + } + return totalTime; + } + } + + /** + * Container for the profile record information. + * + * <p> TODO(bazel-team): (2010) Current Task instance heap size is 72 bytes. And there are + * millions of them. Consider trimming some attributes. + */ + public final class Task implements Comparable<Task> { + public final long threadId; + public final int id; + public final int parentId; + public final long startTime; + public final long duration; + public final ProfilerTask type; + final CompactStatistics stats; + // Contains statistic for a task and all subtasks. Populated only for root tasks. + CompactStatistics aggregatedStats = null; + // Subtasks are stored as an array for performance and memory utilization + // reasons (we can easily deal with millions of those objects). + public Task[] subtasks = NO_TASKS; + final int descIndex; + // Reference to the related task (e.g. ACTION_GRAPH->ACTION task relation). + private Task relatedTask; + + Task(long threadId, int id, int parentId, long startTime, long duration, + ProfilerTask type, int descIndex, CompactStatistics stats) { + this.threadId = threadId; + this.id = id; + this.parentId = parentId; + this.startTime = startTime; + this.duration = duration; + this.type = type; + this.descIndex = descIndex; + this.stats = stats; + relatedTask = null; + } + + public String getDescription() { + return descriptionList.get(descIndex); + } + + public boolean hasStats() { + return !stats.isEmpty(); + } + + public long getInheritedDuration() { + return stats.getTotalTime(); + } + + public AggregateAttr[] getStatAttrArray() { + Preconditions.checkNotNull(stats); + return stats.toArray(); + } + + private void combineStats(int[] counts, long[] duration) { + int ownIndex = type.ordinal(); + if (parentId != 0) { + // Parent task already accounted for this task total duration. We need to adjust + // for the inherited duration. + duration[ownIndex] -= getInheritedDuration(); + } + AggregateAttr[] ownStats = stats.toArray(); + for (int i = 0; i < TASK_COUNT; i++) { + AggregateAttr attr = ownStats[i]; + if (attr != null) { + counts[i] += attr.count; + duration[i] += attr.totalTime; + } + } + for (Task task : subtasks) { + task.combineStats(counts, duration); + } + } + + /** + * Calculates aggregated statistics covering all subtasks (including + * nested ones). Must be called only for parent tasks. + */ + void calculateRootStats() { + Preconditions.checkState(parentId == 0); + int[] counts = new int[TASK_COUNT]; + long[] duration = new long[TASK_COUNT]; + combineStats(counts, duration); + AggregateAttr[] statArray = ProfileInfo.createEmptyStatArray(); + for (int i = 0; i < TASK_COUNT; i++) { + statArray[i] = new AggregateAttr(counts[i], duration[i]); + } + this.aggregatedStats = new CompactStatistics(statArray); + } + + @Override + public boolean equals(Object o) { + return (o instanceof ProfileInfo.Task) && ((Task) o).id == this.id; + } + + @Override + public int hashCode() { + return this.id; + } + + @Override + public String toString() { + return type + "(" + id + "," + getDescription() + ")"; + } + + /** + * Tasks records by default sorted by their id. Since id was obtained using + * AtomicInteger, this comparison will correctly sort tasks in time-ascending + * order regardless of their origin thread. + */ + @Override + public int compareTo(Task task) { + return this.id - task.id; + } + } + + /** + * Represents node on critical build path + */ + public static final class CriticalPathEntry { + public final Task task; + public final long duration; + public final long cumulativeDuration; + public final CriticalPathEntry next; + + private long criticalTime = 0L; + + public CriticalPathEntry(Task task, long duration, CriticalPathEntry next) { + this.task = task; + this.duration = duration; + this.next = next; + this.cumulativeDuration = + duration + (next != null ? next.cumulativeDuration : 0); + } + + private void setCriticalTime(long duration) { + criticalTime = duration; + } + + public long getCriticalTime() { + return criticalTime; + } + } + + /** + * Helper class to create space-efficient task multimap, used to associate + * array of tasks with specific key. + */ + private abstract static class TaskMapCreator<K> implements Comparator<Task> { + @Override + public abstract int compare(Task a, Task b); + public abstract K getKey(Task task); + + public Map<K, Task[]> createTaskMap(List<Task> taskList) { + // Created map usually will end up with thousands of entries, so we + // preinitialize it to the 10000. + Map<K, Task[]> taskMap = Maps.newHashMapWithExpectedSize(10000); + if (taskList.size() == 0) { return taskMap; } + Task[] taskArray = taskList.toArray(new Task[taskList.size()]); + Arrays.sort(taskArray, this); + K key = getKey(taskArray[0]); + int start = 0; + for (int i = 0; i < taskArray.length; i++) { + K currentKey = getKey(taskArray[i]); + if (!key.equals(currentKey)) { + taskMap.put(key, Arrays.copyOfRange(taskArray, start, i)); + key = currentKey; + start = i; + } + } + if (start < taskArray.length) { + taskMap.put(key, Arrays.copyOfRange(taskArray, start, taskArray.length)); + } + return taskMap; + } + } + + /** + * An interface to pass back profile loading and aggregation messages. + */ + public interface InfoListener { + void info(String text); + void warn(String text); + } + + private static final Task[] NO_TASKS = new Task[0]; + private static final AggregateAttr ZERO = new AggregateAttr(0, 0); + + public final String comment; + private boolean corruptedOrIncomplete = false; + + // TODO(bazel-team): (2010) In one case, this list took 277MB of heap. Ideally it should be + // replaced with a trie. + private final List<String> descriptionList; + private final Map<Task, Task> parallelBuilderCompletionQueueTasks; + public final Map<Long, Task[]> tasksByThread; + public final List<Task> allTasksById; + public List<Task> rootTasksById; // Not final due to the late initialization. + public final List<Task> phaseTasks; + + public final Map<Task, Task[]> actionDependencyMap; + // Used to create fake Action tasks if ACTIONG_GRAPH task does not have + // corresponding ACTION task. For action dependency calculations we will + // create fake ACTION tasks and assign them negative ids. + private int fakeActionId = 0; + + private ProfileInfo(String comment) { + this.comment = comment; + + descriptionList = Lists.newArrayListWithExpectedSize(10000); + tasksByThread = Maps.newHashMap(); + parallelBuilderCompletionQueueTasks = Maps.newHashMap(); + allTasksById = Lists.newArrayListWithExpectedSize(50000); + phaseTasks = Lists.newArrayList(); + actionDependencyMap = Maps.newHashMapWithExpectedSize(10000); + } + + private void addTask(Task task) { + allTasksById.add(task); + } + + /** + * Returns true if profile datafile was corrupted or incomplete + * and false otherwise. + */ + public boolean isCorruptedOrIncomplete() { + return corruptedOrIncomplete; + } + + /** + * Returns number of missing actions which were faked in order to complete + * action graph. + */ + public int getMissingActionsCount() { + return -fakeActionId; + } + + /** + * Initializes minimum internal data structures necessary to obtain individual + * task statistic. This method is sufficient to initialize data for dumping. + */ + public void calculateStats() { + if (allTasksById.size() == 0) { + return; + } + + Collections.sort(allTasksById); + + Map<Integer, Task[]> subtaskMap = new TaskMapCreator<Integer>() { + @Override + public int compare(Task a, Task b) { + return a.parentId != b.parentId ? a.parentId - b.parentId : a.compareTo(b); + } + @Override + public Integer getKey(Task task) { return task.parentId; } + }.createTaskMap(allTasksById); + for (Task task : allTasksById) { + Task[] subtasks = subtaskMap.get(task.id); + if (subtasks != null) { + task.subtasks = subtasks; + } + } + rootTasksById = Arrays.asList(subtaskMap.get(0)); + + for (Task task : rootTasksById) { + task.calculateRootStats(); + if (task.type == ProfilerTask.PHASE) { + if (!phaseTasks.isEmpty()) { + phaseTasks.get(phaseTasks.size() - 1).relatedTask = task; + } + phaseTasks.add(task); + } + } + } + + /** + * Analyzes task relationships and dependencies. Used for the detailed profile + * analysis. + */ + public void analyzeRelationships() { + tasksByThread.putAll(new TaskMapCreator<Long>() { + @Override + public int compare(Task a, Task b) { + return a.threadId != b.threadId ? (a.threadId < b.threadId ? -1 : 1) : a.compareTo(b); + } + @Override + public Long getKey(Task task) { return task.threadId; } + }.createTaskMap(rootTasksById)); + + buildDependencyMap(); + } + + /** + * Calculates cumulative time attributed to the specific task type. + * Expects to be called only for root (parentId = 0) tasks. + * calculateStats() must have been called first. + */ + public AggregateAttr getStatsForType(ProfilerTask type, Collection<Task> tasks) { + long totalTime = 0; + int count = 0; + for (Task task : tasks) { + if (task.parentId > 0) { + throw new IllegalArgumentException("task " + task.id + " is not a root task"); + } + AggregateAttr attr = task.aggregatedStats.getAttr(type); + count += attr.count; + totalTime += attr.totalTime; + if (task.type == type) { + count++; + totalTime += (task.duration - task.getInheritedDuration()); + } + } + return new AggregateAttr(count, totalTime); + } + + /** + * Returns list of all root tasks related to (in other words, started during) + * the specified phase task. + */ + public List<Task> getTasksForPhase(Task phaseTask) { + Preconditions.checkArgument(phaseTask.type == ProfilerTask.PHASE, + "Unsupported task type %s", phaseTask.type); + + // Algorithm below takes into account fact that rootTasksById list is sorted + // by the task id and task id values are monotonically increasing with time + // (this property is guaranteed by the profiler). Thus list is effectively + // sorted by the startTime. We are trying to select a sublist that includes + // all tasks that were started later than the given task but earlier than + // its completion time. + int startIndex = Collections.binarySearch(rootTasksById, phaseTask); + Preconditions.checkState(startIndex >= 0, + "Phase task %s is not a root task", phaseTask.id); + int endIndex = (phaseTask.relatedTask != null) + ? Collections.binarySearch(rootTasksById, phaseTask.relatedTask) + : rootTasksById.size(); + Preconditions.checkState(endIndex >= startIndex, + "Failed to find end of the phase marked by the task %s", phaseTask.id); + return rootTasksById.subList(startIndex, endIndex); + } + + /** + * Returns task with "Build artifacts" description - corresponding to the + * execution phase. Usually used to location ACTION_GRAPH task tree. + */ + public Task getPhaseTask(ProfilePhase phase) { + for (Task task : phaseTasks) { + if (task.getDescription().equals(phase.description)) { + return task; + } + } + return null; + } + + /** + * Returns duration of the given phase in ns. + */ + public long getPhaseDuration(Task phaseTask) { + Preconditions.checkArgument(phaseTask.type == ProfilerTask.PHASE, + "Unsupported task type %s", phaseTask.type); + + long duration; + if (phaseTask.relatedTask != null) { + duration = phaseTask.relatedTask.startTime - phaseTask.startTime; + } else { + Task lastTask = rootTasksById.get(rootTasksById.size() - 1); + duration = lastTask.startTime + lastTask.duration - phaseTask.startTime; + } + Preconditions.checkState(duration >= 0); + return duration; + } + + + /** + * Builds map of dependencies between ACTION tasks based on dependencies + * between ACTION_GRAPH tasks + */ + private Task buildActionTaskTree(Task actionGraphTask, List<Task> actionTasksByDescription) { + Task actionTask = actionGraphTask.relatedTask; + if (actionTask == null) { + actionTask = actionTasksByDescription.get(actionGraphTask.descIndex); + if (actionTask == null) { + // If we cannot find ACTION task that corresponds to the ACTION_GRAPH task, + // most likely scenario is that we dealing with either aborted or failed + // build. In this case we will find or create fake zero-duration action + // task and still reconstruct dependency graph. + actionTask = new Task(-1, --fakeActionId, 0, 0, 0, + ProfilerTask.ACTION, actionGraphTask.descIndex, new CompactStatistics((byte[]) null)); + actionTask.calculateRootStats(); + actionTasksByDescription.set(actionGraphTask.descIndex, actionTask); + } + actionGraphTask.relatedTask = actionTask; + } + if (actionGraphTask.subtasks.length != 0) { + List<Task> list = Lists.newArrayListWithCapacity(actionGraphTask.subtasks.length); + for (Task task : actionGraphTask.subtasks) { + if (task.type == ProfilerTask.ACTION_GRAPH) { + list.add(buildActionTaskTree(task, actionTasksByDescription)); + } + } + if (!list.isEmpty()) { + Task[] actionPrerequisites = list.toArray(new Task[list.size()]); + Arrays.sort(actionPrerequisites); + actionDependencyMap.put(actionTask, actionPrerequisites); + } + } + return actionTask; + } + + /** + * Builds map of dependencies between ACTION tasks based on dependencies + * between ACTION_GRAPH tasks. Root of that dependency tree would be + * getBuildPhaseTask(). + * + * <p> Also marks related ACTION and ACTION_SUBMIT tasks. + */ + private void buildDependencyMap() { + Task analysisPhaseTask = getPhaseTask(ProfilePhase.ANALYZE); + Task executionPhaseTask = getPhaseTask(ProfilePhase.EXECUTE); + if ((executionPhaseTask == null) || (analysisPhaseTask == null)) { + return; + } + // Association between ACTION_GRAPH tasks and ACTION tasks can be established through + // description id. So we create appropriate xref list. + List<Task> actionTasksByDescription = Lists.newArrayList(new Task[descriptionList.size()]); + for (Task task : getTasksForPhase(executionPhaseTask)) { + if (task.type == ProfilerTask.ACTION) { + actionTasksByDescription.set(task.descIndex, task); + } + } + List<Task> list = new ArrayList<>(); + for (Task task : getTasksForPhase(analysisPhaseTask)) { + if (task.type == ProfilerTask.ACTION_GRAPH) { + list.add(buildActionTaskTree(task, actionTasksByDescription)); + } + } + Task[] actionPrerequisites = list.toArray(new Task[list.size()]); + Arrays.sort(actionPrerequisites); + actionDependencyMap.put(executionPhaseTask, actionPrerequisites); + + // Scan through all execution phase tasks to identify ACTION_SUBMIT tasks and associate + // them with ACTION task counterparts. ACTION_SUBMIT tasks are not necessarily root + // tasks so we need to scan ALL tasks. + for (Task task : allTasksById.subList(executionPhaseTask.id, allTasksById.size())) { + if (task.type == ProfilerTask.ACTION_SUBMIT) { + Task actionTask = actionTasksByDescription.get(task.descIndex); + if (actionTask != null) { + task.relatedTask = actionTask; + actionTask.relatedTask = task; + } + } else if (task.type == ProfilerTask.ACTION_BUILDER) { + Task actionTask = actionTasksByDescription.get(task.descIndex); + if (actionTask != null) { + parallelBuilderCompletionQueueTasks.put(actionTask, task); + } + } + } + } + + /** + * Calculates critical path for the specific action + * excluding specified nested task types (e.g. VFS-related time) and not + * accounting for overhead related to the Blaze scheduler. + */ + private CriticalPathEntry computeCriticalPathForAction( + Set<ProfilerTask> ignoredTypes, Set<Task> ignoredTasks, + Task actionTask, Map<Task, CriticalPathEntry> cache, Deque<Task> stack) { + + // Loop check is expensive for the Deque (and we don't want to use hash sets because adding + // and removing elements was shown to be very expensive). To avoid quadratic costs we're + // checking for infinite loop only when deque's size equal to the power of 2 and >= 32. + if ((stack.size() & 0x1F) == 0 && Integer.bitCount(stack.size()) == 1) { + if (stack.contains(actionTask)) { + // This situation will appear if build has ended with the + // IllegalStateException thrown by the + // ParallelBuilder.getNextCompletedAction(), warning user about + // possible cycle in the dependency graph. But the exception text + // is more friendly and will actually identify the loop. + // Do not use Preconditions class below due to the very expensive + // toString() calls used in the message. + throw new IllegalStateException ("Dependency graph contains loop:\n" + + actionTask + " in the\n" + Joiner.on('\n').join(stack)); + } + } + stack.addLast(actionTask); + CriticalPathEntry entry; + try { + entry = cache.get(actionTask); + long entryDuration = 0; + if (entry == null) { + Task[] actionPrerequisites = actionDependencyMap.get(actionTask); + if (actionPrerequisites != null) { + for (Task task : actionPrerequisites) { + CriticalPathEntry candidate = + computeCriticalPathForAction(ignoredTypes, ignoredTasks, task, cache, stack); + if (entry == null || entryDuration < candidate.cumulativeDuration) { + entry = candidate; + entryDuration = candidate.cumulativeDuration; + } + } + } + if (actionTask.type == ProfilerTask.ACTION) { + long duration = actionTask.duration; + if (ignoredTasks.contains(actionTask)) { + duration = 0L; + } else { + for (ProfilerTask type : ignoredTypes) { + duration -= actionTask.aggregatedStats.getAttr(type).totalTime; + } + } + + entry = new CriticalPathEntry(actionTask, duration, entry); + cache.put(actionTask, entry); + } + } + } finally { + stack.removeLast(); + } + return entry; + } + + /** + * Returns the critical path information from the {@code CriticalPathComputer} recorded stats. + * This code does not have the "Critical" column (Time difference if we removed this node from + * the critical path). + */ + public CriticalPathEntry getCriticalPathNewVersion() { + for (Task task : rootTasksById) { + if (task.type == CRITICAL_PATH) { + CriticalPathEntry entry = null; + for (Task shared : task.subtasks) { + entry = new CriticalPathEntry(shared, shared.duration, entry); + } + return entry; + } + } + return null; + } + + /** + * Calculates critical path for the given action graph excluding + * specified tasks (usually ones that belong to the "real" critical path). + */ + public CriticalPathEntry getCriticalPath(Set<ProfilerTask> ignoredTypes) { + Task actionTask = getPhaseTask(ProfilePhase.EXECUTE); + if (actionTask == null) { + return null; + } + Map <Task, CriticalPathEntry> cache = Maps.newHashMapWithExpectedSize(1000); + CriticalPathEntry result = computeCriticalPathForAction(ignoredTypes, + new HashSet<Task>(), actionTask, cache, + new ArrayDeque<Task>()); + if (result != null) { + return result; + } + return getCriticalPathNewVersion(); + } + + /** + * Calculates critical path time that will be saved by eliminating specific + * entry from the critical path + */ + public void analyzeCriticalPath(Set<ProfilerTask> ignoredTypes, CriticalPathEntry path) { + // With light critical path we do not need to analyze since it is already preprocessed + // by blaze build. + if (path != null && path.task.type == CRITICAL_PATH_COMPONENT) { + return; + } + for (CriticalPathEntry entry = path; entry != null; entry = entry.next) { + Map <Task, CriticalPathEntry> cache = Maps.newHashMapWithExpectedSize(1000); + entry.setCriticalTime(path.cumulativeDuration - + computeCriticalPathForAction(ignoredTypes, Sets.newHashSet(entry.task), + getPhaseTask(ProfilePhase.EXECUTE), cache, new ArrayDeque<Task>()) + .cumulativeDuration); + } + } + + /** + * Return the next critical path entry for the task or null if there is none. + */ + public CriticalPathEntry getNextCriticalPathEntryForTask(CriticalPathEntry path, Task task) { + for (CriticalPathEntry entry = path; entry != null; entry = entry.next) { + if (entry.task.id == task.id) { + return entry; + } + } + return null; + } + + /** + * Returns time action waited in the execution queue (difference between + * ACTION task start time and ACTION_SUBMIT task start time). + */ + public long getActionWaitTime(Task actionTask) { + // Light critical path does not record wait time. + if (actionTask.type == ProfilerTask.CRITICAL_PATH_COMPONENT) { + return 0; + } + Preconditions.checkArgument(actionTask.type == ProfilerTask.ACTION); + if (actionTask.relatedTask != null) { + Preconditions.checkState(actionTask.relatedTask.type == ProfilerTask.ACTION_SUBMIT); + long time = actionTask.startTime - actionTask.relatedTask.startTime; + Preconditions.checkState(time >= 0); + return time; + } else { + return 0L; // submission time is not available. + } + } + + /** + * Returns time action waited in the parallel builder completion queue + * (difference between ACTION task end time and ACTION_BUILDER start time). + */ + public long getActionQueueTime(Task actionTask) { + // Light critical path does not record queue time. + if (actionTask.type == ProfilerTask.CRITICAL_PATH_COMPONENT) { + return 0; + } + Preconditions.checkArgument(actionTask.type == ProfilerTask.ACTION); + Task related = parallelBuilderCompletionQueueTasks.get(actionTask); + if (related != null) { + Preconditions.checkState(related.type == ProfilerTask.ACTION_BUILDER); + long time = related.startTime - (actionTask.startTime + actionTask.duration); + Preconditions.checkState(time >= 0); + return time; + } else { + return 0L; // queue task is not available. + } + } + + /** + * Returns an empty array used to store task statistics. Array index + * corresponds to the ProfilerTask ordinal() value associated with the + * given statistic. Absent statistics are stored as null. + * <p> + * In essence, it is a fast equivalent of Map<ProfilerTask, AggregateAttr>. + */ + public static AggregateAttr[] createEmptyStatArray() { + return new AggregateAttr[TASK_COUNT]; + } + + /** + * Loads and parses Blaze profile file. + * + * @param profileFile profile file path + * + * @return ProfileInfo object with some fields populated (call calculateStats() + * and analyzeRelationships() to populate the remaining fields) + * @throws UnsupportedEncodingException if the file format is invalid + * @throws IOException if the file can't be read + */ + public static ProfileInfo loadProfile(Path profileFile) + throws IOException { + // It is extremely important to wrap InflaterInputStream using + // BufferedInputStream because majority of reads would be done using + // readInt()/readLong() methods and InflaterInputStream is very inefficient + // in handling small read requests (performance difference with 1MB buffer + // used below is almost 10x). + DataInputStream in = new DataInputStream( + new BufferedInputStream(new InflaterInputStream( + profileFile.getInputStream(), new Inflater(false), 65536), 1024 * 1024)); + + if (in.readInt() != Profiler.MAGIC) { + in.close(); + throw new UnsupportedEncodingException("Invalid profile datafile format"); + } + if (in.readInt() != Profiler.VERSION) { + in.close(); + throw new UnsupportedEncodingException("Incompatible profile datafile version"); + } + String fileComment = in.readUTF(); + + // Read list of used record types + int typeCount = in.readInt(); + boolean hasUnknownTypes = false; + Set<String> supportedTasks = new HashSet<>(); + for (ProfilerTask task : ProfilerTask.values()) { + supportedTasks.add(task.toString()); + } + List<ProfilerTask> typeList = new ArrayList<>(); + for (int i = 0; i < typeCount; i++) { + String name = in.readUTF(); + if (supportedTasks.contains(name)) { + typeList.add(ProfilerTask.valueOf(name)); + } else { + hasUnknownTypes = true; + typeList.add(ProfilerTask.UNKNOWN); + } + } + + ProfileInfo info = new ProfileInfo(fileComment); + + // Read record until we encounter end marker (-1). + // TODO(bazel-team): Maybe this still should handle corrupted(truncated) files. + try { + int size; + while ((size = in.readInt()) != Profiler.EOF_MARKER) { + byte[] backingArray = new byte[size]; + in.readFully(backingArray); + ByteBuffer buffer = ByteBuffer.wrap(backingArray); + long threadId = VarInt.getVarLong(buffer); + int id = VarInt.getVarInt(buffer); + int parentId = VarInt.getVarInt(buffer); + long startTime = VarInt.getVarLong(buffer); + long duration = VarInt.getVarLong(buffer); + int descIndex = VarInt.getVarInt(buffer) - 1; + if (descIndex == -1) { + String desc = in.readUTF(); + descIndex = info.descriptionList.size(); + info.descriptionList.add(desc); + } + ProfilerTask type = typeList.get(buffer.get()); + byte[] stats = null; + if (buffer.hasRemaining()) { + // Copy aggregated stats. + int offset = buffer.position(); + stats = Arrays.copyOfRange(backingArray, offset, size); + if (hasUnknownTypes) { + while (buffer.hasRemaining()) { + byte attrType = buffer.get(); + if (typeList.get(attrType) == ProfilerTask.UNKNOWN) { + // We're dealing with unknown aggregated type - update stats array to + // use ProfilerTask.UNKNOWN.ordinal() value. + stats[buffer.position() - 1 - offset] = (byte) ProfilerTask.UNKNOWN.ordinal(); + } + VarInt.getVarInt(buffer); + VarInt.getVarLong(buffer); + } + } + } + ProfileInfo.Task task = info.new Task(threadId, id, parentId, startTime, duration, type, + descIndex, new CompactStatistics(stats)); + info.addTask(task); + } + } catch (IOException e) { + info.corruptedOrIncomplete = true; + } finally { + in.close(); + } + + return info; + } + + /** + * Loads and parses Blaze profile file, and reports what it is doing. + * + * @param profileFile profile file path + * @param reporter for progress messages and warnings + * + * @return ProfileInfo object with most fields populated + * (call analyzeRelationships() to populate the remaining fields) + * @throws UnsupportedEncodingException if the file format is invalid + * @throws IOException if the file can't be read + */ + public static ProfileInfo loadProfileVerbosely(Path profileFile, InfoListener reporter) + throws IOException { + reporter.info("Loading " + profileFile.getPathString()); + ProfileInfo profileInfo = ProfileInfo.loadProfile(profileFile); + if (profileInfo.isCorruptedOrIncomplete()) { + reporter.warn("Profile file is incomplete or corrupted - not all records were parsed"); + } + reporter.info(profileInfo.comment + ", " + profileInfo.allTasksById.size() + " record(s)"); + return profileInfo; + } + + /* + * Sorts and aggregates Blaze profile file, and reports what it is doing. + */ + public static void aggregateProfile(ProfileInfo profileInfo, InfoListener reporter) { + reporter.info("Aggregating task statistics"); + profileInfo.calculateStats(); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfilePhase.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfilePhase.java new file mode 100644 index 0000000..fa4b862 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/ProfilePhase.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler; + +/** + * Build phase markers. Used as a separators between different build phases. + */ +public enum ProfilePhase { + LAUNCH("launch", "Launch Blaze", 0x3F9FCF9F), // 9C9 + INIT("init", "Initialize command", 0x3F9F9FCF), // 99C + LOAD("loading", "Load packages", 0x3FCFFFCF), // CFC + ANALYZE("analysis", "Analyze dependencies", 0x3FCFCFFF), // CCF + LICENSE("license checking", "Analyze licenses", 0x3FCFFFFF), // CFF + PREPARE("preparation", "Prepare for build", 0x3FFFFFCF), // FFC + EXECUTE("execution", "Build artifacts", 0x3FFFCFCF), // FCC + FINISH("finish", "Complete build",0x3FFFCFFF); // FCF + + /** Short name for the phase */ + public final String nick; + /** Human readable description for the phase. */ + public final String description; + /** Default color of the task, when rendered in a chart. */ + public final int color; + + ProfilePhase(String nick, String description, int color) { + this.nick = nick; + this.description = description; + this.color = color; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfilePhaseStatistics.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfilePhaseStatistics.java new file mode 100644 index 0000000..f3eb525 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/ProfilePhaseStatistics.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler; + +/** + * Hold pre-formatted statistics of a profiled execution phase. + * + * TODO(bazel-team): Change String statistics into StatisticsTable[], where StatisticsTable is an + * Object with a title (can be null), header[columns] (can be null), data[rows][columns], + * alignment[columns] (left/right). + * The HtmlChartsVisitor can turn that into HTML tables, the text formatter can calculate the max + * for each column and format the text accordingly. + */ +public class ProfilePhaseStatistics { + private final String title; + private final String statistics; + + public ProfilePhaseStatistics (String title, String statistics) { + this.title = title; + this.statistics = statistics; + } + + public String getTitle(){ + return title; + } + + public String getStatistics(){ + return statistics; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/Profiler.java b/src/main/java/com/google/devtools/build/lib/profiler/Profiler.java new file mode 100644 index 0000000..d592848 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/Profiler.java
@@ -0,0 +1,871 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.profiler; + +import static com.google.devtools.build.lib.profiler.ProfilerTask.TASK_COUNT; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.VarInt; + +import java.io.BufferedOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.Queue; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; + +/** + * Blaze internal profiler. Provides facility to report various Blaze tasks and + * store them (asynchronously) in the file for future analysis. + * <p> + * Implemented as singleton so any caller should use Profiler.instance() to + * obtain reference. + * <p> + * Internally, profiler uses two data structures - ThreadLocal task stack to track + * nested tasks and single ConcurrentLinkedQueue to gather all completed tasks. + * <p> + * Also, due to the nature of the provided functionality (instrumentation of all + * Blaze components), build.lib.profiler package will be used by almost every + * other Blaze package, so special attention should be paid to avoid any + * dependencies on the rest of the Blaze code, including build.lib.util and + * build.lib.vfs. This is important because build.lib.util and build.lib.vfs + * contain Profiler invocations and any dependency on those two packages would + * create circular relationship. + * <p> + * All gathered instrumentation data will be stored in the file. Please, note, + * that while file format is described here it is considered internal and can + * change at any time. For scripting, using blaze analyze-profile --dump=raw + * would be more robust and stable solution. + * <p> + * <pre> + * Profiler file consists of the deflated stream with following overall structure: + * HEADER + * TASK_TYPE_TABLE + * TASK_RECORD... + * EOF_MARKER + * + * HEADER: + * int32: magic token (Profiler.MAGIC) + * int32: version format (Profiler.VERSION) + * string: file comment + * + * TASK_TYPE_TABLE: + * int32: number of type names below + * string... : type names. Each of the type names is assigned id according to + * their position in this table starting from 0. + * + * TASK_RECORD: + * int32 size: size of the encoded task record + * byte[size] encoded_task_record: + * varint64: thread id - as was returned by Thread.getId() + * varint32: task id - starting from 1. + * varint32: parent task id for subtasks or 0 for root tasks + * varint64: start time in ns, relative to the Profiler.start() invocation + * varint64: task duration in ns + * byte: task type id (see TASK_TYPE_TABLE) + * varint32: description string index incremented by 1 (>0) or 0 this is + * a first occurrence of the description string + * AGGREGATED_STAT...: remainder of the field (if present) represents + * aggregated stats for that task + * string: *optional* description string, will appear only if description + * string index above was 0. In that case this string will be + * assigned next sequential id so every unique description string + * will appear in the file only once - after that it will be + * referenced by id. + * + * AGGREGATE_STAT: + * byte: stat type + * varint32: total number of subtask invocations + * varint64: cumulative duration of subtask invocations in ns. + * + * EOF_MARKER: + * int64: -1 - please note that this corresponds to the thread id in the + * TASK_RECORD which is always > 0 + * </pre> + * + * @see ProfilerTask enum for recognized task types. + */ +//@ThreadSafe - commented out to avoid cyclic dependency with lib.util package +public final class Profiler { + static final int MAGIC = 0x11223344; + + // File version number. Note that merely adding new record types in + // the ProfilerTask does not require bumping version number as long as original + // enum values are not renamed or deleted. + static final int VERSION = 0x03; + + // EOF marker. Must be < 0. + static final int EOF_MARKER = -1; + + // Profiler will check for gathered data and persist all of it in the + // separate thread every SAVE_DELAY ms. + private static final int SAVE_DELAY = 2000; // ms + + /** + * The profiler (a static singleton instance). Inactive by default. + */ + private static final Profiler instance = new Profiler(); + + /** + * A task that was very slow. + */ + public final class SlowTask implements Comparable<SlowTask> { + final long durationNanos; + final Object object; + ProfilerTask type; + + private SlowTask(TaskData taskData) { + this.durationNanos = taskData.duration; + this.object = taskData.object; + this.type = taskData.type; + } + + @Override + public int compareTo(SlowTask other) { + long delta = durationNanos - other.durationNanos; + if (delta < 0) { // Very clumsy + return -1; + } else if (delta > 0) { + return 1; + } else { + return 0; + } + } + + public long getDurationNanos() { + return durationNanos; + } + + public String getDescription() { + return toDescription(object); + } + + public ProfilerTask getType() { + return type; + } + } + + /** + * Container for the single task record. + * Should never be instantiated directly - use TaskStack.create() instead. + * + * Class itself is not thread safe, but all access to it from Profiler + * methods is. + */ + //@ThreadCompatible - commented out to avoid cyclic dependency with lib.util. + private final class TaskData { + final long threadId; + final long startTime; + long duration = 0L; + final int id; + final int parentId; + int[] counts; // number of invocations per ProfilerTask type + long[] durations; // time spend in the task per ProfilerTask type + final ProfilerTask type; + final Object object; + + TaskData(long startTime, TaskData parent, + ProfilerTask eventType, Object object) { + threadId = Thread.currentThread().getId(); + counts = null; + durations = null; + id = taskId.incrementAndGet(); + parentId = (parent == null ? 0 : parent.id); + this.startTime = startTime; + this.type = eventType; + this.object = Preconditions.checkNotNull(object); + } + + /** + * Aggregates information about an *immediate* subtask. + */ + public void aggregateChild(ProfilerTask type, long duration) { + int index = type.ordinal(); + if (counts == null) { + // one entry for each ProfilerTask type + counts = new int[TASK_COUNT]; + durations = new long[TASK_COUNT]; + } + counts[index]++; + durations[index] += duration; + } + + @Override + public String toString() { + return "Thread " + threadId + ", task " + id + ", type " + type + ", " + object; + } + } + + /** + * Tracks nested tasks for each thread. + * + * java.util.ArrayDeque is the most efficient stack implementation in the + * Java Collections Framework (java.util.Stack class is older synchronized + * alternative). It is, however, used here strictly for LIFO operations. + * However, ArrayDeque is 1.6 only. For 1.5 best approach would be to utilize + * ArrayList and emulate stack using it. + */ + //@ThreadSafe - commented out to avoid cyclic dependency with lib.util. + private final class TaskStack extends ThreadLocal<List<TaskData>> { + + @Override + public List<TaskData> initialValue() { + return new ArrayList<>(); + } + + public TaskData peek() { + List<TaskData> list = get(); + if (list.isEmpty()) { + return null; + } + return list.get(list.size() - 1); + } + + public TaskData pop() { + List<TaskData> list = get(); + return list.remove(list.size() - 1); + } + + public boolean isEmpty() { + return get().isEmpty(); + } + + public void push(ProfilerTask eventType, Object object) { + get().add(create(clock.nanoTime(), eventType, object)); + } + + public TaskData create(long startTime, ProfilerTask eventType, Object object) { + return new TaskData(startTime, peek(), eventType, object); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder( + "Current task stack for thread " + Thread.currentThread().getName() + ":\n"); + List<TaskData> list = get(); + for (int i = list.size() - 1; i >= 0; i--) { + builder.append(list.get(i).toString()); + builder.append("\n"); + } + return builder.toString(); + } + } + + private static String toDescription(Object object) { + return (object instanceof Describable) + ? ((Describable) object).describe() + : object.toString(); + } + + /** + * Implements datastore for object description indices. Intended to be used + * only by the Profiler.save() method. + */ + //@ThreadCompatible - commented out to avoid cyclic dependency with lib.util. + private final class ObjectDescriber { + private Map<Object, Integer> descMap = new IdentityHashMap<>(2000); + private int indexCounter = 0; + + ObjectDescriber() { } + + int getDescriptionIndex(Object object) { + Integer index = descMap.get(object); + return (index != null) ? index : -1; + } + + String getDescription(Object object) { + String description = toDescription(object); + + Integer oldIndex = descMap.put(object, indexCounter++); + // Do not use Preconditions class below due to the rather expensive + // toString() calls used in the message. + if (oldIndex != null) { + throw new IllegalStateException(" Object '" + description + "' @ " + + System.identityHashCode(object) + " already had description index " + + oldIndex + " while assigning index " + descMap.get(object)); + } else if (description.length() > 20000) { + // Note size 64k byte limitation in DataOutputStream#writeUTF(). + description = description.substring(0, 20000); + } + return description; + } + + boolean isUnassigned(int index) { + return (index < 0); + } + } + + /** + * Aggregator class that keeps track of the slowest tasks of the specified type. + * + * <p><code>priorityQueues</p> is sharded so that all threads need not compete for the same + * lock if they do the same operation at the same time. Access to the individual queues is + * synchronized on the queue objects themselves. + */ + private final class SlowestTaskAggregator { + private static final int SHARDS = 16; + private final int size; + + @SuppressWarnings({"unchecked", "rawtypes"}) + private final PriorityQueue<SlowTask>[] priorityQueues = new PriorityQueue[SHARDS]; + + SlowestTaskAggregator(int size) { + this.size = size; + + for (int i = 0; i < SHARDS; i++) { + priorityQueues[i] = new PriorityQueue<SlowTask>(size + 1); + } + } + + // @ThreadSafe + void add(TaskData taskData) { + PriorityQueue<SlowTask> queue = + priorityQueues[(int) (Thread.currentThread().getId() % SHARDS)]; + synchronized (queue) { + if (queue.size() == size) { + // Optimization: check if we are faster than the fastest element. If we are, we would + // be the ones to fall off the end of the queue, therefore, we can safely return early. + if (queue.peek().getDurationNanos() > taskData.duration) { + return; + } + + queue.add(new SlowTask(taskData)); + queue.remove(); + } else { + queue.add(new SlowTask(taskData)); + } + } + } + + // @ThreadSafe + void clear() { + for (int i = 0; i < SHARDS; i++) { + PriorityQueue<SlowTask> queue = priorityQueues[i]; + synchronized (queue) { + queue.clear(); + } + } + } + + // @ThreadSafe + Iterable<SlowTask> getSlowestTasks() { + // This is slow, but since it only happens during the end of the invocation, it's OK + PriorityQueue<SlowTask> merged = new PriorityQueue<>(size * SHARDS); + for (int i = 0; i < SHARDS; i++) { + PriorityQueue<SlowTask> queue = priorityQueues[i]; + synchronized (queue) { + merged.addAll(queue); + } + } + + while (merged.size() > size) { + merged.remove(); + } + + return merged; + } + } + + /** + * Which {@link ProfilerTask}s are profiled. + */ + public enum ProfiledTaskKinds { + /** + * Do not profile anything. + * + * <p>Performance is best with this case, but we lose critical path analysis and slowest + * operation tracking. + */ + NONE { + @Override + boolean isProfiling(ProfilerTask type) { + return false; + } + }, + + /** + * Profile on a few, known-to-be-slow tasks. + * + * <p>Performance is somewhat decreased in comparison to {@link #NONE}, but we still track the + * slowest operations (VFS). + */ + SLOWEST { + @Override + boolean isProfiling(ProfilerTask type) { + return type.collectsSlowestInstances(); + } + }, + + /** + * Profile all tasks. + * + * <p>This is in use when {@code --profile} is specified. + */ + ALL { + @Override + boolean isProfiling(ProfilerTask type) { + return true; + } + }; + + /** Whether the Profiler collects data for the given task type. */ + abstract boolean isProfiling(ProfilerTask type); + } + + private Clock clock; + private ProfiledTaskKinds profiledTaskKinds; + private volatile long profileStartTime = 0L; + private volatile boolean recordAllDurations = false; + private AtomicInteger taskId = new AtomicInteger(); + + private TaskStack taskStack; + private Queue<TaskData> taskQueue; + private DataOutputStream out; + private Timer timer; + private IOException saveException; + private ObjectDescriber describer; + @SuppressWarnings("unchecked") + private final SlowestTaskAggregator[] slowestTasks = + new SlowestTaskAggregator[ProfilerTask.values().length]; + + private Profiler() { + for (ProfilerTask task : ProfilerTask.values()) { + if (task.slowestInstancesCount != 0) { + slowestTasks[task.ordinal()] = new SlowestTaskAggregator(task.slowestInstancesCount); + } + } + } + + public static Profiler instance() { + return instance; + } + + /** + * Returns the nanoTime of the current profiler instance, or an arbitrary + * constant if not active. + */ + public static long nanoTimeMaybe() { + if (instance.isActive()) { + return instance.clock.nanoTime(); + } + return -1; + } + + /** + * Enable profiling. + * + * <p>Subsequent calls to beginTask/endTask will be recorded + * in the provided output stream. Please note that stream performance is + * extremely important and buffered streams should be utilized. + * + * @param profiledTaskKinds which kinds of {@link ProfilerTask}s to track + * @param stream output stream to store profile data. Note: passing unbuffered stream object + * reference may result in significant performance penalties + * @param comment a comment to insert in the profile data + * @param recordAllDurations iff true, record all tasks regardless of their duration; otherwise + * some tasks may get aggregated if they finished quick enough + * @param clock a {@code BlazeClock.instance()} + * @param execStartTimeNanos execution start time in nanos obtained from {@code clock.nanoTime()} + */ + public synchronized void start(ProfiledTaskKinds profiledTaskKinds, OutputStream stream, + String comment, boolean recordAllDurations, Clock clock, long execStartTimeNanos) + throws IOException { + Preconditions.checkState(!isActive(), "Profiler already active"); + taskStack = new TaskStack(); + taskQueue = new ConcurrentLinkedQueue<>(); + describer = new ObjectDescriber(); + + this.profiledTaskKinds = profiledTaskKinds; + this.clock = clock; + + // sanity check for current limitation on the number of supported types due + // to using enum.ordinal() to store them instead of EnumSet for performance reasons. + Preconditions.checkState(TASK_COUNT < 256, + "The profiler implementation supports only up to 255 different ProfilerTask values."); + + // reset state for the new profiling session + taskId.set(0); + this.recordAllDurations = recordAllDurations; + this.saveException = null; + if (stream != null) { + this.timer = new Timer("ProfilerTimer", true); + // Wrapping deflater stream in the buffered stream proved to reduce CPU consumption caused by + // the save() method. Values for buffer sizes were chosen by running small amount of tests + // and identifying point of diminishing returns - but I have not really tried to optimize + // them. + this.out = new DataOutputStream(new BufferedOutputStream(new DeflaterOutputStream( + stream, new Deflater(Deflater.BEST_SPEED, false), 65536), 262144)); + + this.out.writeInt(MAGIC); // magic + this.out.writeInt(VERSION); // protocol_version + this.out.writeUTF(comment); + // ProfileTask.values() method sorts enums using their ordinal() value, so + // there there is no need to store ordinal() value for each entry. + this.out.writeInt(TASK_COUNT); + for (ProfilerTask type : ProfilerTask.values()) { + this.out.writeUTF(type.toString()); + } + + // Start save thread + timer.schedule(new TimerTask() { + @Override public void run() { save(); } + }, SAVE_DELAY, SAVE_DELAY); + } else { + this.out = null; + } + + // activate profiler + profileStartTime = execStartTimeNanos; + } + + public synchronized Iterable<SlowTask> getSlowestTasks() { + List<Iterable<SlowTask>> slowestTasksByType = new ArrayList<>(); + + for (SlowestTaskAggregator aggregator : slowestTasks) { + if (aggregator != null) { + slowestTasksByType.add(aggregator.getSlowestTasks()); + } + } + + return Iterables.concat(slowestTasksByType); + } + + /** + * Disable profiling and complete profile file creation. + * Subsequent calls to beginTask/endTask will no longer + * be recorded in the profile. + */ + public synchronized void stop() throws IOException { + if (saveException != null) { + throw saveException; + } + if (!isActive()) { + return; + } + // Log a final event to update the duration of ProfilePhase.FINISH. + logEvent(ProfilerTask.INFO, "Finishing"); + save(); + clear(); + + for (SlowestTaskAggregator aggregator : slowestTasks) { + if (aggregator != null) { + aggregator.clear(); + } + } + + if (saveException != null) { + throw saveException; + } + if (out != null) { + out.writeInt(EOF_MARKER); + out.close(); + out = null; + } + } + + /** + * Returns true iff profiling is currently enabled. + */ + public boolean isActive() { + return profileStartTime != 0L; + } + + public boolean isProfiling(ProfilerTask type) { + return profiledTaskKinds.isProfiling(type); + } + + /** + * Saves all gathered information from taskQueue queue to the file. + * Method is invoked internally by the Timer-based thread and at the end of + * profiling session. + */ + private synchronized void save() { + if (out == null) { + return; + } + try { + // Allocate the sink once to avoid GC + ByteBuffer sink = ByteBuffer.allocate(1024); + while (!taskQueue.isEmpty()) { + sink.clear(); + TaskData data = taskQueue.poll(); + + VarInt.putVarLong(data.threadId, sink); + VarInt.putVarInt(data.id, sink); + VarInt.putVarInt(data.parentId, sink); + VarInt.putVarLong(data.startTime - profileStartTime, sink); + VarInt.putVarLong(data.duration, sink); + + // To save space (and improve performance), convert all description + // strings to the canonical object and use IdentityHashMap to assign + // unique numbers for each string. + int descIndex = describer.getDescriptionIndex(data.object); + VarInt.putVarInt(descIndex + 1, sink); // Add 1 to avoid encoding negative values. + + // Save types using their ordinal() value + sink.put((byte) data.type.ordinal()); + + // Save aggregated data stats. + if (data.counts != null) { + for (int i = 0; i < TASK_COUNT; i++) { + if (data.counts[i] > 0) { + sink.put((byte) i); // aggregated type ordinal value + VarInt.putVarInt(data.counts[i], sink); + VarInt.putVarLong(data.durations[i], sink); + } + } + } + + this.out.writeInt(sink.position()); + this.out.write(sink.array(), 0, sink.position()); + if (describer.isUnassigned(descIndex)) { + this.out.writeUTF(describer.getDescription(data.object)); + } + } + this.out.flush(); + } catch (IOException e) { + saveException = e; + clear(); + try { + out.close(); + } catch (IOException e2) { + // ignore it + } + } + } + + private synchronized void clear() { + profileStartTime = 0L; + if (timer != null) { + timer.cancel(); + timer = null; + } + taskStack = null; + taskQueue = null; + describer = null; + + // Note that slowest task aggregator are not cleared here because clearing happens + // periodically over the course of a command invocation. + } + + /** + * Unless --record_full_profiler_data is given we drop small tasks and add their time to the + * parents duration. + */ + private boolean wasTaskSlowEnoughToRecord(ProfilerTask type, long duration) { + return (recordAllDurations || duration >= type.minDuration); + } + + /** + * Adds task directly to the main queue bypassing task stack. Used for simple + * tasks that are known to not have any subtasks. + * + * @param startTime task start time (obtained through {@link Profiler#nanoTimeMaybe()}) + * @param duration task duration + * @param type task type + * @param object object associated with that task. Can be String object that + * describes it. + */ + private void logTask(long startTime, long duration, ProfilerTask type, Object object) { + Preconditions.checkNotNull(object); + Preconditions.checkState(startTime > 0, "startTime was " + startTime); + if (duration < 0) { + // See note in Clock#nanoTime, which is used by Profiler#nanoTimeMaybe. + duration = 0; + } + + TaskData parent = taskStack.peek(); + if (parent != null) { + parent.aggregateChild(type, duration); + } + if (wasTaskSlowEnoughToRecord(type, duration)) { + TaskData data = taskStack.create(startTime, type, object); + data.duration = duration; + if (out != null) { + taskQueue.add(data); + } + + SlowestTaskAggregator aggregator = slowestTasks[type.ordinal()]; + + if (aggregator != null) { + aggregator.add(data); + } + } + } + + /** + * Used externally to submit simple task (one that does not have any subtasks). + * Depending on the minDuration attribute of the task type, task may be + * just aggregated into the parent task and not stored directly. + * + * @param startTime task start time (obtained through {@link + * Profiler#nanoTimeMaybe()}) + * @param type task type + * @param object object associated with that task. Can be String object that + * describes it. + */ + public void logSimpleTask(long startTime, ProfilerTask type, Object object) { + if (isActive() && isProfiling(type)) { + logTask(startTime, clock.nanoTime() - startTime, type, object); + } + } + + /** + * Used externally to submit simple task (one that does not have any + * subtasks). Depending on the minDuration attribute of the task type, task + * may be just aggregated into the parent task and not stored directly. + * + * <p>Note that start and stop time must both be acquired from the same clock + * instance. + * + * @param startTime task start time + * @param stopTime task stop time + * @param type task type + * @param object object associated with that task. Can be String object that + * describes it. + */ + public void logSimpleTask(long startTime, long stopTime, ProfilerTask type, Object object) { + if (isActive() && isProfiling(type)) { + logTask(startTime, stopTime - startTime, type, object); + } + } + + /** + * Used externally to submit simple task (one that does not have any + * subtasks). Depending on the minDuration attribute of the task type, task + * may be just aggregated into the parent task and not stored directly. + * + * @param startTime task start time (obtained through {@link + * Profiler#nanoTimeMaybe()}) + * @param duration the duration of the task + * @param type task type + * @param object object associated with that task. Can be String object that + * describes it. + */ + public void logSimpleTaskDuration(long startTime, long duration, ProfilerTask type, + Object object) { + if (isActive() && isProfiling(type)) { + logTask(startTime, duration, type, object); + } + } + + /** + * Used to log "events" - tasks with zero duration. + */ + public void logEvent(ProfilerTask type, Object object) { + if (isActive() && isProfiling(type)) { + logTask(clock.nanoTime(), 0, type, object); + } + } + + /** + * Records the beginning of the task specified by the parameters. This method + * should always be followed by completeTask() invocation to mark the end of + * task execution (usually ensured by try {} finally {} block). Failure to do + * so will result in task stack corruption. + * + * Use of this method allows to support nested task monitoring. For tasks that + * are known to not have any subtasks, logSimpleTask() should be used instead. + * + * @param type predefined task type - see ProfilerTask for available types. + * @param object object associated with that task. Can be String object that + * describes it. + */ + public void startTask(ProfilerTask type, Object object) { + // ProfilerInfo.allTasksById is supposed to be an id -> Task map, but it is in fact a List, + // which means that we cannot drop tasks to which we had already assigned ids. Therefore, + // non-leaf tasks must not have a minimum duration. However, we don't quite consistently + // enforce this, and Blaze only works because we happen not to add child tasks to those parent + // tasks that have a minimum duration. + Preconditions.checkNotNull(object); + if (isActive() && isProfiling(type)) { + taskStack.push(type, object); + } + } + + /** + * Records the end of the task and moves tasks from the thread-local stack to + * the main queue. Will validate that given task type matches task at the top + * of the stack. + * + * @param type task type. + */ + public void completeTask(ProfilerTask type) { + if (isActive() && isProfiling(type)) { + long endTime = clock.nanoTime(); + TaskData data = taskStack.pop(); + // Do not use Preconditions class below due to the very expensive + // toString() calls used in the message. + if (data.type != type) { + throw new IllegalStateException("Inconsistent Profiler.completeTask() call for the " + + type + " task.\n " + taskStack); + } + data.duration = endTime - data.startTime; + if (data.parentId > 0) { + taskStack.peek().aggregateChild(data.type, data.duration); + } + boolean shouldRecordTask = wasTaskSlowEnoughToRecord(type, data.duration); + if (out != null && (shouldRecordTask || data.counts != null)) { + taskQueue.add(data); + } + + if (shouldRecordTask) { + SlowestTaskAggregator aggregator = slowestTasks[type.ordinal()]; + + if (aggregator != null) { + aggregator.add(data); + } + } + } + } + + /** + * Convenience method to log phase marker tasks. + */ + public void markPhase(ProfilePhase phase) { + MemoryProfiler.instance().markPhase(phase); + if (isActive() && isProfiling(ProfilerTask.PHASE)) { + Preconditions.checkState(taskStack.isEmpty(), "Phase tasks must not be nested"); + logEvent(ProfilerTask.PHASE, phase.description); + } + } + + /** + * Convenience method to log spawn tasks. + * + * TODO(bazel-team): Right now method expects single string of the spawn action + * as task description (usually either argv[0] or a name of the main executable + * in case of complex shell commands). Maybe it should accept Command object + * and create more user friendly description. + */ + public void logSpawn(long startTime, String arg0) { + if (isActive() && isProfiling(ProfilerTask.SPAWN)) { + logTask(startTime, clock.nanoTime() - startTime, ProfilerTask.SPAWN, arg0); + } + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java new file mode 100644 index 0000000..d06626c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java
@@ -0,0 +1,101 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.profiler; + +/** + * All possible types of profiler tasks. Each type also defines description and + * minimum duration in nanoseconds for it to be recorded as separate event and + * not just be aggregated into the parent event. + */ +public enum ProfilerTask { + /* WARNING: + * Add new Tasks at the end (before Unknown) to not break the profiles that people have created! + * The profile file format uses the ordinal() of this enumeration to identify the task. + */ + PHASE("build phase marker", -1, 0x336699, 0), + ACTION("action processing", -1, 0x666699, 0), + ACTION_BUILDER("parallel builder completion queue", -1, 0xCC3399, 0), + ACTION_SUBMIT("execution queue submission", -1, 0xCC3399, 0), + ACTION_CHECK("action dependency checking", 10000000, 0x999933, 0), + ACTION_EXECUTE("action execution", -1, 0x99CCFF, 0), + ACTION_LOCK("action resource lock", 10000000, 0xCC9933, 0), + ACTION_RELEASE("action resource release", 10000000, 0x006666, 0), + ACTION_GRAPH("action graph dependency", -1, 0x3399FF, 0), + ACTION_UPDATE("update action information", 10000000, 0x993300, 0), + ACTION_COMPLETE("complete action execution", -1, 0xCCCC99, 0), + INFO("general information", -1, 0x000066, 0), + EXCEPTION("exception", -1, 0xFFCC66, 0), + CREATE_PACKAGE("package creation", -1, 0x6699CC, 0), + PACKAGE_VALIDITY_CHECK("package validity check", -1, 0x336699, 0), + SPAWN("local process spawn", -1, 0x663366, 0), + REMOTE_EXECUTION("remote action execution", -1, 0x9999CC, 0), + LOCAL_EXECUTION("local action execution", -1, 0xCCCCCC, 0), + SCANNER("include scanner", -1, 0x669999, 0), + // 30 is a good number because the slowest items are stored in a heap, with temporarily + // one more element, and with 31 items, a heap becomes a complete binary tree + LOCAL_PARSE("Local parse to prepare for remote execution", 50000000, 0x6699CC, 30), + UPLOAD_TIME("Remote execution upload time", 50000000, 0x6699CC, 0), + PROCESS_TIME("Remote execution process wall time", 50000000, 0xF999CC, 0), + REMOTE_QUEUE("Remote execution queuing time", 50000000, 0xCC6600, 0), + REMOTE_SETUP("Remote execution setup", 50000000, 0xA999CC, 0), + FETCH("Remote execution file fetching", 50000000, 0xBB99CC, 0), + VFS_STAT("VFS stat", 10000000, 0x9999FF, 30), + VFS_DIR("VFS readdir", 10000000, 0x0066CC, 30), + VFS_LINK("VFS readlink", 10000000, 0x99CCCC, 30), + VFS_MD5("VFS md5", 10000000, 0x999999, 30), + VFS_XATTR("VFS xattr", 10000000, 0x9999DD, 30), + VFS_DELETE("VFS delete", 10000000, 0xFFCC00, 0), + VFS_OPEN("VFS open", 10000000, 0x009999, 30), + VFS_READ("VFS read", 10000000, 0x99CC33, 30), + VFS_WRITE("VFS write", 10000000, 0xFF9900, 30), + VFS_GLOB("globbing", -1, 0x999966, 30), + VFS_VMFS_STAT("VMFS stat", 10000000, 0x9999FF, 0), + VFS_VMFS_DIR("VMFS readdir", 10000000, 0x0066CC, 0), + VFS_VMFS_READ("VMFS read", 10000000, 0x99CC33, 0), + WAIT("thread wait", 5000000, 0x66CCCC, 0), + CONFIGURED_TARGET("configured target creation", -1, 0x663300, 0), + TRANSITIVE_CLOSURE("transitive closure creation", -1, 0x996600, 0), + TEST("for testing only", -1, 0x000000, 0), + SKYFRAME_EVAL("skyframe evaluator", -1, 0xCC9900, 0), + SKYFUNCTION("skyfunction", -1, 0xCC6600, 0), + CRITICAL_PATH("critical path", -1, 0x666699, 0), + CRITICAL_PATH_COMPONENT("critical path component", -1, 0x666699, 0), + IDE_BUILD_INFO("ide_build_info", -1, 0xCC6633, 0), + UNKNOWN("Unknown event", -1, 0x339966, 0); + + // Size of the ProfilerTask value space. + public static final int TASK_COUNT = ProfilerTask.values().length; + + /** Human readable description for the task. */ + public final String description; + /** Threshold for skipping tasks in the profile in nanoseconds, unless --record_full_profiler_data + * is used */ + public final long minDuration; + /** Default color of the task, when rendered in a chart. */ + public final int color; + /** How many of the slowest instances to keep. If 0, no slowest instance calculation is done. */ + public final int slowestInstancesCount; + + ProfilerTask(String description, long minDuration, int color, int slowestInstanceCount) { + this.description = description; + this.minDuration = minDuration; + this.color = color; + this.slowestInstancesCount = slowestInstanceCount; + } + + /** Whether the Profiler collects the slowest instances of this task. */ + public boolean collectsSlowestInstances() { + return slowestInstancesCount > 0; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/AggregatingChartCreator.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/AggregatingChartCreator.java new file mode 100644 index 0000000..469e605 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/AggregatingChartCreator.java
@@ -0,0 +1,161 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler.chart; + +import com.google.devtools.build.lib.profiler.ProfileInfo; +import com.google.devtools.build.lib.profiler.ProfileInfo.Task; +import com.google.devtools.build.lib.profiler.ProfilePhaseStatistics; +import com.google.devtools.build.lib.profiler.ProfilerTask; + +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +/** + * Implementation of {@link ChartCreator} that creates Gantt Charts that try to + * minimize the number of bars while preserving as much information about the + * execution of actions as possible. + * + * <p>Profiler tasks are categorized into four categories: + * <ul> + * <li>Actions: Actions executed. + * <li>Blaze Internal: This category contains internal blaze tasks, like loading + * packages, saving the action cache etc. + * <li>Locks: Contains tasks that indicate that a thread is waiting for + * resources. + * <li>VFS: Contains tasks that access the file system. + * </ul> + */ +public class AggregatingChartCreator implements ChartCreator { + + /** The tasks in the 'actions' category. */ + private static final Set<ProfilerTask> ACTION_TASKS = EnumSet.of(ProfilerTask.ACTION, + ProfilerTask.ACTION_SUBMIT); + + /** The tasks in the 'blaze internal' category. */ + private static Set<ProfilerTask> BLAZE_TASKS = + EnumSet.of(ProfilerTask.CREATE_PACKAGE, ProfilerTask.PACKAGE_VALIDITY_CHECK, + ProfilerTask.CONFIGURED_TARGET, ProfilerTask.TRANSITIVE_CLOSURE, + ProfilerTask.EXCEPTION, ProfilerTask.INFO, ProfilerTask.UNKNOWN); + + /** The tasks in the 'locks' category. */ + private static Set<ProfilerTask> LOCK_TASKS = + EnumSet.of(ProfilerTask.ACTION_LOCK, ProfilerTask.WAIT); + + /** The tasks in the 'VFS' category. */ + private static Set<ProfilerTask> VFS_TASKS = + EnumSet.of(ProfilerTask.VFS_STAT, ProfilerTask.VFS_DIR, ProfilerTask.VFS_LINK, + ProfilerTask.VFS_MD5, ProfilerTask.VFS_DELETE, ProfilerTask.VFS_OPEN, + ProfilerTask.VFS_READ, ProfilerTask.VFS_WRITE, ProfilerTask.VFS_GLOB, + ProfilerTask.VFS_XATTR); + + /** The data of the profiled build. */ + private final ProfileInfo info; + + /** + * Statistics of the profiled build. This is expected to be a formatted + * string, ready to be printed out. + */ + private final List<ProfilePhaseStatistics> statistics; + + /** If true, VFS related information is added to the chart. */ + private final boolean showVFS; + + /** The type for bars of category 'blaze internal'. */ + private ChartBarType blazeType; + + /** The type for bars of category 'actions'. */ + private ChartBarType actionType; + + /** The type for bars of category 'locks'. */ + private ChartBarType lockType; + + /** The type for bars of category 'VFS'. */ + private ChartBarType vfsType; + + /** + * Creates the chart creator. The created {@link ChartCreator} does not add + * VFS related data to the generated chart. + * + * @param info the data of the profiled build + * @param statistics Statistics of the profiled build. This is expected to be + * a formatted string, ready to be printed out. + */ + public AggregatingChartCreator(ProfileInfo info, List<ProfilePhaseStatistics> statistics) { + this(info, statistics, false); + } + + /** + * Creates the chart creator. + * + * @param info the data of the profiled build + * @param statistics Statistics of the profiled build. This is expected to be + * a formatted string, ready to be printed out. + * @param showVFS if true, VFS related information is added to the chart + */ + public AggregatingChartCreator(ProfileInfo info, List<ProfilePhaseStatistics> statistics, + boolean showVFS) { + this.info = info; + this.statistics = statistics; + this.showVFS = showVFS; + } + + @Override + public Chart create() { + Chart chart = new Chart(info.comment, statistics); + CommonChartCreator.createCommonChartItems(chart, info); + createTypes(chart); + + for (ProfileInfo.Task task : info.allTasksById) { + if (ACTION_TASKS.contains(task.type)) { + createBar(chart, task, actionType); + } else if (LOCK_TASKS.contains(task.type)) { + createBar(chart, task, lockType); + } else if (BLAZE_TASKS.contains(task.type)) { + createBar(chart, task, blazeType); + } else if (showVFS && VFS_TASKS.contains(task.type)) { + createBar(chart, task, vfsType); + } + } + + return chart; + } + + /** + * Creates a bar and adds it to the chart. + * + * @param chart the chart to add the types to + * @param task the profiler task from which the bar is created + * @param type the type of the bar + */ + private void createBar(Chart chart, Task task, ChartBarType type) { + String label = task.type.description + ": " + task.getDescription(); + chart.addBar(task.threadId, task.startTime, task.startTime + task.duration, type, label); + } + + /** + * Creates the {@link ChartBarType}s and adds them to the chart. + * + * @param chart the chart to add the types to + */ + private void createTypes(Chart chart) { + actionType = chart.createType("Action processing", new Color(0x000099)); + blazeType = chart.createType("Blaze internal processing", new Color(0x999999)); + lockType = chart.createType("Waiting for resources", new Color(0x990000)); + if (showVFS) { + vfsType = chart.createType("File system access", new Color(0x009900)); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/Chart.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/Chart.java new file mode 100644 index 0000000..93c7c81 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/Chart.java
@@ -0,0 +1,233 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler.chart; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.profiler.ProfilePhaseStatistics; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Data of a Gantt Chart to visualize the data of a profiled build. + */ +public class Chart { + + /** The type that is returned when an unknown type is looked up. */ + public static final ChartBarType UNKNOWN_TYPE = new ChartBarType("Unknown type", Color.RED); + + /** The title of the chart. */ + private final String title; + + /** Statistics of the profiled build. */ + private final List<ProfilePhaseStatistics> statistics; + + /** The rows of the chart. */ + private final Map<Long, ChartRow> rows = new HashMap<>(); + + /** The columns on the chart. */ + private final List<ChartColumn> columns = new ArrayList<>(); + + /** The lines on the chart. */ + private final List<ChartLine> lines = new ArrayList<>(); + + /** The types of the bars in the chart. */ + private final Map<String, ChartBarType> types = new HashMap<>(); + + /** The running index of the rows in the chart. */ + private int rowIndex = 0; + + /** The maximum stop value of any bar in the chart. */ + private long maxStop; + + /** + * Creates a chart. + * + * @param title the title of the chart + * @param statistics Statistics of the profiled build. This is expected to be + * a formatted string, ready to be printed out. + */ + public Chart(String title, List<ProfilePhaseStatistics> statistics) { + Preconditions.checkNotNull(title); + Preconditions.checkNotNull(statistics); + this.title = title; + this.statistics = statistics; + } + + /** + * Adds a bar to a row of the chart. If a row with the given id already + * exists, the bar is added to the row, otherwise a new row is created and the + * bar is added to it. + * + * @param id the id of the row the new bar belongs to + * @param start the start value of the bar + * @param stop the stop value of the bar + * @param type the type of the bar + * @param highlight emphasize the bar + * @param label the label of the bar + */ + public void addBar(long id, long start, long stop, ChartBarType type, boolean highlight, + String label) { + ChartRow slot = addSlotIfAbsent(id); + ChartBar bar = new ChartBar(slot, start, stop, type, highlight, label); + slot.addBar(bar); + maxStop = Math.max(maxStop, stop); + } + + /** + * Adds a bar to a row of the chart. If a row with the given id already + * exists, the bar is added to the row, otherwise a new row is created and the + * bar is added to it. + * + * @param id the id of the row the new bar belongs to + * @param start the start value of the bar + * @param stop the stop value of the bar + * @param type the type of the bar + * @param label the label of the bar + */ + public void addBar(long id, long start, long stop, ChartBarType type, String label) { + addBar(id, start, stop, type, false, label); + } + + /** + * Adds a vertical line to the chart. + */ + public void addVerticalLine(long startId, long stopId, long pos) { + ChartRow startSlot = addSlotIfAbsent(startId); + ChartRow stopSlot = addSlotIfAbsent(stopId); + ChartLine line = new ChartLine(startSlot, stopSlot, pos, pos); + lines.add(line); + } + + /** + * Adds a column to the chart. + * + * @param start the start value of the bar + * @param stop the stop value of the bar + * @param type the type of the bar + * @param label the label of the bar + */ + public void addTimeRange(long start, long stop, ChartBarType type, String label) { + ChartColumn column = new ChartColumn(start, stop, type, label); + columns.add(column); + maxStop = Math.max(maxStop, stop); + } + + /** + * Creates a new {@link ChartBarType} and adds it to the list of types of the + * chart. + * + * @param name the name of the type + * @param color the color of the chart + * @return the newly created type + */ + public ChartBarType createType(String name, Color color) { + ChartBarType type = new ChartBarType(name, color); + types.put(name, type); + return type; + } + + /** + * Returns the type with the given name. If no type with the given name + * exists, a type with name 'Unknown type' is added to the chart and returned. + * + * @param name the name of the type to look up + */ + public ChartBarType lookUpType(String name) { + ChartBarType type = types.get(name); + if (type == null) { + type = UNKNOWN_TYPE; + types.put(type.getName(), type); + } + return type; + } + + /** + * Creates a new row with the given id if no row with this id existed. + * Otherwise the existing row with the given id is returned. + * + * @param id the ID of the row + * @return the existing row, if it was already present, the newly created one + * otherwise + */ + private ChartRow addSlotIfAbsent(long id) { + ChartRow slot = rows.get(id); + if (slot == null) { + slot = new ChartRow(Long.toString(id), rowIndex++); + rows.put(id, slot); + } + return slot; + } + + /** + * Accepts a {@link ChartVisitor}. Calls {@link ChartVisitor#visit(Chart)}, + * delegates the visitor to the rows of the chart and calls + * {@link ChartVisitor#endVisit(Chart)}. + * + * @param visitor the visitor to accept + */ + public void accept(ChartVisitor visitor) { + visitor.visit(this); + for (ChartRow slot : rows.values()) { + slot.accept(visitor); + } + int rowCount = getRowCount(); + for (ChartColumn column : columns) { + column.setRowCount(rowCount); + column.accept(visitor); + } + for (ChartLine line : lines) { + line.accept(visitor); + } + visitor.endVisit(this); + } + + /** + * Returns the {@link ChartBarType}s, sorted by name. + */ + public List<ChartBarType> getSortedTypes() { + List<ChartBarType> list = new ArrayList<>(types.values()); + Collections.sort(list); + return list; + } + + /** + * Returns the {@link ChartRow}s, sorted by their index. + */ + public List<ChartRow> getSortedRows() { + List<ChartRow> list = new ArrayList<>(rows.values()); + Collections.sort(list); + return list; + } + + public String getTitle() { + return title; + } + + public List<ProfilePhaseStatistics> getStatistics() { + return statistics; + } + + public int getRowCount() { + return rows.size(); + } + + public long getMaxStop() { + return maxStop; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartBar.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartBar.java new file mode 100644 index 0000000..20d92f0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartBar.java
@@ -0,0 +1,106 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler.chart; + +import com.google.common.base.Preconditions; + +/** + * A bar in a row of a Gantt Chart. + */ +public class ChartBar { + + /** + * The start value of the bar. This value has no unit. The interpretation of + * the value is up to the user of the class. + */ + private final long start; + + /** + * The stop value of the bar. This value has no unit. The interpretation of + * the value is up to the user of the class. + */ + private final long stop; + + /** The type of the bar. */ + private final ChartBarType type; + + /** Emphasize the bar */ + private boolean highlight; + + /** The label of the bar. */ + private final String label; + + /** The chart row this bar belongs to. */ + private final ChartRow row; + + /** + * Creates a chart bar. + * + * @param row the chart row this bar belongs to + * @param start the start value of the bar + * @param stop the stop value of the bar + * @param type the type of the bar + * @param label the label of the bar + */ + public ChartBar(ChartRow row, long start, long stop, ChartBarType type, boolean highlight, + String label) { + Preconditions.checkNotNull(row); + Preconditions.checkNotNull(type); + Preconditions.checkNotNull(label); + this.row = row; + this.start = start; + this.stop = stop; + this.type = type; + this.highlight = highlight; + this.label = label; + } + + /** + * Accepts a {@link ChartVisitor}. Calls {@link ChartVisitor#visit(ChartBar)}. + * + * @param visitor the visitor to accept + */ + public void accept(ChartVisitor visitor) { + visitor.visit(this); + } + + public long getStart() { + return start; + } + + public long getStop() { + return stop; + } + + public long getWidth() { + return stop - start; + } + + public ChartBarType getType() { + return type; + } + + public boolean getHighlight() { + return highlight; + } + + public String getLabel() { + return label; + } + + public ChartRow getRow() { + return row; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartBarType.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartBarType.java new file mode 100644 index 0000000..e0f3885 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartBarType.java
@@ -0,0 +1,82 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler.chart; + +import com.google.common.base.Preconditions; + +/** + * The type of a bar in a Gantt Chart. A type consists of a name and a color. + * Types are used to create the legend of a Gantt Chart. + */ +public class ChartBarType implements Comparable<ChartBarType> { + + /** The name of the type. */ + private final String name; + + /** The color of the type. */ + private final Color color; + + /** + * Creates a {@link ChartBarType}. + * + * @param name the name of the type + * @param color the color of the type + */ + public ChartBarType(String name, Color color) { + Preconditions.checkNotNull(name); + Preconditions.checkNotNull(color); + this.name = name; + this.color = color; + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + /** + * {@inheritDoc} + * + * <p>Equality of two types is defined by the equality of their names. + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + return name.equals(((ChartBarType) obj).name); + } + + /** + * {@inheritDoc} + * + * <p>Compares types by their names. + */ + @Override + public int compareTo(ChartBarType o) { + return name.compareTo(o.name); + } + + public String getName() { + return name; + } + + public Color getColor() { + return color; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartColumn.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartColumn.java new file mode 100644 index 0000000..25fffe8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartColumn.java
@@ -0,0 +1,93 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler.chart; + +import com.google.common.base.Preconditions; + +/** + * A chart column. The column can be used to highlight a time-range. + */ +public class ChartColumn { + + /** + * The start value of the bar. This value has no unit. The interpretation of + * the value is up to the user of the class. + */ + private final long start; + + /** + * The stop value of the bar. This value has no unit. The interpretation of + * the value is up to the user of the class. + */ + private final long stop; + + /** The type of the bar. */ + private final ChartBarType type; + + /** The label of the bar. */ + private final String label; + + private int rowCount; + + /** + * Creates a chart column. + * + * @param start the start value of the bar + * @param stop the stop value of the bar + * @param type the type of the bar + * @param label the label of the bar + */ + public ChartColumn(long start, long stop, ChartBarType type, String label) { + Preconditions.checkNotNull(type); + Preconditions.checkNotNull(label); + this.start = start; + this.stop = stop; + this.type = type; + this.label = label; + } + + /** + * Accepts a {@link ChartVisitor}. Calls {@link ChartVisitor#visit(ChartBar)}. + * + * @param visitor the visitor to accept + */ + public void accept(ChartVisitor visitor) { + visitor.visit(this); + } + + public long getStart() { + return start; + } + + public long getWidth() { + return stop - start; + } + + public ChartBarType getType() { + return type; + } + + public String getLabel() { + return label; + } + + public int getRowCount() { + return rowCount; + } + + public void setRowCount(int rowCount) { + this.rowCount = rowCount; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartCreator.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartCreator.java new file mode 100644 index 0000000..31781c8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartCreator.java
@@ -0,0 +1,28 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler.chart; + +import com.google.devtools.build.lib.profiler.chart.Chart; + +/** + * Interface for classes that are capable of creating {@link Chart}s. + */ +public interface ChartCreator { + + /** + * Creates a {@link Chart}. + */ + Chart create(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartLine.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartLine.java new file mode 100644 index 0000000..d9bd755 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartLine.java
@@ -0,0 +1,62 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler.chart; + +import com.google.common.base.Preconditions; + +/** + * A chart line. Such lines can be used to connect boxes. + */ +public class ChartLine { + private final ChartRow startRow; + private final ChartRow stopRow; + private final long startTime; + /** + * Creates a chart line. + * + * @param startRow the start row + * @param stopRow the end row + * @param startTime the start time + * @param stopTime the end time + */ + public ChartLine(ChartRow startRow, ChartRow stopRow, long startTime, long stopTime) { + Preconditions.checkNotNull(startRow); + Preconditions.checkNotNull(stopRow); + this.startRow = startRow; + this.stopRow = stopRow; + this.startTime = startTime; + } + + /** + * Accepts a {@link ChartVisitor}. Calls {@link ChartVisitor#visit(ChartBar)}. + * + * @param visitor the visitor to accept + */ + public void accept(ChartVisitor visitor) { + visitor.visit(this); + } + + public ChartRow getStartRow() { + return startRow; + } + + public ChartRow getStopRow() { + return stopRow; + } + + public long getStartTime() { + return startTime; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartRow.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartRow.java new file mode 100644 index 0000000..96597c9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartRow.java
@@ -0,0 +1,96 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler.chart; + +import com.google.common.base.Preconditions; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * A row of a Gantt Chart. A chart row is identified by its id and has an index that + * determines its location in the chart. + */ +public class ChartRow implements Comparable<ChartRow> { + + /** The unique id of this row. */ + private final String id; + + /** The index, i.e., the row number of the row in the chart. */ + private final int index; + + /** The list of bars in this row. */ + private final List<ChartBar> bars = new ArrayList<>(); + + /** + * Creates a chart row. + * + * @param id the unique id of this row + * @param index the index, i.e., the row number, of the row in the chart + */ + public ChartRow(String id, int index) { + Preconditions.checkNotNull(id); + this.id = id; + this.index = index; + } + + /** + * Adds a bar to the chart row. + * + * @param bar the {@link ChartBar} to add + */ + public void addBar(ChartBar bar) { + bars.add(bar); + } + + /** + * Returns the bars of the row as an unmodifieable list. + */ + public List<ChartBar> getBars() { + return Collections.unmodifiableList(bars); + } + + /** + * Accepts a {@link ChartVisitor}. Calls {@link ChartVisitor#visit(ChartRow)} + * and delegates the visitor to the bars of the chart row. + * + * @param visitor the visitor to accept + */ + public void accept(ChartVisitor visitor) { + visitor.visit(this); + for (ChartBar bar : bars) { + bar.accept(visitor); + } + } + + /** + * {@inheritDoc} + * + * <p>Compares to rows by their index. + */ + @Override + public int compareTo(ChartRow other) { + return index - other.index; + } + + public int getIndex() { + return index; + } + + public String getId() { + return id; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartVisitor.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartVisitor.java new file mode 100644 index 0000000..b06b3ab --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/ChartVisitor.java
@@ -0,0 +1,65 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler.chart; + +/** + * Visitor for {@link Chart} objects. + */ +public interface ChartVisitor { + + /** + * Visits a {@link Chart} object before its children, i.e., rows and bars, are + * visited. + * + * @param chart the {@link Chart} to visit + */ + void visit(Chart chart); + + /** + * Visits a {@link Chart} object after its children, i.e., rows and bars, are + * visited. + * + * @param chart the {@link Chart} to visit + */ + void endVisit(Chart chart); + + /** + * Visits a {@link ChartRow} object. + * + * @param chartRow the {@link ChartRow} to visit + */ + void visit(ChartRow chartRow); + + /** + * Visits a {@link ChartBar} object. + * + * @param chartBar the {@link ChartBar} to visit + */ + void visit(ChartBar chartBar); + + /** + * Visits a {@link ChartColumn} object. + * + * @param chartColumn the {@link ChartColumn} to visit + */ + void visit(ChartColumn chartColumn); + + /** + * Visits a {@link ChartLine} object. + * + * @param chartLine the {@link ChartLine} to visit + */ + void visit(ChartLine chartLine); +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/Color.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/Color.java new file mode 100644 index 0000000..d527093 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/Color.java
@@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler.chart; + +/** + * Represents a color in ARGB format, 8 bits per channel. + */ +public final class Color { + public static final Color RED = new Color(0xff0000); + public static final Color GREEN = new Color(0x00ff00); + public static final Color GRAY = new Color(0x808080); + public static final Color BLACK = new Color(0x000000); + + private final int argb; + + public Color(int rgb) { + this.argb = rgb | 0xff000000; + } + + public Color(int argb, boolean hasAlpha) { + this.argb = argb; + } + + public int getRed() { + return (argb >> 16) & 0xFF; + } + + public int getGreen() { + return (argb >> 8) & 0xFF; + } + + public int getBlue() { + return argb & 0xFF; + } + + public int getAlpha() { + return (argb >> 24) & 0xFF; + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/CommonChartCreator.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/CommonChartCreator.java new file mode 100644 index 0000000..ed1da20 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/CommonChartCreator.java
@@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler.chart; + +import com.google.devtools.build.lib.profiler.ProfileInfo; +import com.google.devtools.build.lib.profiler.ProfilePhase; + +/** + * Provides some common functions for {@link ChartCreator}s. + */ +public final class CommonChartCreator { + + static void createCommonChartItems(Chart chart, ProfileInfo info) { + createTypes(chart); + + // add common info + for (ProfilePhase phase : ProfilePhase.values()) { + addColumn(chart,info,phase); + } + } + + private static void addColumn(Chart chart, ProfileInfo info, ProfilePhase phase) { + ProfileInfo.Task task = info.getPhaseTask(phase); + if (task != null) { + String label = task.type.description + ": " + task.getDescription(); + ChartBarType type = chart.lookUpType(task.getDescription()); + long stop = task.startTime + info.getPhaseDuration(task); + chart.addTimeRange(task.startTime, stop, type, label); + } + } + + /** + * Creates the {@link ChartBarType}s and adds them to the chart. + */ + private static void createTypes(Chart chart) { + for (ProfilePhase phase : ProfilePhase.values()) { + chart.createType(phase.description, new Color(phase.color, true)); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/DetailedChartCreator.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/DetailedChartCreator.java new file mode 100644 index 0000000..1e097c3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/DetailedChartCreator.java
@@ -0,0 +1,103 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler.chart; + +import com.google.devtools.build.lib.profiler.ProfileInfo; +import com.google.devtools.build.lib.profiler.ProfileInfo.CriticalPathEntry; +import com.google.devtools.build.lib.profiler.ProfileInfo.Task; +import com.google.devtools.build.lib.profiler.ProfilePhaseStatistics; +import com.google.devtools.build.lib.profiler.ProfilerTask; + +import java.util.EnumSet; +import java.util.List; + +/** + * Implementation of {@link ChartCreator} that creates Gantt Charts that contain + * bars for all tasks in the profile. + */ +public class DetailedChartCreator implements ChartCreator { + + /** The data of the profiled build. */ + private final ProfileInfo info; + + /** + * Statistics of the profiled build. This is expected to be a formatted + * string, ready to be printed out. + */ + private final List<ProfilePhaseStatistics> statistics; + + /** + * Creates the chart creator. + * + * @param info the data of the profiled build + * @param statistics Statistics of the profiled build. This is expected to be + * a formatted string, ready to be printed out. + */ + public DetailedChartCreator(ProfileInfo info, List<ProfilePhaseStatistics> statistics) { + this.info = info; + this.statistics = statistics; + } + + @Override + public Chart create() { + Chart chart = new Chart(info.comment, statistics); + CommonChartCreator.createCommonChartItems(chart, info); + createTypes(chart); + + // calculate the critical path + EnumSet<ProfilerTask> typeFilter = EnumSet.noneOf(ProfilerTask.class); + CriticalPathEntry criticalPath = info.getCriticalPath(typeFilter); + info.analyzeCriticalPath(typeFilter, criticalPath); + + for (Task task : info.allTasksById) { + String label = task.type.description + ": " + task.getDescription(); + ChartBarType type = chart.lookUpType(task.type.description); + long stop = task.startTime + task.duration; + CriticalPathEntry entry = null; + + // for top level tasks, check if they are on the critical path + if (task.parentId == 0 && criticalPath != null) { + entry = info.getNextCriticalPathEntryForTask(criticalPath, task); + // find next top-level entry + if (entry != null) { + CriticalPathEntry nextEntry = entry.next; + while (nextEntry != null && nextEntry.task.parentId != 0) { + nextEntry = nextEntry.next; + } + if (nextEntry != null) { + // time is start and not stop as we traverse the critical back backwards + chart.addVerticalLine(task.threadId, nextEntry.task.threadId, task.startTime); + } + } + } + + chart.addBar(task.threadId, task.startTime, stop, type, (entry != null), label); + } + + return chart; + } + + /** + * Creates a {@link ChartBarType} for every known {@link ProfilerTask} and + * adds it to the chart. + * + * @param chart the chart to add the types to + */ + private void createTypes(Chart chart) { + for (ProfilerTask task : ProfilerTask.values()) { + chart.createType(task.description, new Color(task.color)); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/profiler/chart/HtmlChartVisitor.java b/src/main/java/com/google/devtools/build/lib/profiler/chart/HtmlChartVisitor.java new file mode 100644 index 0000000..8fa3d0e4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/profiler/chart/HtmlChartVisitor.java
@@ -0,0 +1,368 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.profiler.chart; + +import com.google.devtools.build.lib.profiler.ProfilePhaseStatistics; + +import java.io.PrintStream; +import java.util.List; + +/** + * {@link ChartVisitor} that builds HTML from the visited chart and prints it + * out to the given {@link PrintStream}. + */ +public class HtmlChartVisitor implements ChartVisitor { + + /** The default width of a second in the chart. */ + private static final int DEFAULT_PIXEL_PER_SECOND = 50; + + /** The horizontal offset of second zero. */ + private static final int H_OFFSET = 40; + + /** The font size of the row labels. */ + private static final int ROW_LABEL_FONT_SIZE = 7; + + /** The height of a bar in pixels. */ + private static final int BAR_HEIGHT = 8; + + /** The space between twp bars in pixels. */ + private static final int BAR_SPACE = 2; + + /** The height of a row. */ + private static final int ROW_HEIGHT = BAR_HEIGHT + BAR_SPACE; + + /** The {@link PrintStream} to output the HTML to. */ + private final PrintStream out; + + /** The maxmimum stop time of any bar in the chart. */ + private long maxStop; + + /** The width of a second in the chart. */ + private final int pixelsPerSecond; + + /** + * Creates the visitor, with a default width of a second of 50 pixels. + * + * @param out the {@link PrintStream} to output the HTML to + */ + public HtmlChartVisitor(PrintStream out) { + this(out, DEFAULT_PIXEL_PER_SECOND); + } + + /** + * Creates the visitor. + * + * @param out the {@link PrintStream} to output the HTML to + * @param pixelsPerSecond The width of a second in the chart. (In pixels) + */ + public HtmlChartVisitor(PrintStream out, int pixelsPerSecond) { + this.out = out; + this.pixelsPerSecond = pixelsPerSecond; + } + + @Override + public void visit(Chart chart) { + maxStop = chart.getMaxStop(); + out.println("<html><head>"); + out.printf("<title>%s</title>", chart.getTitle()); + out.println("<style type=\"text/css\"><!--"); + + printCss(chart.getSortedTypes()); + + out.println("--></style>"); + out.println("</head>"); + out.println("<body>"); + + heading(chart.getTitle(), 1); + + printContentBox(); + + heading("Tasks", 2); + out.println("<p>To get more information about a task point the mouse at one of the bars.</p>"); + + out.printf("<div style='position:relative; height: %dpx; margin: %dpx'>\n", + chart.getRowCount() * ROW_HEIGHT, H_OFFSET + 10); + } + + @Override + public void endVisit(Chart chart) { + printTimeAxis(chart); + out.println("</div>"); + + heading("Legend", 2); + printLegend(chart.getSortedTypes()); + + heading("Statistics", 2); + printStatistics(chart.getStatistics()); + + out.println("</body>"); + out.println("</html>"); +} + + @Override + public void visit(ChartColumn column) { + int width = scale(column.getWidth()); + if (width == 0) { + return; + } + int left = scale(column.getStart()); + int height = column.getRowCount() * ROW_HEIGHT; + String style = chartTypeNameAsCSSClass(column.getType().getName()); + box(left, 0, width, height, style, column.getLabel(), 10); + } + + + @Override + public void visit(ChartRow slot) { + String style = slot.getIndex() % 2 == 0 ? "shade-even" : "shade-odd"; + int top = slot.getIndex() * ROW_HEIGHT; + int width = scale(maxStop) + 1; + + label(-H_OFFSET, top, width + H_OFFSET, ROW_HEIGHT, ROW_LABEL_FONT_SIZE, slot.getId()); + box(0, top, width, ROW_HEIGHT, style, "", 0); + } + + @Override + public void visit(ChartBar bar) { + int width = scale(bar.getWidth()); + if (width == 0) { + return; + } + int left = scale(bar.getStart()); + int top = bar.getRow().getIndex() * ROW_HEIGHT; + String style = chartTypeNameAsCSSClass(bar.getType().getName()); + if (bar.getHighlight()) { + style += "-highlight"; + } + box(left, top + 2, width, BAR_HEIGHT, style, bar.getLabel(), 20); + } + + @Override + public void visit(ChartLine chartLine) { + int start = chartLine.getStartRow().getIndex() * ROW_HEIGHT; + int stop = chartLine.getStopRow().getIndex() * ROW_HEIGHT; + int time = scale(chartLine.getStartTime()); + + if (start < stop) { + verticalLine(time, start + 1, 1, (stop - start) + ROW_HEIGHT, Color.RED); + } else { + verticalLine(time, stop + 1, 1, (start - stop) + ROW_HEIGHT, Color.RED); + } + } + + /** + * Converts the given value from the bar of the chart to pixels. + */ + private int scale(long value) { + return (int) (value / (1000000000L / pixelsPerSecond)); + } + + /** + * Prints a box with links to the sections of the generated HTML document. + */ + private void printContentBox() { + out.println("<div style='position:fixed; top:1em; right:1em; z-index:50; padding: 1ex;" + + "border:1px solid #888; background-color:#eee; width:100px'><h3>Content</h3>"); + out.println("<p style='text-align:left;font-size:small;margin:2px'>" + + "<a href='#Tasks'>Tasks</a></p>"); + out.println("<p style='text-align:left;font-size:small;margin:2px'>" + + "<a href='#Legend'>Legend</a></p>"); + out.println("<p style='text-align:left;font-size:small;margin:2px'>" + + "<a href='#Statistics'>Statistics</a></p></div>"); + } + + /** + * Prints the time axis of the chart and vertical lines for every second. + */ + private void printTimeAxis(Chart chart) { + int location = 0; + int second = 0; + int end = scale(chart.getMaxStop()); + while (location < end) { + label(location + 4, -17, pixelsPerSecond, ROW_HEIGHT, 0, second + "s"); + verticalLine(location, -20, 1, chart.getRowCount() * ROW_HEIGHT + 20, Color.GRAY); + location += pixelsPerSecond; + second += 1; + } + } + + private void printCss(List<ChartBarType> types) { + out.println("body { font-family: Sans; }"); + out.printf("div.shade-even { position:absolute; border: 0px; background-color:#dddddd }\n"); + out.printf("div.shade-odd { position:absolute; border: 0px; background-color:#eeeeee }\n"); + for (ChartBarType type : types) { + String name = chartTypeNameAsCSSClass(type.getName()); + String color = formatColor(type.getColor()); + + out.printf( + "div.%s-border { position:absolute; border:1px solid grey; background-color:%s }\n", + name, color); + out.printf( + "div.%s-highlight { position:absolute; border:1px solid red; background-color:%s }\n", + name, color); + out.printf("div.%s { position:absolute; border:0px; margin:1px; background-color:%s }\n", + name, color); + } + } + + /** + * Prints the legend for the chart at the current position in the document. The + * legend is printed in columns of 10 rows each. + * + * @param types the list of {@link ChartBarType}s to print in the legend. + */ + private void printLegend(List<ChartBarType> types) { + final int boxHeight = 20; + final int lineHeight = 25; + final int entriesPerColumn = 10; + final int legendWidth = 350; + int legendHeight; + if (types.size() / entriesPerColumn >= 1) { + legendHeight = entriesPerColumn; + } else { + legendHeight = types.size() % entriesPerColumn; + } + + out.printf("<div style='position:relative; height: %dpx;'>", + (legendHeight + 1) * lineHeight); + + int left = -legendWidth; + int top; + int i = 0; + for (ChartBarType type : types) { + if (i % entriesPerColumn == 0) { + left += legendWidth; + i = 0; + } + top = lineHeight * i; + String style = chartTypeNameAsCSSClass(type.getName()) + "-border"; + box(left, top, boxHeight, boxHeight, style, type.getName(), 0); + label(left + lineHeight + 10, top, legendWidth - 10, boxHeight, 0, type.getName()); + i++; + } + out.println("</div>"); + } + + private void printStatistics(List<ProfilePhaseStatistics> statistics) { + boolean first = true; + + out.println("<table border=\"0\" width=\"100%\"><tr>"); + for (ProfilePhaseStatistics stat : statistics) { + if (!first) { + out.println("<td><div style=\"width:20px;\"> </div></td>"); + } else { + first = false; + } + out.println("<td valign=\"top\">"); + String title = stat.getTitle(); + if (title != "") { + heading(title, 3); + } + out.println("<pre>" + stat.getStatistics() + "</pre></td>"); + } + out.println("</tr></table>"); + } + + /** + * Prints a head-line at the current position in the document. + * + * @param text the text to print + * @param level the headline level + */ + private void heading(String text, int level) { + anchor(text); + out.printf("<h%d >%s</h%d>\n", level, text, level); + } + + /** + * Prints a box with the given location, size, background color and border. + * + * @param x the x location of the top left corner of the box + * @param y the y location of the top left corner of the box + * @param width the width location of the box + * @param height the height location of the box + * @param style the CSS style class to use for the box + * @param title the text displayed when the mouse hovers over the box + */ + private void box(int x, int y, int width, int height, String style, String title, int zIndex) { + out.printf("<div class=\"%s\" title=\"%s\" " + + "style=\"left:%dpx; top:%dpx; width:%dpx; height:%dpx; z-index:%d\"></div>\n", + style, title, x, y, width, height, zIndex); + } + + /** + * Prints a label with the given location, size, background color and border. + * + * @param x the x location of the top left corner of the box + * @param y the y location of the top left corner of the box + * @param width the width location of the box + * @param height the height location of the box + * @param fontSize the font size of text in the box, 0 for default + * @param text the text displayed in the box + */ + private void label(int x, int y, int width, int height, int fontSize, String text) { + if (fontSize > 0) { + out.printf("<div style=\"position:absolute; left:%dpx; top:%dpx; width:%dpx; " + + "height:%dpx; font-size:%dpt\">%s</div>\n", + x, y, width, height, fontSize, text); + } else { + out.printf("<div style=\"position:absolute; left:%dpx; top:%dpx; width:%dpx; " + + "height:%dpx\">%s</div>\n", + x, y, width, height, text); + } + } + + /** + * Prints a vertical line of given width, height and color at the given + * location. + * + * @param x the x location of the start point of the line + * @param y the y location of the start point of the line + * @param width the width of the line + * @param length the length of the line + * @param color the color of the line + */ + private void verticalLine(int x, int y, int width, int length, Color color) { + out.printf("<div style='position: absolute; left: %dpx; top: %dpx; width: %dpx; " + + "height: %dpx; border-left: %dpx solid %s'" + "></div>\n", + x, y, width, length, width, formatColor(color)); + } + + /** + * Prints an HTML anchor with the given name, + */ + private void anchor(String name) { + out.println("<a name='" + name + "'/>"); + } + + /** + * Formats the given {@link Color} to a css style color string. + */ + private String formatColor(Color color) { + int r = color.getRed(); + int g = color.getGreen(); + int b = color.getBlue(); + int a = color.getAlpha(); + + return String.format("rgba(%d,%d,%d,%f)", r, g, b, (a / 255.0)); + } + + /** + * Transform the name into a form suitable as a css class. + */ + private String chartTypeNameAsCSSClass(String name) { + return name.replace(' ', '_'); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/BlazeQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/BlazeQueryEnvironment.java new file mode 100644 index 0000000..d6c2992 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/BlazeQueryEnvironment.java
@@ -0,0 +1,633 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2; + +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.TRISTATE; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.cmdline.ResolvedTargets; +import com.google.devtools.build.lib.cmdline.TargetParsingException; +import com.google.devtools.build.lib.events.ErrorSensingEventHandler; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.graph.Digraph; +import com.google.devtools.build.lib.graph.Node; +import com.google.devtools.build.lib.packages.AggregatingAttributeMapper; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.pkgcache.PackageProvider; +import com.google.devtools.build.lib.pkgcache.TargetEdgeObserver; +import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator; +import com.google.devtools.build.lib.pkgcache.TargetProvider; +import com.google.devtools.build.lib.query2.engine.BlazeQueryEvalResult; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment; +import com.google.devtools.build.lib.query2.engine.QueryException; +import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.engine.SkyframeRestartQueryException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.BinaryPredicate; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * The environment of a Blaze query. Not thread-safe. + */ +public class BlazeQueryEnvironment implements QueryEnvironment<Target> { + protected final ErrorSensingEventHandler eventHandler; + private final TargetProvider targetProvider; + private final TargetPatternEvaluator targetPatternEvaluator; + private final Digraph<Target> graph = new Digraph<>(); + private final ErrorPrintingTargetEdgeErrorObserver errorObserver; + private final LabelVisitor labelVisitor; + private final Map<String, Set<Target>> letBindings = new HashMap<>(); + private final Map<String, ResolvedTargets<Target>> resolvedTargetPatterns = new HashMap<>(); + protected final boolean keepGoing; + private final boolean strictScope; + protected final int loadingPhaseThreads; + + private final BinaryPredicate<Rule, Attribute> dependencyFilter; + private final Predicate<Label> labelFilter; + + private final Set<Setting> settings; + private final List<QueryFunction> extraFunctions; + private final BlazeTargetAccessor accessor = new BlazeTargetAccessor(); + + /** + * Note that the correct operation of this class critically depends on the Reporter being a + * singleton object, shared by all cooperating classes contributing to Query. + * @param strictScope if true, fail the whole query if a label goes out of scope. + * @param loadingPhaseThreads the number of threads to use during loading + * the packages for the query. + * @param labelFilter a predicate that determines if a specific label is + * allowed to be visited during query execution. If it returns false, + * the query execution is stopped with an error message. + * @param settings a set of enabled settings + */ + public BlazeQueryEnvironment(PackageProvider packageProvider, + TargetPatternEvaluator targetPatternEvaluator, + boolean keepGoing, + boolean strictScope, + int loadingPhaseThreads, + Predicate<Label> labelFilter, + EventHandler eventHandler, + Set<Setting> settings, + Iterable<QueryFunction> extraFunctions) { + this.eventHandler = new ErrorSensingEventHandler(eventHandler); + this.targetProvider = packageProvider; + this.targetPatternEvaluator = targetPatternEvaluator; + this.errorObserver = new ErrorPrintingTargetEdgeErrorObserver(this.eventHandler); + this.keepGoing = keepGoing; + this.strictScope = strictScope; + this.loadingPhaseThreads = loadingPhaseThreads; + this.dependencyFilter = constructDependencyFilter(settings); + this.labelVisitor = new LabelVisitor(packageProvider, dependencyFilter); + this.labelFilter = labelFilter; + this.settings = Sets.immutableEnumSet(settings); + this.extraFunctions = ImmutableList.copyOf(extraFunctions); + } + + /** + * Note that the correct operation of this class critically depends on the Reporter being a + * singleton object, shared by all cooperating classes contributing to Query. + * @param loadingPhaseThreads the number of threads to use during loading + * the packages for the query. + * @param settings a set of enabled settings + */ + public BlazeQueryEnvironment(PackageProvider packageProvider, + TargetPatternEvaluator targetPatternEvaluator, + boolean keepGoing, + int loadingPhaseThreads, + EventHandler eventHandler, + Set<Setting> settings, + Iterable<QueryFunction> extraFunctions) { + this(packageProvider, targetPatternEvaluator, keepGoing, /*strictScope=*/true, + loadingPhaseThreads, Rule.ALL_LABELS, eventHandler, settings, extraFunctions); + } + + private static BinaryPredicate<Rule, Attribute> constructDependencyFilter(Set<Setting> settings) { + BinaryPredicate<Rule, Attribute> specifiedFilter = + settings.contains(Setting.NO_HOST_DEPS) ? Rule.NO_HOST_DEPS : Rule.ALL_DEPS; + if (settings.contains(Setting.NO_IMPLICIT_DEPS)) { + specifiedFilter = Rule.and(specifiedFilter, Rule.NO_IMPLICIT_DEPS); + } + if (settings.contains(Setting.NO_NODEP_DEPS)) { + specifiedFilter = Rule.and(specifiedFilter, Rule.NO_NODEP_ATTRIBUTES); + } + return specifiedFilter; + } + + /** + * Evaluate the specified query expression in this environment. + * + * @return a {@link BlazeQueryEvalResult} object that contains the resulting set of targets, the + * partial graph, and a bit to indicate whether errors occured during evaluation; note that the + * success status can only be false if {@code --keep_going} was in effect + * @throws QueryException if the evaluation failed and {@code --nokeep_going} was in + * effect + */ + public BlazeQueryEvalResult<Target> evaluateQuery(QueryExpression expr) throws QueryException { + // Some errors are reported as QueryExceptions and others as ERROR events + // (if --keep_going). + eventHandler.resetErrors(); + resolvedTargetPatterns.clear(); + + // In the --nokeep_going case, errors are reported in the order in which the patterns are + // specified; using a linked hash set here makes sure that the left-most error is reported. + Set<String> targetPatternSet = new LinkedHashSet<>(); + expr.collectTargetPatterns(targetPatternSet); + try { + resolvedTargetPatterns.putAll(preloadOrThrow(targetPatternSet)); + } catch (TargetParsingException e) { + // Unfortunately, by evaluating the patterns in parallel, we lose some location information. + throw new QueryException(expr, e.getMessage()); + } + + Set<Target> resultNodes; + try { + resultNodes = expr.eval(this); + } catch (QueryException e) { + throw new QueryException(e, expr); + } + + if (eventHandler.hasErrors()) { + if (!keepGoing) { + // This case represents loading-phase errors reported during evaluation + // of target patterns that don't cause evaluation to fail per se. + throw new QueryException("Evaluation of query \"" + expr + + "\" failed due to BUILD file errors"); + } else { + eventHandler.handle(Event.warn("--keep_going specified, ignoring errors. " + + "Results may be inaccurate")); + } + } + + return new BlazeQueryEvalResult<>(!eventHandler.hasErrors(), resultNodes, graph); + } + + public BlazeQueryEvalResult<Target> evaluateQuery(String query) throws QueryException { + return evaluateQuery(QueryExpression.parse(query, this)); + } + + @Override + public void reportBuildFileError(QueryExpression caller, String message) throws QueryException { + if (!keepGoing) { + throw new QueryException(caller, message); + } else { + // Keep consistent with evaluateQuery() above. + eventHandler.handle(Event.error("Evaluation of query \"" + caller + "\" failed: " + message)); + } + } + + @Override + public Set<Target> getTargetsMatchingPattern(QueryExpression caller, + String pattern) throws QueryException { + // We can safely ignore the boolean error flag. The evaluateQuery() method above wraps the + // entire query computation in an error sensor. + + Set<Target> targets = new LinkedHashSet<>(resolvedTargetPatterns.get(pattern).getTargets()); + + // Sets.filter would be more convenient here, but can't deal with exceptions. + Iterator<Target> targetIterator = targets.iterator(); + while (targetIterator.hasNext()) { + Target target = targetIterator.next(); + if (!validateScope(target.getLabel(), strictScope)) { + targetIterator.remove(); + } + } + + Set<PathFragment> packages = new HashSet<>(); + for (Target target : targets) { + packages.add(target.getLabel().getPackageFragment()); + } + + Set<Target> result = new LinkedHashSet<>(); + for (Target target : targets) { + result.add(getOrCreate(target)); + + // Preservation of graph order: it is important that targets obtained via + // a wildcard such as p:* are correctly ordered w.r.t. each other, so to + // ensure this, we add edges between any pair of directly connected + // targets in this set. + if (target instanceof OutputFile) { + OutputFile outputFile = (OutputFile) target; + if (targets.contains(outputFile.getGeneratingRule())) { + makeEdge(outputFile, outputFile.getGeneratingRule()); + } + } else if (target instanceof Rule) { + Rule rule = (Rule) target; + for (Label label : rule.getLabels(dependencyFilter)) { + if (!packages.contains(label.getPackageFragment())) { + continue; // don't cause additional package loading + } + try { + if (!validateScope(label, strictScope)) { + continue; // Don't create edges to targets which are out of scope. + } + Target to = getTargetOrThrow(label); + if (targets.contains(to)) { + makeEdge(rule, to); + } + } catch (NoSuchThingException e) { + /* ignore */ + } catch (InterruptedException e) { + throw new QueryException("interrupted"); + } + } + } + } + return result; + } + + public Node<Target> getTarget(Label label) throws TargetNotFoundException, QueryException { + // Can't use strictScope here because we are expecting a target back. + validateScope(label, true); + try { + return getNode(getTargetOrThrow(label)); + } catch (NoSuchThingException e) { + throw new TargetNotFoundException(e); + } catch (InterruptedException e) { + throw new QueryException("interrupted"); + } + } + + private Node<Target> getNode(Target target) { + return graph.createNode(target); + } + + private Collection<Node<Target>> getNodes(Iterable<Target> target) { + Set<Node<Target>> result = new LinkedHashSet<>(); + for (Target t : target) { + result.add(getNode(t)); + } + return result; + } + + @Override + public Target getOrCreate(Target target) { + return getNode(target).getLabel(); + } + + @Override + public Collection<Target> getFwdDeps(Target target) { + return getTargetsFromNodes(getNode(target).getSuccessors()); + } + + @Override + public Collection<Target> getReverseDeps(Target target) { + return getTargetsFromNodes(getNode(target).getPredecessors()); + } + + @Override + public Set<Target> getTransitiveClosure(Set<Target> targetNodes) { + for (Target node : targetNodes) { + checkBuilt(node); + } + return getTargetsFromNodes(graph.getFwdReachable(getNodes(targetNodes))); + } + + /** + * Checks that the graph rooted at 'targetNode' has been completely built; + * fails if not. Callers of {@link #getTransitiveClosure} must ensure that + * {@link #buildTransitiveClosure} has been called before. + * + * <p>It would be inefficient and failure-prone to make getTransitiveClosure + * call buildTransitiveClosure directly. Also, it would cause + * nondeterministic behavior of the operators, since the set of packages + * loaded (and hence errors reported) would depend on the ordering details of + * the query operators' implementations. + */ + private void checkBuilt(Target targetNode) { + Preconditions.checkState( + labelVisitor.hasVisited(targetNode.getLabel()), + "getTransitiveClosure(%s) called without prior call to buildTransitiveClosure()", + targetNode); + } + + protected void preloadTransitiveClosure(Set<Target> targets, int maxDepth) throws QueryException { + } + + @Override + public void buildTransitiveClosure(QueryExpression caller, + Set<Target> targetNodes, + int maxDepth) throws QueryException { + Set<Target> targets = targetNodes; + preloadTransitiveClosure(targets, maxDepth); + + try { + labelVisitor.syncWithVisitor(eventHandler, targets, keepGoing, + loadingPhaseThreads, maxDepth, errorObserver, new GraphBuildingObserver()); + } catch (InterruptedException e) { + throw new QueryException(caller, "transitive closure computation was interrupted"); + } + + if (errorObserver.hasErrors()) { + reportBuildFileError(caller, "errors were encountered while computing transitive closure"); + } + } + + @Override + public Set<Target> getNodesOnPath(Target from, Target to) { + return getTargetsFromNodes(graph.getShortestPath(getNode(from), getNode(to))); + } + + @Override + public Set<Target> getVariable(String name) { + return letBindings.get(name); + } + + @Override + public Set<Target> setVariable(String name, Set<Target> value) { + return letBindings.put(name, value); + } + + /** + * It suffices to synchronize the modifications of this.graph from within the + * GraphBuildingObserver, because that's the only concurrent part. + * Concurrency is always encapsulated within the evaluation of a single query + * operator (e.g. deps(), somepath(), etc). + */ + private class GraphBuildingObserver implements TargetEdgeObserver { + + @Override + public synchronized void edge(Target from, Attribute attribute, Target to) { + Preconditions.checkState(attribute == null || + dependencyFilter.apply(((Rule) from), attribute), + "Disallowed edge from LabelVisitor: %s --> %s", from, to); + makeEdge(from, to); + } + + @Override + public synchronized void node(Target node) { + graph.createNode(node); + } + + @Override + public void missingEdge(Target target, Label to, NoSuchThingException e) { + // No - op. + } + } + + private void makeEdge(Target from, Target to) { + graph.addEdge(from, to); + } + + private boolean validateScope(Label label, boolean strict) throws QueryException { + if (!labelFilter.apply(label)) { + String error = String.format("target '%s' is not within the scope of the query", label); + if (strict) { + throw new QueryException(error); + } else { + eventHandler.handle(Event.warn(error + ". Skipping")); + return false; + } + } + return true; + } + + public Set<Target> evalTargetPattern(QueryExpression caller, String pattern) + throws QueryException { + if (!resolvedTargetPatterns.containsKey(pattern)) { + try { + resolvedTargetPatterns.putAll(preloadOrThrow(ImmutableList.of(pattern))); + } catch (TargetParsingException e) { + // Will skip the target and keep going if -k is specified. + resolvedTargetPatterns.put(pattern, ResolvedTargets.<Target>empty()); + reportBuildFileError(caller, e.getMessage()); + } + } + return getTargetsMatchingPattern(caller, pattern); + } + + private Map<String, ResolvedTargets<Target>> preloadOrThrow(Collection<String> patterns) + throws TargetParsingException { + try { + // Note that this may throw a RuntimeException if deps are missing in Skyframe. + return targetPatternEvaluator.preloadTargetPatterns( + eventHandler, patterns, keepGoing); + } catch (InterruptedException e) { + // TODO(bazel-team): Propagate the InterruptedException from here [skyframe-loading]. + throw new TargetParsingException("interrupted"); + } + } + + private Target getTargetOrThrow(Label label) + throws NoSuchThingException, SkyframeRestartQueryException, InterruptedException { + Target target = targetProvider.getTarget(eventHandler, label); + if (target == null) { + throw new SkyframeRestartQueryException(); + } + return target; + } + + // TODO(bazel-team): rename this to getDependentFiles when all implementations + // of QueryEnvironment is fixed. + @Override + public Set<Target> getBuildFiles(final QueryExpression caller, Set<Target> nodes) + throws QueryException { + Set<Target> dependentFiles = new LinkedHashSet<>(); + Set<Package> seenPackages = new HashSet<>(); + // Keep track of seen labels, to avoid adding a fake subinclude label that also exists as a + // real target. + Set<Label> seenLabels = new HashSet<>(); + + // Adds all the package definition files (BUILD files and build + // extensions) for package "pkg", to "buildfiles". + for (Target x : nodes) { + Package pkg = x.getPackage(); + if (seenPackages.add(pkg)) { + addIfUniqueLabel(getNode(pkg.getBuildFile()), seenLabels, dependentFiles); + for (Label subinclude + : Iterables.concat(pkg.getSubincludeLabels(), pkg.getSkylarkFileDependencies())) { + addIfUniqueLabel(getSubincludeTarget(subinclude, pkg), seenLabels, dependentFiles); + + // Also add the BUILD file of the subinclude. + try { + addIfUniqueLabel(getSubincludeTarget( + subinclude.getLocalTargetLabel("BUILD"), pkg), seenLabels, dependentFiles); + } catch (Label.SyntaxException e) { + throw new AssertionError("BUILD should always parse as a target name", e); + } + } + } + } + return dependentFiles; + } + + private static void addIfUniqueLabel(Node<Target> node, Set<Label> labels, Set<Target> nodes) { + if (labels.add(node.getLabel().getLabel())) { + nodes.add(node.getLabel()); + } + } + + private Node<Target> getSubincludeTarget(final Label label, Package pkg) { + return getNode(new FakeSubincludeTarget(label, pkg.getBuildFile().getLocation())); + } + + @Override + public TargetAccessor<Target> getAccessor() { + return accessor; + } + + @Override + public boolean isSettingEnabled(Setting setting) { + return settings.contains(Preconditions.checkNotNull(setting)); + } + + @Override + public Iterable<QueryFunction> getFunctions() { + ImmutableList.Builder<QueryFunction> builder = ImmutableList.builder(); + builder.addAll(DEFAULT_QUERY_FUNCTIONS); + builder.addAll(extraFunctions); + return builder.build(); + } + + private final class BlazeTargetAccessor implements TargetAccessor<Target> { + + @Override + public String getTargetKind(Target target) { + return target.getTargetKind(); + } + + @Override + public String getLabel(Target target) { + return target.getLabel().toString(); + } + + @Override + public List<Target> getLabelListAttr(QueryExpression caller, Target target, String attrName, + String errorMsgPrefix) throws QueryException { + Preconditions.checkArgument(target instanceof Rule); + + List<Target> result = new ArrayList<>(); + Rule rule = (Rule) target; + + AggregatingAttributeMapper attrMap = AggregatingAttributeMapper.of(rule); + Type<?> attrType = attrMap.getAttributeType(attrName); + if (attrType == null) { + // Return an empty list if the attribute isn't defined for this rule. + return ImmutableList.of(); + } + for (Object value : attrMap.visitAttribute(attrName, attrType)) { + // Computed defaults may have null values. + if (value != null) { + for (Label label : attrType.getLabels(value)) { + try { + result.add(getTarget(label).getLabel()); + } catch (TargetNotFoundException e) { + reportBuildFileError(caller, errorMsgPrefix + e.getMessage()); + } + } + } + } + + return result; + } + + @Override + public List<String> getStringListAttr(Target target, String attrName) { + Preconditions.checkArgument(target instanceof Rule); + return NonconfigurableAttributeMapper.of((Rule) target).get(attrName, Type.STRING_LIST); + } + + @Override + public String getStringAttr(Target target, String attrName) { + Preconditions.checkArgument(target instanceof Rule); + return NonconfigurableAttributeMapper.of((Rule) target).get(attrName, Type.STRING); + } + + @Override + public Iterable<String> getAttrAsString(Target target, String attrName) { + Preconditions.checkArgument(target instanceof Rule); + List<String> values = new ArrayList<>(); // May hold null values. + Attribute attribute = ((Rule) target).getAttributeDefinition(attrName); + if (attribute != null) { + Type<?> attributeType = attribute.getType(); + for (Object attrValue : AggregatingAttributeMapper.of((Rule) target).visitAttribute( + attribute.getName(), attributeType)) { + + // Ugly hack to maintain backward 'attr' query compatibility for BOOLEAN and TRISTATE + // attributes. These are internally stored as actual Boolean or TriState objects but were + // historically queried as integers. To maintain compatibility, we inspect their actual + // value and return the integer equivalent represented as a String. This code is the + // opposite of the code in BooleanType and TriStateType respectively. + if (attributeType == BOOLEAN) { + values.add(Type.BOOLEAN.cast(attrValue) ? "1" : "0"); + } else if (attributeType == TRISTATE) { + switch (Type.TRISTATE.cast(attrValue)) { + case AUTO : + values.add("-1"); + break; + case NO : + values.add("0"); + break; + case YES : + values.add("1"); + break; + default : + throw new AssertionError("This can't happen!"); + } + } else { + values.add(attrValue == null ? null : attrValue.toString()); + } + } + } + return values; + } + + @Override + public boolean isRule(Target target) { + return target instanceof Rule; + } + + @Override + public boolean isTestRule(Target target) { + return TargetUtils.isTestRule(target); + } + + @Override + public boolean isTestSuite(Target target) { + return TargetUtils.isTestSuiteRule(target); + } + } + + /** Given a set of target nodes, returns the targets. */ + private static Set<Target> getTargetsFromNodes(Iterable<Node<Target>> input) { + Set<Target> result = new LinkedHashSet<>(); + for (Node<Target> node : input) { + result.add(node.getLabel()); + } + return result; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/ErrorPrintingTargetEdgeErrorObserver.java b/src/main/java/com/google/devtools/build/lib/query2/ErrorPrintingTargetEdgeErrorObserver.java new file mode 100644 index 0000000..d47b8f3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/ErrorPrintingTargetEdgeErrorObserver.java
@@ -0,0 +1,53 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.query2; + +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.syntax.Label; + +/** + * Record errors, such as missing package/target or rules containing errors, + * encountered during visitation. Emit an error message upon encountering + * missing edges + * + * The accessor {@link #hasErrors}) may not be called until the concurrent phase + * is over, i.e. all external calls to visit() methods have completed. + */ +@ThreadSafety.ConditionallyThreadSafe // condition: only call hasErrors + // once the visitation is complete. +class ErrorPrintingTargetEdgeErrorObserver extends TargetEdgeErrorObserver { + + private final EventHandler eventHandler; + + /** + * @param eventHandler eventHandler to route exceptions to as errors. + */ + public ErrorPrintingTargetEdgeErrorObserver(EventHandler eventHandler) { + this.eventHandler = eventHandler; + } + + @ThreadSafety.ThreadSafe + @Override + public void missingEdge(Target target, Label label, NoSuchThingException e) { + eventHandler.handle(Event.error(TargetUtils.getLocationMaybe(target), + TargetUtils.formatMissingEdge(target, label, e))); + super.missingEdge(target, label, e); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/FakeSubincludeTarget.java b/src/main/java/com/google/devtools/build/lib/query2/FakeSubincludeTarget.java new file mode 100644 index 0000000..f1a6469 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/FakeSubincludeTarget.java
@@ -0,0 +1,86 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.ConstantRuleVisibility; +import com.google.devtools.build.lib.packages.License; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleVisibility; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Set; + +/** + * A fake Target - Use only so that "blaze query" can report subincluded files as Targets. + */ +public class FakeSubincludeTarget implements Target { + + private final Label label; + private final Location location; + + FakeSubincludeTarget(Label label, Location location) { + this.label = Preconditions.checkNotNull(label); + this.location = Preconditions.checkNotNull(location); + } + + @Override + public Label getLabel() { + return label; + } + + @Override + public String getName() { + return label.getName(); + } + + @Override + public Package getPackage() { + throw new UnsupportedOperationException(); + } + + @Override + public String getTargetKind() { + return "source file"; + } + + @Override + public Rule getAssociatedRule() { + return null; + } + + @Override + public License getLicense() { + throw new UnsupportedOperationException(); + } + + @Override + public Location getLocation() { + return location; + } + + @Override + public Set<License.DistributionType> getDistributions() { + return ImmutableSet.of(); + } + + @Override + public RuleVisibility getVisibility() { + return ConstantRuleVisibility.PUBLIC; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/LabelVisitor.java b/src/main/java/com/google/devtools/build/lib/query2/LabelVisitor.java new file mode 100644 index 0000000..b72e3aa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/LabelVisitor.java
@@ -0,0 +1,481 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.query2; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Throwables; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.MapMaker; +import com.google.common.collect.Multimaps; +import com.google.common.collect.SetMultimap; +import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.AggregatingAttributeMapper; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageGroup; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.PackageProvider; +import com.google.devtools.build.lib.pkgcache.TargetEdgeObserver; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.BinaryPredicate; + +import java.util.Collection; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * <p>Visit the transitive closure of a label. Primarily used to "fault in" + * packages to the packageProvider and ensure the necessary targets exists, in + * advance of the configuration step, which is intolerant of missing + * packages/targets. + * + * <p>LabelVisitor loads packages concurrently where possible, to increase I/O + * parallelism. However, the public interface is not thread-safe: calls to + * public methods should not be made concurrently. + * + * <p>LabelVisitor is stateful: It remembers the previous visitation and can + * check its validity on subsequent calls to sync() instead of doing the normal + * visitation. + * + * <p>TODO(bazel-team): (2009) a small further optimization could be achieved if we + * create tasks at the package (not individual label) level, since package + * loading is the expensive step. This would require additional bookkeeping to + * maintain the list of labels that we need to visit once a package becomes + * available. Profiling suggests that there is still a potential benefit to be + * gained: when the set of packages is known a-priori, loading a set of packages + * that took 20 seconds can be done under 5 in the sequential case or 7 in the + * current (parallel) case. + * + * <h4>Concurrency</h4> + * + * <p>The sync() methods of this class is thread-compatible. The accessor + * ({@link #hasVisited} and similar must not be called until the concurrent phase + * is over, i.e. all external calls to visit() methods have completed. + */ +final class LabelVisitor { + + /** + * Attributes of a visitation which determine whether it is up-to-date or not. + */ + private class VisitationAttributes { + private Collection<Target> targetsToVisit; + private boolean success = false; + private boolean visitSubincludes = true; + private int maxDepth = 0; + + /** + * Returns true if and only if this visitation attribute is still up-to-date. + */ + boolean current() { + return targetsToVisit.equals(lastVisitation.targetsToVisit) + && maxDepth <= lastVisitation.maxDepth + && visitSubincludes == lastVisitation.visitSubincludes; + } + } + + /* + * Interrupts during the loading phase =================================== + * + * Bazel can be interrupted in the middle of the loading phase. The mechanics + * of this are far from trivial, so there is an explanation of how they are + * supposed to work. For a description how the same thing works in the + * execution phase, see ParallelBuilder.java . + * + * The sequence of events that happen when the user presses Ctrl-C is the + * following: + * + * 1. A SIGINT gets delivered to the Bazel client process. + * + * 2. The client process delivers the SIGINT to the server process. + * + * 3. The interruption state of the main thread is set to true. + * + * 4. Sooner or later, this results in an InterruptedException being thrown. + * Usually this takes place because the main thread is interrupted during + * AbstractQueueVisitor.awaitTermination(). The only exception to this is when + * the interruption occurs during the loading of a package of a label + * specified on the command line; in this case, the InterruptedException is + * thrown during the loading of an individual package (see below where this + * can occur) + * + * 5. The main thread calls ThreadPoolExecutor.shutdown(), which in turn + * interrupts every worker thread. Then the main thread waits for their + * termination. + * + * 6. An InterruptedException is thrown during the loading of an individual + * package in the worker threads. + * + * 7. All worker threads terminate. + * + * 8. An InterruptedException is thrown from + * AbstractQueueVisitor.awaitTermination() + * + * 9. This exception causes the execution of the currently running command to + * terminate prematurely. + * + * The interruption of the loading of an individual package can happen in two + * different ways depending on whether Python preprocessing is in effect or + * not. + * + * If there is no Python preprocessing: + * + * 1. We periodically check the interruption state of the thread in + * UnixGlob.reallyGlob(). If it is interrupted, an InterruptedException is + * thrown. + * + * 2. The stack is unwound until we are out of the part of the call stack + * responsible for package loading. This either means that the worker thread + * terminates or that the label parsing terminates if the package that is + * being loaded was specified on the command line. + * + * If there is Python preprocessing, events are a bit more complicated. In + * this case, the real work happens on the thread the Python preprocessor is + * called from, but in a bit more convoluted way: a new thread is spawned by + * to handle the input from the Python process and + * the output to the Python process is handled on the main thread. The reading + * thread parses requests from the preprocessor, and passes them using a queue + * to the writing thread (that is, the main thread), so that we can do the + * work there. This is important because this way, we don't have any work that + * we need to interrupt in a thread that is not spawned by us. So: + * + * 1. The interrupted state of the main thread is set. + * + * 2. This results in an InterruptedException during the execution of the task + * in PythonStdinInputStream.getNextMessage(). + * + * 3. We exit from RequestParser.Request.run() prematurely, set a flag to + * signal that we were interrupted, and throw an InterruptedIOException. + * + * 4. The Python child process and reading thread are terminated. + * + * 5. Based on the flag we set in step 3, we realize that the termination was + * due to an interruption, and an InterruptedException is thrown. This can + * either raise an AbnormalTerminationException, or make Command.execute() + * return normally, so we check for both cases. + * + * 6. This InterruptedException causes the loading of the package to terminate + * prematurely. + * + * Life is not simple. + */ + private final PackageProvider packageProvider; + private final BinaryPredicate<Rule, Attribute> edgeFilter; + private final SetMultimap<Package, Target> visitedMap = + Multimaps.synchronizedSetMultimap(HashMultimap.<Package, Target>create()); + private final ConcurrentMap<Label, Integer> visitedTargets = new MapMaker().makeMap(); + + private VisitationAttributes lastVisitation; + + /** + * Constant for limiting the permitted depth of recursion. + */ + private static final int RECURSION_LIMIT = 100; + + /** + * Construct a LabelVisitor. + * + * @param packageProvider how to resolve labels to targets. + * @param edgeFilter which edges may be traversed. + */ + public LabelVisitor(PackageProvider packageProvider, + BinaryPredicate<Rule, Attribute> edgeFilter) { + this.packageProvider = packageProvider; + this.lastVisitation = new VisitationAttributes(); + this.edgeFilter = edgeFilter; + } + + boolean syncWithVisitor(EventHandler eventHandler, Collection<Target> targetsToVisit, + boolean keepGoing, int parallelThreads, int maxDepth, TargetEdgeObserver... observers) + throws InterruptedException { + VisitationAttributes nextVisitation = new VisitationAttributes(); + nextVisitation.targetsToVisit = targetsToVisit; + nextVisitation.maxDepth = maxDepth; + + if (!lastVisitation.success || !nextVisitation.current()) { + try { + nextVisitation.success = redoVisitation(eventHandler, nextVisitation, keepGoing, + parallelThreads, maxDepth, observers); + return nextVisitation.success; + } finally { + lastVisitation = nextVisitation; + } + } else { + return true; + } + } + + // Does a bounded transitive visitation starting at the given top-level targets. + private boolean redoVisitation(EventHandler eventHandler, + VisitationAttributes visitation, + boolean keepGoing, + int parallelThreads, + int maxDepth, + TargetEdgeObserver... observers) + throws InterruptedException { + visitedMap.clear(); + visitedTargets.clear(); + + Visitor visitor = new Visitor(eventHandler, keepGoing, parallelThreads, maxDepth, observers); + + Throwable uncaught = null; + boolean result; + try { + visitor.visitTargets(visitation.targetsToVisit); + } catch (Throwable t) { + visitor.stopNewActions(); + uncaught = t; + } finally { + // Run finish() in finally block to ensure we don't leak threads on exceptions. + result = visitor.finish(); + } + Throwables.propagateIfPossible(uncaught); + return result; + } + + boolean hasVisited(Label target) { + return visitedTargets.containsKey(target); + } + + @VisibleForTesting class Visitor extends AbstractQueueVisitor { + + private final static String THREAD_NAME = "LabelVisitor"; + + private final EventHandler eventHandler; + private final boolean keepGoing; + private final int maxDepth; + private final Iterable<TargetEdgeObserver> observers; + private final TargetEdgeErrorObserver errorObserver; + private final AtomicBoolean stopNewActions = new AtomicBoolean(false); + private static final boolean CONCURRENT = true; + + + public Visitor(EventHandler eventHandler, boolean keepGoing, int parallelThreads, + int maxDepth, TargetEdgeObserver... observers) { + // Observing the loading phase of a typical large package (with all subpackages) shows + // maximum thread-level concurrency of ~20. Limiting the total number of threads to 200 is + // therefore conservative and should help us avoid hitting native limits. + super(CONCURRENT, parallelThreads, parallelThreads, 1L, TimeUnit.SECONDS, !keepGoing, + THREAD_NAME); + this.eventHandler = eventHandler; + this.maxDepth = maxDepth; + this.errorObserver = new TargetEdgeErrorObserver(); + ImmutableList.Builder<TargetEdgeObserver> builder = ImmutableList.builder(); + for (TargetEdgeObserver observer : observers) { + builder.add(observer); + } + builder.add(errorObserver); + this.observers = builder.build(); + this.keepGoing = keepGoing; + } + + /** + * Visit the specified labels and follow the transitive closure of their + * outbound dependencies. + * + * @param targets the targets to visit + */ + @ThreadSafe + public void visitTargets(Iterable<Target> targets) { + for (Target target : targets) { + visit(null, null, target, 0, 0); + } + } + + @ThreadSafe + public boolean finish() throws InterruptedException { + work(true); + return !errorObserver.hasErrors(); + } + + @Override + protected boolean blockNewActions() { + return (!keepGoing && errorObserver.hasErrors()) || super.blockNewActions() || + stopNewActions.get(); + } + + public void stopNewActions() { + stopNewActions.set(true); + } + + private void enqueueTarget( + final Target from, final Attribute attr, final Label label, final int depth, + final int count) { + // Don't perform the targetProvider lookup if at the maximum depth already. + if (depth >= maxDepth) { + return; + } else if (attr != null && from instanceof Rule) { + if (!edgeFilter.apply((Rule) from, attr)) { + return; + } + } + + // Avoid thread-related overhead when not crossing packages. + // Can start a new thread when count reaches 100, to prevent infinite recursion. + if (from != null && from.getLabel().getPackageFragment() == label.getPackageFragment() && + !blockNewActions() && count < RECURSION_LIMIT) { + newVisitRunnable(from, attr, label, depth, count + 1).run(); + } else { + enqueue(newVisitRunnable(from, attr, label, depth, 0)); + } + } + + private Runnable newVisitRunnable(final Target from, final Attribute attr, final Label label, + final int depth, final int count) { + return new Runnable () { + @Override + public void run() { + try { + Target target = packageProvider.getTarget(eventHandler, label); + if (target == null) { + // Let target visitation continue so we can discover additional unknown inputs. + return; + } + visit(from, attr, packageProvider.getTarget(eventHandler, label), depth + 1, count); + } catch (NoSuchThingException e) { + observeError(from, label, e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + }; + } + + private void visitTargetVisibility(Target target, int depth, int count) { + Attribute attribute = null; + if (target instanceof Rule) { + attribute = ((Rule) target).getRuleClassObject().getAttributeByName("visibility"); + } + + for (Label label : target.getVisibility().getDependencyLabels()) { + enqueueTarget(target, attribute, label, depth, count); + } + } + + /** + * Visit all the labels in a given rule. + * + * <p>Called in a worker thread if CONCURRENT. + * + * @param rule the rule to visit + */ + @ThreadSafe + private void visitRule(final Rule rule, final int depth, final int count) { + // Follow all labels defined by this rule: + AggregatingAttributeMapper.of(rule).visitLabels(new AttributeMap.AcceptsLabelAttribute() { + @Override + public void acceptLabelAttribute(Label label, Attribute attribute) { + enqueueTarget(rule, attribute, label, depth, count); + } + }); + } + + @ThreadSafe + private void visitPackageGroup(PackageGroup packageGroup, int depth, int count) { + for (final Label include : packageGroup.getIncludes()) { + enqueueTarget(packageGroup, null, include, depth, count); + } + } + + /** + * Visits the target and its package. + * + * <p>Potentially blocking invocations into the package cache are + * enqueued in the worker pool if CONCURRENT. + */ + private void visit( + Target from, Attribute attribute, final Target target, int depth, int count) { + if (depth > maxDepth) { + return; + } + + if (from != null) { + observeEdge(from, attribute, target); + } + + visitedMap.put(target.getPackage(), target); + visitTargetNode(target, depth, count); + } + + /** + * Visit the specified target. + * Called in a worker thread if CONCURRENT. + * + * @param target the target to visit + */ + private void visitTargetNode(Target target, int depth, int count) { + Integer minTargetDepth = visitedTargets.putIfAbsent(target.getLabel(), depth); + if (minTargetDepth != null) { + // The target was already visited at a greater depth. + // The closure we are about to build is therefore a subset of what + // has already been built, and we can skip it. + // Also special case MAX_VALUE, where we never want to revisit targets. + // (This avoids loading phase overhead outside of queries). + if (maxDepth == Integer.MAX_VALUE || minTargetDepth <= depth) { + return; + } + // Check again in case it was overwritten by another thread. + synchronized (visitedTargets) { + if (visitedTargets.get(target.getLabel()) <= depth) { + return; + } + visitedTargets.put(target.getLabel(), depth); + } + } + + observeNode(target); + if (target instanceof OutputFile) { + Rule rule = ((OutputFile) target).getGeneratingRule(); + observeEdge(target, null, rule); + // This is the only recursive call to visit which doesn't pass through enqueueTarget(). + visit(null, null, rule, depth + 1, count + 1); + visitTargetVisibility(target, depth, count); + } else if (target instanceof InputFile) { + visitTargetVisibility(target, depth, count); + } else if (target instanceof Rule) { + visitTargetVisibility(target, depth, count); + visitRule((Rule) target, depth, count); + } else if (target instanceof PackageGroup) { + visitPackageGroup((PackageGroup) target, depth, count); + } + } + + private void observeEdge(Target from, Attribute attribute, Target to) { + for (TargetEdgeObserver observer : observers) { + observer.edge(from, attribute, to); + } + } + + private void observeNode(Target target) { + for (TargetEdgeObserver observer : observers) { + observer.node(target); + } + } + + private void observeError(Target from, Label label, NoSuchThingException e) { + for (TargetEdgeObserver observer : observers) { + observer.missingEdge(from, label, e); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/SkyframeQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/SkyframeQueryEnvironment.java new file mode 100644 index 0000000..318d844 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/SkyframeQueryEnvironment.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.PackageProvider; +import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator; +import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader; +import com.google.devtools.build.lib.query2.engine.QueryException; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Set; + +/** + * A BlazeQueryEnvironment for Skyframe builds. Currently, this is used to preload transitive + * closures of targets quickly. + */ +public final class SkyframeQueryEnvironment extends BlazeQueryEnvironment { + + private final TransitivePackageLoader transitivePackageLoader; + private static final int MAX_DEPTH_FULL_SCAN_LIMIT = 20; + + public SkyframeQueryEnvironment( + TransitivePackageLoader transitivePackageLoader, PackageProvider packageProvider, + TargetPatternEvaluator targetPatternEvaluator, boolean keepGoing, int loadingPhaseThreads, + EventHandler eventHandler, Set<Setting> settings, Iterable<QueryFunction> functions) { + super(packageProvider, targetPatternEvaluator, keepGoing, loadingPhaseThreads, eventHandler, + settings, functions); + this.transitivePackageLoader = transitivePackageLoader; + } + + @Override + protected void preloadTransitiveClosure(Set<Target> targets, int maxDepth) throws QueryException { + if (maxDepth >= MAX_DEPTH_FULL_SCAN_LIMIT) { + // Only do the full visitation if "maxDepth" is large enough. Otherwise, the benefits of + // preloading will be outweighed by the cost of doing more work than necessary. + try { + transitivePackageLoader.sync(eventHandler, targets, ImmutableSet.<Label>of(), keepGoing, + loadingPhaseThreads, -1); + } catch (InterruptedException e) { + throw new QueryException("interrupted"); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/TargetEdgeErrorObserver.java b/src/main/java/com/google/devtools/build/lib/query2/TargetEdgeErrorObserver.java new file mode 100644 index 0000000..5a075d2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/TargetEdgeErrorObserver.java
@@ -0,0 +1,83 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.query2; + +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.TargetEdgeObserver; +import com.google.devtools.build.lib.syntax.Label; + +/** + * Record errors, such as missing package/target or rules containing errors, + * encountered during visitation. Emit an error message upon encountering + * missing edges + * + * The accessor {@link #hasErrors}) may not be called until the concurrent phase + * is over, i.e. all external calls to visit() methods have completed. + * + * If you need to report errors to the console during visitation, use the + * subclass {@link com.google.devtools.build.lib.query2.ErrorPrintingTargetEdgeErrorObserver}. + */ +class TargetEdgeErrorObserver implements TargetEdgeObserver { + + /** + * True iff errors were encountered. Note, may be set to "true" during the + * concurrent phase. Volatile, because it is assigned by worker threads and + * read by the main thread without monitor synchronization. + */ + private volatile boolean hasErrors = false; + + /** + * Reports an unresolved label error and records the fact that an error was + * encountered. + * @param target the target that referenced the unresolved label + * @param label the label that could not be resolved + * @param e the exception that was thrown when the label could not be resolved + */ + @ThreadSafety.ThreadSafe + @Override + public void missingEdge(Target target, Label label, NoSuchThingException e) { + hasErrors = true; + } + + /** + * Returns true iff any errors (such as missing targets or packages, or rules + * with errors) have been encountered during any work. + * + * <p>Not thread-safe; do not call during visitation. + * + * @return true iff no errors (such as missing targets or packages, or rules + * with errors) have been encountered during any work. + */ + public boolean hasErrors() { + return hasErrors; + } + + @Override + public void edge(Target from, Attribute attribute, Target to) { + // No-op. + } + + @Override + public void node(Target node) { + if (node.getPackage().containsErrors() || + ((node instanceof Rule) && ((Rule) node).containsErrors())) { + this.hasErrors = true; // Note, this is thread-safe. + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java new file mode 100644 index 0000000..d2d5f05 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java
@@ -0,0 +1,96 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; + +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +/** + * Implementation of the <code>allpaths()</code> function. + */ +public class AllPathsFunction implements QueryFunction { + AllPathsFunction() { + } + + @Override + public String getName() { + return "allpaths"; + } + + @Override + public int getMandatoryArguments() { + return 2; + } + + @Override + public List<ArgumentType> getArgumentTypes() { + return ImmutableList.of(ArgumentType.EXPRESSION, ArgumentType.EXPRESSION); + } + + @Override + public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args) + throws QueryException { + QueryExpression from = args.get(0).getExpression(); + QueryExpression to = args.get(1).getExpression(); + + Set<T> fromValue = from.eval(env); + Set<T> toValue = to.eval(env); + + // Algorithm: compute "reachableFromX", the forward transitive closure of + // the "from" set, then find the intersection of "reachableFromX" with the + // reverse transitive closure of the "to" set. The reverse transitive + // closure and intersection operations are interleaved for efficiency. + // "result" holds the intersection. + + env.buildTransitiveClosure(expression, fromValue, Integer.MAX_VALUE); + + Set<T> reachableFromX = env.getTransitiveClosure(fromValue); + Set<T> result = intersection(reachableFromX, toValue); + LinkedList<T> worklist = new LinkedList<>(result); + + T n; + while ((n = worklist.poll()) != null) { + for (T np : env.getReverseDeps(n)) { + if (reachableFromX.contains(np)) { + if (result.add(np)) { + worklist.add(np); + } + } + } + } + return result; + } + + /** + * Returns a (new, mutable, unordered) set containing the intersection of the + * two specified sets. + */ + private static <T> Set<T> intersection(Set<T> x, Set<T> y) { + Set<T> result = new HashSet<>(); + if (x.size() > y.size()) { + Sets.intersection(y, x).copyInto(result); + } else { + Sets.intersection(x, y).copyInto(result); + } + return result; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/AttrFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/AttrFunction.java new file mode 100644 index 0000000..054cf78b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/AttrFunction.java
@@ -0,0 +1,78 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; + +import java.util.List; + +/** + * An attr(attribute, pattern, argument) filter expression, which computes + * the set of subset of nodes in 'argument' which correspond to rules with + * defined attribute 'attribute' with attribute value matching the unanchored + * regexp 'pattern'. For list attributes, the attribute value will be defined as + * a usual List.toString() representation (using '[' as first character, ']' as + * last character and ", " as a delimiter between multiple values). Also, all + * label-based attributes will use fully-qualified label names instead of + * original value specified in the BUILD file. + * + * <pre>expr ::= ATTR '(' ATTRNAME ',' WORD ',' expr ')'</pre> + * + * Examples + * <pre> + * attr(linkshared,1,//project/...) find all rules under in the //project/... that + * have attribute linkshared set to 1. + * </pre> + */ +class AttrFunction extends RegexFilterExpression { + AttrFunction() { + } + + @Override + public String getName() { + return "attr"; + } + + @Override + protected String getPattern(List<Argument> args) { + return args.get(1).getWord(); + } + + @Override + public int getMandatoryArguments() { + return 3; + } + + @Override + public List<ArgumentType> getArgumentTypes() { + return ImmutableList.of(ArgumentType.WORD, ArgumentType.WORD, ArgumentType.EXPRESSION); + } + + @Override + protected <T> String getFilterString(QueryEnvironment<T> env, List<Argument> args, T target) { + throw new IllegalStateException( + "The 'attr' regex filter gets its match values directly from getFilterStrings"); + } + + @Override + protected <T> Iterable<String> getFilterStrings(QueryEnvironment<T> env, + List<Argument> args, T target) { + if (env.getAccessor().isRule(target)) { + return env.getAccessor().getAttrAsString(target, args.get(0).getWord()); + } + return ImmutableList.of(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/BinaryOperatorExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/BinaryOperatorExpression.java new file mode 100644 index 0000000..e5e600f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/BinaryOperatorExpression.java
@@ -0,0 +1,93 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * A binary algebraic set operation. + * + * <pre> + * expr ::= expr (INTERSECT expr)+ + * | expr ('^' expr)+ + * | expr (UNION expr)+ + * | expr ('+' expr)+ + * | expr (EXCEPT expr)+ + * | expr ('-' expr)+ + * </pre> + */ +class BinaryOperatorExpression extends QueryExpression { + + private final Lexer.TokenKind operator; // ::= INTERSECT/CARET | UNION/PLUS | EXCEPT/MINUS + private final ImmutableList<QueryExpression> operands; + + BinaryOperatorExpression(Lexer.TokenKind operator, + List<QueryExpression> operands) { + Preconditions.checkState(operands.size() > 1); + this.operator = operator; + this.operands = ImmutableList.copyOf(operands); + } + + @Override + public <T> Set<T> eval(QueryEnvironment<T> env) throws QueryException { + Set<T> lhsValue = new LinkedHashSet<>(operands.get(0).eval(env)); + + for (int i = 1; i < operands.size(); i++) { + Set<T> rhsValue = operands.get(i).eval(env); + switch (operator) { + case INTERSECT: + case CARET: + lhsValue.retainAll(rhsValue); + break; + case UNION: + case PLUS: + lhsValue.addAll(rhsValue); + break; + case EXCEPT: + case MINUS: + lhsValue.removeAll(rhsValue); + break; + default: + throw new IllegalStateException("operator=" + operator); + } + } + return lhsValue; + } + + @Override + public void collectTargetPatterns(Collection<String> literals) { + for (QueryExpression subExpression : operands) { + subExpression.collectTargetPatterns(literals); + } + } + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + for (int i = 1; i < operands.size(); i++) { + result.append("("); + } + result.append(operands.get(0)); + for (int i = 1; i < operands.size(); i++) { + result.append(" " + operator.getPrettyName() + " " + operands.get(i) + ")"); + } + return result.toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/BlazeQueryEvalResult.java b/src/main/java/com/google/devtools/build/lib/query2/engine/BlazeQueryEvalResult.java new file mode 100644 index 0000000..7f2060f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/BlazeQueryEvalResult.java
@@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.graph.Digraph; + +import java.util.Set; + +/** {@link QueryEvalResult} along with a digraph giving the structure of the results. */ +public class BlazeQueryEvalResult<T> extends QueryEvalResult<T> { + + private final Digraph<T> graph; + + public BlazeQueryEvalResult(boolean success, Set<T> resultSet, Digraph<T> graph) { + super(success, resultSet); + this.graph = Preconditions.checkNotNull(graph); + } + + /** Returns the result as a directed graph over elements. */ + public Digraph<T> getResultGraph() { + return graph.extractSubgraph(resultSet); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java new file mode 100644 index 0000000..d606a6a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java
@@ -0,0 +1,56 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; + +import java.util.List; +import java.util.Set; + +/** + * A buildfiles(x) query expression, which computes the set of BUILD files and + * subincluded files for each target in set x. The result is unordered. This + * operator is typically used for determinining what files or packages to check + * out. + * + * <pre>expr ::= BUILDFILES '(' expr ')'</pre> + */ +class BuildFilesFunction implements QueryFunction { + BuildFilesFunction() { + } + + @Override + public String getName() { + return "buildfiles"; + } + + @Override + public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args) + throws QueryException { + return env.getBuildFiles(expression, args.get(0).getExpression().eval(env)); + } + + @Override + public int getMandatoryArguments() { + return 1; + } + + @Override + public List<ArgumentType> getArgumentTypes() { + return ImmutableList.of(ArgumentType.EXPRESSION); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java new file mode 100644 index 0000000..2b693eb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java
@@ -0,0 +1,88 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * A "deps" query expression, which computes the dependencies of the argument. An optional + * integer-literal second argument may be specified; its value bounds the search from the arguments. + * + * <pre>expr ::= DEPS '(' expr ')'</pre> + * <pre> | DEPS '(' expr ',' WORD ')'</pre> + */ +final class DepsFunction implements QueryFunction { + DepsFunction() { + } + + @Override + public String getName() { + return "deps"; + } + + @Override + public int getMandatoryArguments() { + return 1; // last argument is optional + } + + @Override + public List<ArgumentType> getArgumentTypes() { + return ImmutableList.of(ArgumentType.EXPRESSION, ArgumentType.INTEGER); + } + + /** + * Breadth-first search from the arguments. + */ + @Override + public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args) + throws QueryException { + Set<T> argumentValue = args.get(0).getExpression().eval(env); + int depthBound = args.size() > 1 ? args.get(1).getInteger() : Integer.MAX_VALUE; + env.buildTransitiveClosure(expression, argumentValue, depthBound); + + Set<T> visited = new LinkedHashSet<>(); + Collection<T> current = argumentValue; + + // We need to iterate depthBound + 1 times. + for (int i = 0; i <= depthBound; i++) { + List<T> next = new ArrayList<>(); + for (T node : current) { + if (!visited.add(node)) { + // Already visited; if we see a node in a later round, then we don't need to visit it + // again, because the depth at which we see it at must be greater than or equal to the + // last visit. + continue; + } + + next.addAll(env.getFwdDeps(node)); + } + if (next.isEmpty()) { + // Exit when there are no more nodes to visit. + break; + } + current = next; + } + + return visited; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/FilterFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/FilterFunction.java new file mode 100644 index 0000000..bd89950 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/FilterFunction.java
@@ -0,0 +1,63 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; + +import java.util.List; + +/** + * A label(pattern, argument) filter expression, which computes the set of subset + * of nodes in 'argument' whose label matches the unanchored regexp 'pattern'. + * + * <pre>expr ::= FILTER '(' WORD ',' expr ')'</pre> + * + * Example patterns: + * <pre> + * '//third_party' Match all targets in the //third_party/... + * (equivalent to 'intersect //third_party/...) + * '\.jar$' Match all *.jar targets. + * </pre> + */ +class FilterFunction extends RegexFilterExpression { + FilterFunction() { + } + + @Override + public String getName() { + return "filter"; + } + + @Override + protected String getPattern(List<Argument> args) { + return args.get(0).getWord(); + } + + @Override + public int getMandatoryArguments() { + return 2; + } + + @Override + public List<ArgumentType> getArgumentTypes() { + return ImmutableList.of(ArgumentType.WORD, ArgumentType.EXPRESSION); + } + + @Override + protected <T> String getFilterString(QueryEnvironment<T> env, List<Argument> args, T target) { + return env.getAccessor().getLabel(target); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/FunctionExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/FunctionExpression.java new file mode 100644 index 0000000..62734fd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/FunctionExpression.java
@@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.base.Functions; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; + +import java.util.Collection; +import java.util.List; +import java.util.Set; + +/** + * A query expression for user-defined query functions. + */ +public class FunctionExpression extends QueryExpression { + QueryFunction function; + List<Argument> args; + + public FunctionExpression(QueryFunction function, List<Argument> args) { + this.function = function; + this.args = ImmutableList.copyOf(args); + } + + @Override + public <T> Set<T> eval(QueryEnvironment<T> env) throws QueryException { + return function.<T>eval(env, this, args); + } + + @Override + public void collectTargetPatterns(Collection<String> literals) { + for (Argument arg : args) { + if (arg.getType() == ArgumentType.EXPRESSION) { + arg.getExpression().collectTargetPatterns(literals); + } + } + } + + @Override + public String toString() { + return function.getName() + + "(" + Joiner.on(", ").join(Iterables.transform(args, Functions.toStringFunction())) + ")"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/KindFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/KindFunction.java new file mode 100644 index 0000000..7ae80b8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/KindFunction.java
@@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; + +import java.util.List; + +/** + * A kind(pattern, argument) filter expression, which computes the set of subset + * of nodes in 'argument' whose kind matches the unanchored regexp 'pattern'. + * + * <pre>expr ::= KIND '(' WORD ',' expr ')'</pre> + * + * Example patterns: + * <pre> + * ' file' Match all file targets. + * 'source file' Match all test source file targets. + * 'generated file' Match all test generated file targets. + * ' rule' Match all rule targets. + * 'foo_*' Match all rules starting with "foo_", + * 'test' Match all test (rule) targets. + * </pre> + * + * Note, the space before "file" is needed to prevent unwanted matches against + * (e.g.) "filegroup rule". + */ +class KindFunction extends RegexFilterExpression { + + KindFunction() { + } + + @Override + public String getName() { + return "kind"; + } + + @Override + protected String getPattern(List<Argument> args) { + return args.get(0).getWord(); + } + + @Override + public int getMandatoryArguments() { + return 2; + } + + @Override + public List<ArgumentType> getArgumentTypes() { + return ImmutableList.of(ArgumentType.WORD, ArgumentType.EXPRESSION); + } + + @Override + protected <T> String getFilterString(QueryEnvironment<T> env, List<Argument> args, T target) { + return env.getAccessor().getTargetKind(target); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java new file mode 100644 index 0000000..1093d85 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java
@@ -0,0 +1,72 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * A label(attr_name, argument) expression, which computes the set of targets + * whose labels appear in the specified attribute of some rule in 'argument'. + * + * <pre>expr ::= LABELS '(' WORD ',' expr ')'</pre> + * + * Example: + * <pre> + * labels(srcs, //foo) The 'srcs' source files to the //foo rule. + * </pre> + */ +class LabelsFunction implements QueryFunction { + LabelsFunction() { + } + + @Override + public String getName() { + return "labels"; + } + + @Override + public int getMandatoryArguments() { + return 2; + } + + @Override + public List<ArgumentType> getArgumentTypes() { + return ImmutableList.of(ArgumentType.WORD, ArgumentType.EXPRESSION); + } + + @Override + public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args) + throws QueryException { + Set<T> inputs = args.get(1).getExpression().eval(env); + Set<T> result = new LinkedHashSet<>(); + String attrName = args.get(0).getWord(); + for (T input : inputs) { + if (env.getAccessor().isRule(input)) { + List<T> targets = env.getAccessor().getLabelListAttr(expression, input, attrName, + "in '" + attrName + "' of rule " + env.getAccessor().getLabel(input) + ": "); + for (T target : targets) { + result.add(env.getOrCreate(target)); + } + } + } + return result; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/LetExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/LetExpression.java new file mode 100644 index 0000000..3e17cce --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/LetExpression.java
@@ -0,0 +1,78 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import java.util.Collection; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * A let expression. + * + * <pre>expr ::= LET WORD = expr IN expr</pre> + */ +class LetExpression extends QueryExpression { + + private static final String VAR_NAME_PATTERN = "[a-zA-Z_][a-zA-Z0-9_]*$"; + + // Variables names may be any legal identifier in the C programming language + private static final Pattern NAME_PATTERN = Pattern.compile("^" + VAR_NAME_PATTERN); + + // Variable references are prepended with the "$" character. + // A variable named "x" is referenced as "$x". + private static final Pattern REF_PATTERN = Pattern.compile("^\\$" + VAR_NAME_PATTERN); + + static boolean isValidVarReference(String varName) { + return REF_PATTERN.matcher(varName).matches(); + } + + static String getNameFromReference(String reference) { + return reference.substring(1); + } + + private final String varName; + private final QueryExpression varExpr; + private final QueryExpression bodyExpr; + + LetExpression(String varName, QueryExpression varExpr, QueryExpression bodyExpr) { + this.varName = varName; + this.varExpr = varExpr; + this.bodyExpr = bodyExpr; + } + + @Override + public <T> Set<T> eval(QueryEnvironment<T> env) throws QueryException { + if (!NAME_PATTERN.matcher(varName).matches()) { + throw new QueryException(this, "invalid variable name '" + varName + "' in let expression"); + } + Set<T> varValue = varExpr.eval(env); + Set<T> prevValue = env.setVariable(varName, varValue); + try { + return bodyExpr.eval(env); + } finally { + env.setVariable(varName, prevValue); // restore + } + } + + @Override + public void collectTargetPatterns(Collection<String> literals) { + varExpr.collectTargetPatterns(literals); + bodyExpr.collectTargetPatterns(literals); + } + + @Override + public String toString() { + return "let " + varName + " = " + varExpr + " in " + bodyExpr; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/Lexer.java b/src/main/java/com/google/devtools/build/lib/query2/engine/Lexer.java new file mode 100644 index 0000000..45b6f61 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/Lexer.java
@@ -0,0 +1,281 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * A tokenizer for the Blaze query language, revision 2. + * + * Note, we can avoid a lot of quoting by noting that the characters [() ,] do + * not appear in any label, filename, function name, or regular expression we care about. + * + * No string escapes are allowed ("\"). Given the domain, that's not currently + * a problem. + */ +final class Lexer { + + /** + * Discriminator for different kinds of tokens. + */ + public enum TokenKind { + WORD("word"), + EOF("EOF"), + + COMMA(","), + EQUALS("="), + LPAREN("("), + MINUS("-"), + PLUS("+"), + RPAREN(")"), + CARET("^"), + + __ALL_IDENTIFIERS_FOLLOW(""), // See below + + IN("in"), + LET("let"), + SET("set"), + + INTERSECT("intersect"), + EXCEPT("except"), + UNION("union"); + + private final String prettyName; + + private TokenKind(String prettyName) { + this.prettyName = prettyName; + } + + public String getPrettyName() { + return prettyName; + } + } + + public static final Set<TokenKind> BINARY_OPERATORS = EnumSet.of( + TokenKind.INTERSECT, + TokenKind.CARET, + TokenKind.UNION, + TokenKind.PLUS, + TokenKind.EXCEPT, + TokenKind.MINUS); + + private static final Map<String, TokenKind> keywordMap = new HashMap<>(); + static { + for (TokenKind kind : EnumSet.allOf(TokenKind.class)) { + if (kind.ordinal() > TokenKind.__ALL_IDENTIFIERS_FOLLOW.ordinal()) { + keywordMap.put(kind.getPrettyName(), kind); + } + } + } + + /** + * Returns true iff 'word' is a reserved word of the language. + */ + static boolean isReservedWord(String word) { + return keywordMap.containsKey(word); + } + + /** + * Tokens returned by the Lexer. + */ + static class Token { + + public final TokenKind kind; + public final String word; + + Token(TokenKind kind) { + this.kind = kind; + this.word = null; + } + + Token(String word) { + this.kind = TokenKind.WORD; + this.word = word; + } + + @Override + public String toString() { + return kind == TokenKind.WORD ? word : kind.getPrettyName(); + } + } + + /** + * Entry point to the lexer. Returns the list of tokens for the specified + * input, or throws QueryException. + */ + public static List<Token> scan(char[] buffer) throws QueryException { + Lexer lexer = new Lexer(buffer); + lexer.tokenize(); + return lexer.tokens; + } + + // Input buffer and position + private char[] buffer; + private int pos; + + private final List<Token> tokens = new ArrayList<>(); + + private Lexer(char[] buffer) { + this.buffer = buffer; + this.pos = 0; + } + + private void addToken(Token s) { + tokens.add(s); + } + + /** + * Scans a quoted word delimited by 'quot'. + * + * ON ENTRY: 'pos' is 1 + the index of the first delimiter + * ON EXIT: 'pos' is 1 + the index of the last delimiter. + * + * @return the word token. + */ + private Token quotedWord(char quot) throws QueryException { + int oldPos = pos - 1; + while (pos < buffer.length) { + char c = buffer[pos++]; + switch (c) { + case '\'': + case '"': + if (c == quot) { + // close-quote, all done. + return new Token(bufferSlice(oldPos + 1, pos - 1)); + } + } + } + throw new QueryException("unclosed quotation"); + } + + private TokenKind getTokenKindForWord(String word) { + TokenKind kind = keywordMap.get(word); + return kind == null ? TokenKind.WORD : kind; + } + + // Unquoted words may contain [-*$], but not start with them. For user convenience, unquoted + // words must include UNIX filenames, labels and target label patterns, and simple regexps + // (e.g. cc_.*). Keep consistent with TargetLiteral.toString()! + private String scanWord() { + int oldPos = pos - 1; + while (pos < buffer.length) { + switch (buffer[pos]) { + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': + case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': + case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': + case 's': case 't': case 'u': case 'v': case 'w': case 'x': + case 'y': case 'z': + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': + case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': + case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': + case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': + case 'Y': case 'Z': + case '0': case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': + case '*': case '/': case '@': case '.': case '-': case '_': + case ':': case '$': + pos++; + break; + default: + return bufferSlice(oldPos, pos); + } + } + return bufferSlice(oldPos, pos); + } + + /** + * Scans a word or keyword. + * + * ON ENTRY: 'pos' is 1 + the index of the first char in the word. + * ON EXIT: 'pos' is 1 + the index of the last char in the word. + * + * @return the word or keyword token. + */ + private Token wordOrKeyword() { + String word = scanWord(); + TokenKind kind = getTokenKindForWord(word); + return kind == TokenKind.WORD ? new Token(word) : new Token(kind); + } + + /** + * Performs tokenization of the character buffer of file contents provided to + * the constructor. + */ + private void tokenize() throws QueryException { + while (pos < buffer.length) { + char c = buffer[pos]; + pos++; + switch (c) { + case '(': { + addToken(new Token(TokenKind.LPAREN)); + break; + } + case ')': { + addToken(new Token(TokenKind.RPAREN)); + break; + } + case ',': { + addToken(new Token(TokenKind.COMMA)); + break; + } + case '+': { + addToken(new Token(TokenKind.PLUS)); + break; + } + case '-': { + addToken(new Token(TokenKind.MINUS)); + break; + } + case '=': { + addToken(new Token(TokenKind.EQUALS)); + break; + } + case '^': { + addToken(new Token(TokenKind.CARET)); + break; + } + case '\n': + case ' ': + case '\t': + case '\r': { + /* ignore */ + break; + } + case '\'': + case '\"': { + addToken(quotedWord(c)); + break; + } + default: { + addToken(wordOrKeyword()); + break; + } // default + } // switch + } // while + + addToken(new Token(TokenKind.EOF)); + + this.buffer = null; // release buffer now that we have our tokens + } + + private String bufferSlice(int start, int end) { + return new String(this.buffer, start, end - start); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java new file mode 100644 index 0000000..46a7afd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java
@@ -0,0 +1,351 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.collect.ImmutableList; + +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nonnull; + +/** + * The environment of a Blaze query. Implementations do not need to be thread-safe. The generic type + * T represents a node of the graph on which the query runs; as such, there is no restriction on T. + * However, query assumes a certain graph model, and the {@link TargetAccessor} class is used to + * access properties of these nodes. + * + * @param <T> the node type of the dependency graph + */ +public interface QueryEnvironment<T> { + /** + * Type of an argument of a user-defined query function. + */ + public enum ArgumentType { + EXPRESSION, WORD, INTEGER; + } + + /** + * Value of an argument of a user-defined query function. + */ + public static class Argument { + private final ArgumentType type; + private final QueryExpression expression; + private final String word; + private final int integer; + + private Argument(ArgumentType type, QueryExpression expression, String word, int integer) { + this.type = type; + this.expression = expression; + this.word = word; + this.integer = integer; + } + + static Argument of(QueryExpression expression) { + return new Argument(ArgumentType.EXPRESSION, expression, null, 0); + } + + static Argument of(String word) { + return new Argument(ArgumentType.WORD, null, word, 0); + } + + static Argument of(int integer) { + return new Argument(ArgumentType.INTEGER, null, null, integer); + } + + public ArgumentType getType() { + return type; + } + + public QueryExpression getExpression() { + return expression; + } + + public String getWord() { + return word; + } + + public int getInteger() { + return integer; + } + + @Override + public String toString() { + switch (type) { + case WORD: return "'" + word + "'"; + case EXPRESSION: return expression.toString(); + case INTEGER: return Integer.toString(integer); + default: throw new IllegalStateException(); + } + } + } + + /** + * A user-defined query function. + */ + public interface QueryFunction { + /** + * Name of the function as it appears in the query language. + */ + String getName(); + + /** + * The number of arguments that are required. The rest is optional. + * + * <p>This should be greater than or equal to zero and at smaller than or equal to the length + * of the list returned by {@link #getArgumentTypes}. + */ + int getMandatoryArguments(); + + /** + * The types of the arguments of the function. + */ + List<ArgumentType> getArgumentTypes(); + + /** + * Called when a user-defined function is to be evaluated. + * + * @param env the query environment this function is evaluated in. + * @param expression the expression being evaluated. + * @param args the input arguments. These are type-checked against the specification returned + * by {@link #getArgumentTypes} and {@link #getMandatoryArguments} + */ + <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args) + throws QueryException; + } + + /** + * Exception type for the case where a target cannot be found. It's basically a wrapper for + * whatever exception is internally thrown. + */ + public static final class TargetNotFoundException extends Exception { + public TargetNotFoundException(String msg) { + super(msg); + } + + public TargetNotFoundException(Throwable cause) { + super(cause.getMessage(), cause); + } + } + + /** + * Returns the set of target nodes in the graph for the specified target + * pattern, in 'blaze build' syntax. + */ + Set<T> getTargetsMatchingPattern(QueryExpression owner, String pattern) + throws QueryException; + + /** Ensures the specified target exists. */ + // NOTE(bazel-team): this method is left here as scaffolding from a previous refactoring. It may + // be possible to remove it. + T getOrCreate(T target); + + /** Returns the direct forward dependencies of the specified target. */ + Collection<T> getFwdDeps(T target); + + /** Returns the direct reverse dependencies of the specified target. */ + Collection<T> getReverseDeps(T target); + + /** + * Returns the forward transitive closure of all of the targets in + * "targets". Callers must ensure that {@link #buildTransitiveClosure} + * has been called for the relevant subgraph. + */ + Set<T> getTransitiveClosure(Set<T> targets); + + /** + * Construct the dependency graph for a depth-bounded forward transitive closure + * of all nodes in "targetNodes". The identity of the calling expression is + * required to produce error messages. + * + * <p>If a larger transitive closure was already built, returns it to + * improve incrementality, since all depth-constrained methods filter it + * after it is built anyway. + */ + void buildTransitiveClosure(QueryExpression caller, + Set<T> targetNodes, + int maxDepth) throws QueryException; + + /** + * Returns the set of nodes on some path from "from" to "to". + */ + Set<T> getNodesOnPath(T from, T to); + + /** + * Returns the value of the specified variable, or null if it is undefined. + */ + Set<T> getVariable(String name); + + /** + * Sets the value of the specified variable. If value is null the variable + * becomes undefined. Returns the previous value, if any. + */ + Set<T> setVariable(String name, Set<T> value); + + void reportBuildFileError(QueryExpression expression, String msg) throws QueryException; + + /** + * Returns the set of BUILD, included, sub-included and Skylark files that define the given set of + * targets. Each such file is itself represented as a target in the result. + */ + Set<T> getBuildFiles(QueryExpression caller, Set<T> nodes) throws QueryException; + + /** + * Returns an object that can be used to query information about targets. Implementations should + * create a single instance and return that for all calls. A class can implement both {@code + * QueryEnvironment} and {@code TargetAccessor} at the same time, in which case this method simply + * returns {@code this}. + */ + TargetAccessor<T> getAccessor(); + + /** + * Whether the given setting is enabled. The code should default to return {@code false} for all + * unknown settings. The enum is used rather than a method for each setting so that adding more + * settings is backwards-compatible. + * + * @throws NullPointerException if setting is null + */ + boolean isSettingEnabled(@Nonnull Setting setting); + + /** + * Returns the set of query functions implemented by this query environment. + */ + Iterable<QueryFunction> getFunctions(); + + /** + * Settings for the query engine. See {@link QueryEnvironment#isSettingEnabled}. + */ + public static enum Setting { + + /** + * Whether to evaluate tests() expressions in strict mode. If {@link #isSettingEnabled} returns + * true for this setting, then the tests() expression will give an error when expanding tests + * suites, if the test suite contains any non-test targets. + */ + TESTS_EXPRESSION_STRICT, + + /** + * Do not consider implicit deps (any label that was not explicitly specified in the BUILD file) + * when traversing dependency edges. + */ + NO_IMPLICIT_DEPS, + + /** + * Do not consider host dependencies when traversing dependency edges. + */ + NO_HOST_DEPS, + + /** + * Do not consider nodep attributes when traversing dependency edges. + */ + NO_NODEP_DEPS; + } + + /** + * An adapter interface giving access to properties of T. There are four types of targets: rules, + * package groups, source files, and generated files. Of these, only rules can have attributes. + */ + public static interface TargetAccessor<T> { + /** + * Returns the target type represented as a string of the form {@code <type> rule} or + * {@code package group} or {@code source file} or {@code generated file}. This is widely used + * for target filtering, so implementations must use the Blaze rule class naming scheme. + */ + String getTargetKind(T target); + + /** + * Returns the full label of the target as a string, e.g. {@code //some:target}. + */ + String getLabel(T target); + + /** + * Returns whether the given target is a rule. + */ + boolean isRule(T target); + + /** + * Returns whether the given target is a test target. If this returns true, then {@link #isRule} + * must also return true for the target. + */ + boolean isTestRule(T target); + + /** + * Returns whether the given target is a test suite target. If this returns true, then {@link + * #isRule} must also return true for the target, but {@link #isTestRule} must return false; + * test suites are not test rules, and vice versa. + */ + boolean isTestSuite(T target); + + /** + * If the attribute of the given name on the given target is a label or label list, then this + * method returns the list of corresponding target instances. Otherwise returns an empty list. + * If an error occurs during resolution, it throws a {@link QueryException} using the caller and + * error message prefix. + * + * @throws IllegalArgumentException if target is not a rule (according to {@link #isRule}) + */ + List<T> getLabelListAttr(QueryExpression caller, T target, String attrName, + String errorMsgPrefix) throws QueryException; + + /** + * If the attribute of the given name on the given target is a string list, then this method + * returns it. + * + * @throws IllegalArgumentException if target is not a rule (according to {@link #isRule}), or + * if the target does not have an attribute of type string list + * with the given name + */ + List<String> getStringListAttr(T target, String attrName); + + /** + * If the attribute of the given name on the given target is a string, then this method returns + * it. + * + * @throws IllegalArgumentException if target is not a rule (according to {@link #isRule}), or + * if the target does not have an attribute of type string with + * the given name + */ + String getStringAttr(T target, String attrName); + + /** + * Returns the given attribute represented as a list of strings. For "normal" attributes, + * this should just be a list of size one containing the attribute's value. For configurable + * attributes, there should be one entry for each possible value the attribute may take. + * + *<p>Note that for backwards compatibility, tristate and boolean attributes are returned as + * int using the values {@code 0, 1} and {@code -1}. If there is no such attribute, this + * method returns an empty list. + * + * @throws IllegalArgumentException if target is not a rule (according to {@link #isRule}) + */ + Iterable<String> getAttrAsString(T target, String attrName); + } + + /** List of the default query functions. */ + public static final List<QueryFunction> DEFAULT_QUERY_FUNCTIONS = + ImmutableList.<QueryFunction>of( + new AllPathsFunction(), + new BuildFilesFunction(), + new AttrFunction(), + new FilterFunction(), + new LabelsFunction(), + new KindFunction(), + new SomeFunction(), + new SomePathFunction(), + new TestsFunction(), + new DepsFunction(), + new RdepsFunction() + ); +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEvalResult.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEvalResult.java new file mode 100644 index 0000000..5bcea7e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEvalResult.java
@@ -0,0 +1,51 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.base.Preconditions; + +import java.util.Set; + +/** + * The result of a query evaluation, containing a set of elements. + * + * @param <T> the node type of the elements. + */ +public class QueryEvalResult<T> { + + protected final boolean success; + protected final Set<T> resultSet; + + public QueryEvalResult( + boolean success, Set<T> resultSet) { + this.success = success; + this.resultSet = Preconditions.checkNotNull(resultSet); + } + + /** + * Whether the query was successful. This can only be false if the query was run with + * <code>keep_going</code>, otherwise evaluation will throw a {@link QueryException}. + */ + public boolean getSuccess() { + return success; + } + + /** + * Returns the result as a set of targets. + */ + public Set<T> getResultSet() { + return resultSet; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryException.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryException.java new file mode 100644 index 0000000..71c1a8a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryException.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +/** + */ +public class QueryException extends Exception { + + /** + * Returns a better error message for the query. + */ + static String describeFailedQuery(QueryException e, QueryExpression toplevel) { + QueryExpression badQuery = e.getFailedExpression(); + if (badQuery == null) { + return "Evaluation failed: " + e.getMessage(); + } + return badQuery == toplevel + ? "Evaluation of query \"" + toplevel + "\" failed: " + e.getMessage() + : "Evaluation of subquery \"" + badQuery + + "\" failed (did you want to use --keep_going?): " + e.getMessage(); + } + + private final QueryExpression expression; + + public QueryException(QueryException e, QueryExpression toplevel) { + super(describeFailedQuery(e, toplevel), e); + this.expression = null; + } + + public QueryException(QueryExpression expression, String message) { + super(message); + this.expression = expression; + } + + public QueryException(String message) { + this(null, message); + } + + /** + * Returns the subexpression for which evaluation failed, or null if + * the failure occurred during lexing/parsing. + */ + public QueryExpression getFailedExpression() { + return expression; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryExpression.java new file mode 100644 index 0000000..23603f1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryExpression.java
@@ -0,0 +1,83 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import java.util.Collection; +import java.util.Set; + +/** + * Base class for expressions in the Blaze query language, revision 2. + * + * <p>All queries return a subgraph of the dependency graph, represented + * as a set of target nodes. + * + * <p>All queries must ensure that sufficient graph edges are created in the + * QueryEnvironment so that all nodes in the result are correctly ordered + * according to the type of query. For example, "deps" queries require that + * all the nodes in the transitive closure of its argument set are correctly + * ordered w.r.t. each other; "somepath" queries require that the order of the + * nodes on the resulting path are correctly ordered; algebraic set operations + * such as intersect and union are inherently unordered. + * + * <h2>Package overview</h2> + * + * <p>This package consists of two basic class hierarchies. The first, {@code + * QueryExpression}, is the set of different query expressions in the language, + * and the {@link #eval} method of each defines the semantics. The result of + * evaluating a query is set of Blaze {@code Target}s (a file or rule). The + * set may be interpreted as either a set or as nodes of a DAG, depending on + * the context. + * + * <p>The second hierarchy is {@code OutputFormatter}. Its subclasses define + * different ways of printing out the result of a query. Each accepts a {@code + * Digraph} of {@code Target}s, and an output stream. + */ +public abstract class QueryExpression { + + /** + * Scan and parse the specified query expression. + */ + public static QueryExpression parse(String query, QueryEnvironment<?> env) + throws QueryException { + return QueryParser.parse(query, env); + } + + protected QueryExpression() {} + + /** + * Evaluates this query in the specified environment, and returns a subgraph, + * concretely represented a new (possibly-immutable) set of target nodes. + * + * Failures resulting from evaluation of an ill-formed query cause + * QueryException to be thrown. + * + * The reporting of failures arising from errors in BUILD files depends on + * the --keep_going flag. If enabled (the default), then QueryException is + * thrown. If disabled, evaluation will stumble on to produce a (possibly + * inaccurate) result, but a result nonetheless. + */ + public abstract <T> Set<T> eval(QueryEnvironment<T> env) throws QueryException; + + /** + * Collects all target patterns that are referenced anywhere within this query expression and adds + * them to the given collection, which must be mutable. + */ + public abstract void collectTargetPatterns(Collection<String> literals); + + /** + * Returns this query expression pretty-printed. + */ + @Override + public abstract String toString(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryParser.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryParser.java new file mode 100644 index 0000000..bcd89cc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryParser.java
@@ -0,0 +1,261 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import static com.google.devtools.build.lib.query2.engine.Lexer.BINARY_OPERATORS; + +import com.google.devtools.build.lib.query2.engine.Lexer.TokenKind; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * LL(1) recursive descent parser for the Blaze query language, revision 2. + * + * In the grammar below, non-terminals are lowercase and terminals are + * uppercase, or character literals. + * + * <pre> + * expr ::= WORD + * | LET WORD = expr IN expr + * | '(' expr ')' + * | WORD '(' expr ( ',' expr ) * ')' + * | expr INTERSECT expr + * | expr '^' expr + * | expr UNION expr + * | expr '+' expr + * | expr EXCEPT expr + * | expr '-' expr + * | SET '(' WORD * ')' + * </pre> + */ +final class QueryParser { + + private Lexer.Token token; // current lookahead token + private final List<Lexer.Token> tokens; + private final Iterator<Lexer.Token> tokenIterator; + private final Map<String, QueryFunction> functions; + + /** + * Scan and parse the specified query expression. + */ + static QueryExpression parse(String query, QueryEnvironment<?> env) throws QueryException { + QueryParser parser = new QueryParser( + Lexer.scan(query.toCharArray()), env); + QueryExpression expr = parser.parseExpression(); + if (parser.token.kind != TokenKind.EOF) { + throw new QueryException("unexpected token '" + parser.token + + "' after query expression '" + expr + "'"); + } + return expr; + } + + private QueryParser(List<Lexer.Token> tokens, QueryEnvironment<?> env) { + this.functions = new HashMap<>(); + for (QueryFunction queryFunction : env.getFunctions()) { + this.functions.put(queryFunction.getName(), queryFunction); + } + this.tokens = tokens; + this.tokenIterator = tokens.iterator(); + nextToken(); + } + + /** + * Returns an exception. Don't forget to throw it. + */ + private QueryException syntaxError(Lexer.Token token) { + String message = "premature end of input"; + if (token.kind != TokenKind.EOF) { + StringBuilder buf = new StringBuilder("syntax error at '"); + String sep = ""; + for (int index = tokens.indexOf(token), + max = Math.min(tokens.size() - 1, index + 3); // 3 tokens of context + index < max; ++index) { + buf.append(sep).append(tokens.get(index)); + sep = " "; + } + buf.append("'"); + message = buf.toString(); + } + return new QueryException(message); + } + + /** + * Consumes the current token. If it is not of the specified (expected) + * kind, throws QueryException. Returns the value associated with the + * consumed token, if any. + */ + private String consume(TokenKind kind) throws QueryException { + if (token.kind != kind) { + throw syntaxError(token); + } + String word = token.word; + nextToken(); + return word; + } + + /** + * Consumes the current token, which must be a WORD containing an integer + * literal. Returns that integer, or throws a QueryException otherwise. + */ + private int consumeIntLiteral() throws QueryException { + String intString = consume(TokenKind.WORD); + try { + return Integer.parseInt(intString); + } catch (NumberFormatException e) { + throw new QueryException("expected an integer literal: '" + intString + "'"); + } + } + + private void nextToken() { + if (token == null || token.kind != TokenKind.EOF) { + token = tokenIterator.next(); + } + } + + /** + * expr ::= primary + * | expr INTERSECT expr + * | expr '^' expr + * | expr UNION expr + * | expr '+' expr + * | expr EXCEPT expr + * | expr '-' expr + */ + private QueryExpression parseExpression() throws QueryException { + // All operators are left-associative and of equal precedence. + return parseBinaryOperatorTail(parsePrimary()); + } + + /** + * tail ::= ( <op> <primary> )* + * All operators have equal precedence. + * This factoring is required for left-associative binary operators in LL(1). + */ + private QueryExpression parseBinaryOperatorTail(QueryExpression lhs) throws QueryException { + if (!BINARY_OPERATORS.contains(token.kind)) { + return lhs; + } + + List<QueryExpression> operands = new ArrayList<>(); + operands.add(lhs); + TokenKind lastOperator = token.kind; + + while (BINARY_OPERATORS.contains(token.kind)) { + TokenKind operator = token.kind; + consume(operator); + if (operator != lastOperator) { + lhs = new BinaryOperatorExpression(lastOperator, operands); + operands.clear(); + operands.add(lhs); + lastOperator = operator; + } + QueryExpression rhs = parsePrimary(); + operands.add(rhs); + } + return new BinaryOperatorExpression(lastOperator, operands); + } + + /** + * primary ::= WORD + * | LET WORD = expr IN expr + * | '(' expr ')' + * | WORD '(' expr ( ',' expr ) * ')' + * | DEPS '(' expr ')' + * | DEPS '(' expr ',' WORD ')' + * | RDEPS '(' expr ',' expr ')' + * | RDEPS '(' expr ',' expr ',' WORD ')' + * | SET '(' WORD * ')' + */ + private QueryExpression parsePrimary() throws QueryException { + switch (token.kind) { + case WORD: { + String word = consume(TokenKind.WORD); + if (token.kind == TokenKind.LPAREN) { + QueryFunction function = functions.get(word); + if (function == null) { + throw syntaxError(token); + } + List<Argument> args = new ArrayList<>(); + TokenKind tokenKind = TokenKind.LPAREN; + int argsSeen = 0; + for (ArgumentType type : function.getArgumentTypes()) { + if (token.kind == TokenKind.RPAREN && argsSeen >= function.getMandatoryArguments()) { + break; + } + + consume(tokenKind); + tokenKind = TokenKind.COMMA; + switch (type) { + case EXPRESSION: + args.add(Argument.of(parseExpression())); + break; + + case WORD: + args.add(Argument.of(consume(TokenKind.WORD))); + break; + + case INTEGER: + args.add(Argument.of(consumeIntLiteral())); + break; + + default: + throw new IllegalStateException(); + } + + argsSeen++; + } + + consume(TokenKind.RPAREN); + return new FunctionExpression(function, args); + } else { + return new TargetLiteral(word); + } + } + case LET: { + consume(TokenKind.LET); + String name = consume(TokenKind.WORD); + consume(TokenKind.EQUALS); + QueryExpression varExpr = parseExpression(); + consume(TokenKind.IN); + QueryExpression bodyExpr = parseExpression(); + return new LetExpression(name, varExpr, bodyExpr); + } + case LPAREN: { + consume(TokenKind.LPAREN); + QueryExpression expr = parseExpression(); + consume(TokenKind.RPAREN); + return expr; + } + case SET: { + nextToken(); + consume(TokenKind.LPAREN); + List<TargetLiteral> words = new ArrayList<>(); + while (token.kind == TokenKind.WORD) { + words.add(new TargetLiteral(consume(TokenKind.WORD))); + } + consume(TokenKind.RPAREN); + return new SetExpression(words); + } + default: + throw syntaxError(token); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/RdepsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/RdepsFunction.java new file mode 100644 index 0000000..6a8734b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/RdepsFunction.java
@@ -0,0 +1,99 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * An "rdeps" query expression, which computes the reverse dependencies of the argument within the + * transitive closure of the universe. An optional integer-literal third argument may be + * specified; its value bounds the search from the arguments. + * + * <pre>expr ::= RDEPS '(' expr ',' expr ')'</pre> + * <pre> | RDEPS '(' expr ',' expr ',' WORD ')'</pre> + */ +final class RdepsFunction implements QueryFunction { + RdepsFunction() { + } + + @Override + public String getName() { + return "rdeps"; + } + + @Override + public int getMandatoryArguments() { + return 2; // last argument is optional + } + + @Override + public List<ArgumentType> getArgumentTypes() { + return ImmutableList.of( + ArgumentType.EXPRESSION, ArgumentType.EXPRESSION, ArgumentType.INTEGER); + } + + /** + * Compute the transitive closure of the universe, then breadth-first search from the argument + * towards the universe while staying within the transitive closure. + */ + @Override + public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args) + throws QueryException { + Set<T> universeValue = args.get(0).getExpression().eval(env); + Set<T> argumentValue = args.get(1).getExpression().eval(env); + int depthBound = args.size() > 2 ? args.get(2).getInteger() : Integer.MAX_VALUE; + + env.buildTransitiveClosure(expression, universeValue, Integer.MAX_VALUE); + + Set<T> visited = new LinkedHashSet<>(); + Set<T> reachableFromUniverse = env.getTransitiveClosure(universeValue); + Collection<T> current = argumentValue; + + // We need to iterate depthBound + 1 times. + for (int i = 0; i <= depthBound; i++) { + List<T> next = new ArrayList<>(); + for (T node : current) { + if (!reachableFromUniverse.contains(node)) { + // Traversed outside the transitive closure of the universe. + continue; + } + + if (!visited.add(node)) { + // Already visited; if we see a node in a later round, then we don't need to visit it + // again, because the depth at which we see it at must be greater than or equal to the + // last visit. + continue; + } + + next.addAll(env.getReverseDeps(node)); + } + if (next.isEmpty()) { + // Exit when there are no more nodes to visit. + break; + } + current = next; + } + + return visited; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java new file mode 100644 index 0000000..1dbe5e6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java
@@ -0,0 +1,83 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * An abstract class that provides generic regex filter expression. Actual + * expression are implemented by the subclasses. + */ +abstract class RegexFilterExpression implements QueryFunction { + protected RegexFilterExpression() { + } + + @Override + public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args) + throws QueryException { + Pattern compiledPattern; + try { + compiledPattern = Pattern.compile(getPattern(args)); + } catch (IllegalArgumentException e) { + throw new QueryException(expression, "illegal pattern regexp in '" + this + "': " + + e.getMessage()); + } + + QueryExpression argument = args.get(args.size() - 1).getExpression(); + + Set<T> result = new LinkedHashSet<>(); + for (T target : argument.eval(env)) { + for (String str : getFilterStrings(env, args, target)) { + if ((str != null) && compiledPattern.matcher(str).find()) { + result.add(target); + break; + } + } + } + return result; + } + + /** + * Returns string for the given target that must be matched against pattern. + * May return null, in which case matching is guaranteed to fail. + */ + protected abstract <T> String getFilterString( + QueryEnvironment<T> env, List<Argument> args, T target); + + /** + * Returns a list of strings for the given target that must be matched against + * pattern. The filter matches if *any* of these strings matches. + * + * <p>Unless subclasses have an explicit reason to override this method, it's fine + * to keep the default implementation that just delegates to {@link #getFilterString}. + * Overriding this method is useful for subclasses that want to match against a + * universe of possible values. For example, with configurable attributes, an + * attribute might have different values depending on the build configuration. One + * may wish the filter to match if *any* of those values matches. + */ + protected <T> Iterable<String> getFilterStrings( + QueryEnvironment<T> env, List<Argument> args, T target) { + String filterString = getFilterString(env, args, target); + return filterString == null ? ImmutableList.<String>of() : ImmutableList.of(filterString); + } + + protected abstract String getPattern(List<Argument> args); +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SetExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SetExpression.java new file mode 100644 index 0000000..a28c679 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SetExpression.java
@@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.base.Joiner; + +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * A set(word, ..., word) expression, which computes the union of zero or more + * target patterns separated by whitespace. This is intended to support the + * use-case in which a set of labels written to a file by a previous query + * expression can be modified externally, then used as input to another query, + * like so: + * + * <pre> + * % blaze query 'somepath(foo, bar)' | grep ... | sed ... | awk ... >file + * % blaze query "kind(qux_library, set($(<file)))" + * </pre> + * + * <p>The grammar currently restricts the operands of set() to being zero or + * more words (target patterns), with no intervening punctuation. In principle + * this could be extended to arbitrary expressions without grammatical + * ambiguity, but this seems excessively general for now. + * + * <pre>expr ::= SET '(' WORD * ')'</pre> + */ +class SetExpression extends QueryExpression { + + private final List<TargetLiteral> words; + + SetExpression(List<TargetLiteral> words) { + this.words = words; + } + + @Override + public <T> Set<T> eval(QueryEnvironment<T> env) throws QueryException { + Set<T> result = new LinkedHashSet<>(); + for (TargetLiteral expr : words) { + result.addAll(expr.eval(env)); + } + return result; + } + + @Override + public void collectTargetPatterns(Collection<String> literals) { + for (TargetLiteral expr : words) { + expr.collectTargetPatterns(literals); + } + } + + @Override + public String toString() { + return "set(" + Joiner.on(' ').join(words) + ")"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SkyframeRestartQueryException.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SkyframeRestartQueryException.java new file mode 100644 index 0000000..d720ec9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SkyframeRestartQueryException.java
@@ -0,0 +1,24 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +/** + * This exception is thrown when a query operation was unable to complete because of a Skyframe + * missing dependency. + */ +public class SkyframeRestartQueryException extends RuntimeException { + public SkyframeRestartQueryException() { + super("need skyframe retry. missing dep"); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java new file mode 100644 index 0000000..384b474 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java
@@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; + +import java.util.List; +import java.util.Set; + +/** + * A some(x) filter expression, which returns an arbitrary node in set x, or + * fails if x is empty. + * + * <pre>expr ::= SOME '(' expr ')'</pre> + */ +class SomeFunction implements QueryFunction { + SomeFunction() { + } + + @Override + public String getName() { + return "some"; + } + + @Override + public int getMandatoryArguments() { + return 1; + } + + @Override + public List<ArgumentType> getArgumentTypes() { + return ImmutableList.of(ArgumentType.EXPRESSION); + } + + @Override + public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args) + throws QueryException { + Set<T> argumentValue = args.get(0).getExpression().eval(env); + if (argumentValue.isEmpty()) { + throw new QueryException(expression, "argument set is empty"); + } + return ImmutableSet.of(argumentValue.iterator().next()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java new file mode 100644 index 0000000..b90bcdf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java
@@ -0,0 +1,87 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.common.collect.Sets.SetView; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * A somepath(x, y) query expression, which computes the set of nodes + * on some arbitrary path from a target in set x to a target in set y. + * + * <pre>expr ::= SOMEPATH '(' expr ',' expr ')'</pre> + */ +class SomePathFunction implements QueryFunction { + SomePathFunction() { + } + + @Override + public String getName() { + return "somepath"; + } + + @Override + public int getMandatoryArguments() { + return 2; + } + + @Override + public List<ArgumentType> getArgumentTypes() { + return ImmutableList.of(ArgumentType.EXPRESSION, ArgumentType.EXPRESSION); + } + + @Override + public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args) + throws QueryException { + Set<T> fromValue = args.get(0).getExpression().eval(env); + Set<T> toValue = args.get(1).getExpression().eval(env); + + // Implementation strategy: for each x in "from", compute its forward + // transitive closure. If it intersects "to", then do a path search from x + // to an arbitrary node in the intersection, and return the path. This + // avoids computing the full transitive closure of "from" in some cases. + + env.buildTransitiveClosure(expression, fromValue, Integer.MAX_VALUE); + + // This set contains all nodes whose TC does not intersect "toValue". + Set<T> done = new HashSet<>(); + + for (T x : fromValue) { + if (done.contains(x)) { + continue; + } + Set<T> xtc = env.getTransitiveClosure(ImmutableSet.of(x)); + SetView<T> result; + if (xtc.size() > toValue.size()) { + result = Sets.intersection(toValue, xtc); + } else { + result = Sets.intersection(xtc, toValue); + } + if (!result.isEmpty()) { + return env.getNodesOnPath(x, result.iterator().next()); + } + done.addAll(xtc); + } + return ImmutableSet.of(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/TargetLiteral.java b/src/main/java/com/google/devtools/build/lib/query2/engine/TargetLiteral.java new file mode 100644 index 0000000..b6a57cc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/TargetLiteral.java
@@ -0,0 +1,72 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.base.Preconditions; + +import java.util.Collection; +import java.util.Set; + +/** + * A literal set of targets, using 'blaze build' syntax. Or, a reference to a + * variable name. (The syntax of the string "pattern" determines which.) + * + * TODO(bazel-team): Perhaps we should distinguish NAME from WORD in the parser, + * based on the characters in it? Also, perhaps we should not allow NAMEs to + * be quoted like WORDs can be. + * + * <pre>expr ::= NAME | WORD</pre> + */ +final class TargetLiteral extends QueryExpression { + + private final String pattern; + + TargetLiteral(String pattern) { + this.pattern = Preconditions.checkNotNull(pattern); + } + + public boolean isVariableReference() { + return LetExpression.isValidVarReference(pattern); + } + + @Override + public <T> Set<T> eval(QueryEnvironment<T> env) throws QueryException { + if (isVariableReference()) { + String varName = LetExpression.getNameFromReference(pattern); + Set<T> value = env.getVariable(varName); + if (value == null) { + throw new QueryException(this, "undefined variable '" + varName + "'"); + } + return env.getVariable(varName); + } + + return env.getTargetsMatchingPattern(this, pattern); + } + + @Override + public void collectTargetPatterns(Collection<String> literals) { + if (!isVariableReference()) { + literals.add(pattern); + } + } + + @Override + public String toString() { + // Keep predicate consistent with Lexer.scanWord! + boolean needsQuoting = Lexer.isReservedWord(pattern) + || pattern.isEmpty() + || "$-*".indexOf(pattern.charAt(0)) != -1; + return needsQuoting ? ("\"" + pattern + "\"") : pattern; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java new file mode 100644 index 0000000..c902609 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java
@@ -0,0 +1,257 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.engine; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Setting; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * A tests(x) filter expression, which returns all the tests in set x, + * expanding test_suite rules into their constituents. + * + * <p>Unfortunately this class reproduces a substantial amount of logic from + * {@code TestSuiteConfiguredTarget}, albeit in a somewhat simplified form. + * This is basically inevitable since the expansion of test_suites cannot be + * done during the loading phase, because it involves inter-package references. + * We make no attempt to validate the input, or report errors or warnings other + * than missing target. + * + * <pre>expr ::= TESTS '(' expr ')'</pre> + */ +class TestsFunction implements QueryFunction { + TestsFunction() { + } + + @Override + public String getName() { + return "tests"; + } + + @Override + public int getMandatoryArguments() { + return 1; + } + + @Override + public List<ArgumentType> getArgumentTypes() { + return ImmutableList.of(ArgumentType.EXPRESSION); + } + + @Override + public <T> Set<T> eval(QueryEnvironment<T> env, QueryExpression expression, List<Argument> args) + throws QueryException { + Closure<T> closure = new Closure<>(expression, env); + Set<T> result = new HashSet<>(); + for (T target : args.get(0).getExpression().eval(env)) { + if (env.getAccessor().isTestRule(target)) { + result.add(target); + } else if (env.getAccessor().isTestSuite(target)) { + for (T test : closure.getTestsInSuite(target)) { + result.add(env.getOrCreate(test)); + } + } + } + return result; + } + + /** + * Decides whether to include a test in a test_suite or not. + * @param testTags Collection of all tags exhibited by a given test. + * @param positiveTags Tags declared by the suite. A test must match ALL of these. + * @param negativeTags Tags declared by the suite. A test must match NONE of these. + * @return false is the test is to be removed. + */ + private static boolean includeTest(Collection<String> testTags, + Collection<String> positiveTags, Collection<String> negativeTags) { + // Add this test if it matches ALL of the positive tags and NONE of the + // negative tags in the tags attribute. + for (String tag : negativeTags) { + if (testTags.contains(tag)) { + return false; + } + } + for (String tag : positiveTags) { + if (!testTags.contains(tag)) { + return false; + } + } + return true; + } + + /** + * Separates a list of text "tags" into a Pair of Collections, where + * the first element are the required or positive tags and the second element + * are the excluded or negative tags. + * This should work on tag list provided from the command line + * --test_tags_filters flag or on tag filters explicitly declared in the + * suite. + * + * Keep this function in sync with the version in + * java.com.google.devtools.build.lib.view.packages.TestTargetUtils.sortTagsBySense + * + * @param tagList A collection of text tags to separate. + */ + private static void sortTagsBySense( + Collection<String> tagList, Set<String> requiredTags, Set<String> excludedTags) { + for (String tag : tagList) { + if (tag.startsWith("-")) { + excludedTags.add(tag.substring(1)); + } else if (tag.startsWith("+")) { + requiredTags.add(tag.substring(1)); + } else if (tag.equals("manual")) { + // Ignore manual attribute because it is an exception: it is not a filter + // but a property of test_suite + continue; + } else { + requiredTags.add(tag); + } + } + } + + /** + * A closure over the temporary state needed to compute the expression. This makes the evaluation + * thread-safe, as long as instances of this class are used only within a single thread. + */ + private final class Closure<T> { + private final QueryExpression expression; + /** A dynamically-populated mapping from test_suite rules to their tests. */ + private final Map<T, Set<T>> testsInSuite = new HashMap<>(); + + /** The environment in which this query is being evaluated. */ + private final QueryEnvironment<T> env; + + private final boolean strict; + + private Closure(QueryExpression expression, QueryEnvironment<T> env) { + this.expression = expression; + this.env = env; + this.strict = env.isSettingEnabled(Setting.TESTS_EXPRESSION_STRICT); + } + + /** + * Computes and returns the set of test rules in a particular suite. Uses + * dynamic programming---a memoized version of {@link #computeTestsInSuite}. + * + * @precondition env.getAccessor().isTestSuite(testSuite) + */ + private Set<T> getTestsInSuite(T testSuite) throws QueryException { + Set<T> tests = testsInSuite.get(testSuite); + if (tests == null) { + tests = Sets.newHashSet(); + testsInSuite.put(testSuite, tests); // break cycles by inserting empty set early. + computeTestsInSuite(testSuite, tests); + } + return tests; + } + + /** + * Populates 'result' with all the tests associated with the specified + * 'testSuite'. Throws an exception if any target is missing. + * + * <p>CAUTION! Keep this logic consistent with {@code TestsSuiteConfiguredTarget}! + * + * @precondition env.getAccessor().isTestSuite(testSuite) + */ + private void computeTestsInSuite(T testSuite, Set<T> result) throws QueryException { + List<T> testsAndSuites = new ArrayList<>(); + // Note that testsAndSuites can contain input file targets; the test_suite rule does not + // restrict the set of targets that can appear in tests or suites. + testsAndSuites.addAll(getPrerequisites(testSuite, "tests")); + testsAndSuites.addAll(getPrerequisites(testSuite, "suites")); + + // 1. Add all tests + for (T test : testsAndSuites) { + if (env.getAccessor().isTestRule(test)) { + result.add(test); + } else if (strict && !env.getAccessor().isTestSuite(test)) { + // If strict mode is enabled, then give an error for any non-test, non-test-suite targets. + env.reportBuildFileError(expression, "The label '" + + env.getAccessor().getLabel(test) + "' in the test_suite '" + + env.getAccessor().getLabel(testSuite) + "' does not refer to a test or test_suite " + + "rule!"); + } + } + + // 2. Add implicit dependencies on tests in same package, if any. + for (T target : getPrerequisites(testSuite, "$implicit_tests")) { + // The Package construction of $implicit_tests ensures that this check never fails, but we + // add it here anyway for compatibility with future code. + if (env.getAccessor().isTestRule(target)) { + result.add(target); + } + } + + // 3. Filter based on tags, size, env. + filterTests(testSuite, result); + + // 4. Expand all suites recursively. + for (T suite : testsAndSuites) { + if (env.getAccessor().isTestSuite(suite)) { + result.addAll(getTestsInSuite(suite)); + } + } + } + + /** + * Returns the set of rules named by the attribute 'attrName' of test_suite rule 'testSuite'. + * The attribute must be a list of labels. If a target cannot be resolved, then an error is + * reported to the environment (which may throw an exception if {@code keep_going} is disabled). + * + * @precondition env.getAccessor().isTestSuite(testSuite) + */ + private List<T> getPrerequisites(T testSuite, String attrName) throws QueryException { + return env.getAccessor().getLabelListAttr(expression, testSuite, attrName, + "couldn't expand '" + attrName + + "' attribute of test_suite " + env.getAccessor().getLabel(testSuite) + ": "); + } + + /** + * Filters 'tests' (by mutation) according to the 'tags' attribute, specifically those that + * match ALL of the tags in tagsAttribute. + * + * @precondition {@code env.getAccessor().isTestSuite(testSuite)} + * @precondition {@code env.getAccessor().isTestRule(test)} for all test in tests + */ + private void filterTests(T testSuite, Set<T> tests) { + List<String> tagsAttribute = env.getAccessor().getStringListAttr(testSuite, "tags"); + // Split the tags list into positive and negative tags + Set<String> requiredTags = new HashSet<>(); + Set<String> excludedTags = new HashSet<>(); + sortTagsBySense(tagsAttribute, requiredTags, excludedTags); + + Iterator<T> it = tests.iterator(); + while (it.hasNext()) { + T test = it.next(); + List<String> testTags = new ArrayList<>(env.getAccessor().getStringListAttr(test, "tags")); + testTags.add(env.getAccessor().getStringAttr(test, "size")); + if (!includeTest(testTags, requiredTags, excludedTags)) { + it.remove(); + } + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/GraphOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/output/GraphOutputFormatter.java new file mode 100644 index 0000000..5ded5e2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/output/GraphOutputFormatter.java
@@ -0,0 +1,174 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.output; + +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.collect.EquivalenceRelation; +import com.google.devtools.build.lib.graph.Digraph; +import com.google.devtools.build.lib.graph.DotOutputVisitor; +import com.google.devtools.build.lib.graph.LabelSerializer; +import com.google.devtools.build.lib.graph.Node; +import com.google.devtools.build.lib.packages.Target; + +import java.io.PrintStream; +import java.io.PrintWriter; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +/** + * An output formatter that prints the result as factored graph in AT&T + * GraphViz format. + */ +class GraphOutputFormatter extends OutputFormatter { + + private int graphNodeStringLimit; + private boolean graphFactored; + + @Override + public String getName() { + return "graph"; + } + + @Override + public void output(QueryOptions options, Digraph<Target> result, PrintStream out) { + this.graphNodeStringLimit = options.graphNodeStringLimit; + this.graphFactored = options.graphFactored; + + if (graphFactored) { + outputFactored(result, new PrintWriter(out)); + } else { + outputUnfactored(result, new PrintWriter(out)); + } + } + + private void outputUnfactored(Digraph<Target> result, PrintWriter out) { + result.visitNodesBeforeEdges( + new DotOutputVisitor<Target>(out, LABEL_STRINGIFIER) { + @Override + public void beginVisit() { + super.beginVisit(); + // TODO(bazel-team): (2009) make this the default in Digraph. + out.println(" node [shape=box];"); + } + }); + } + + private void outputFactored(Digraph<Target> result, PrintWriter out) { + EquivalenceRelation<Node<Target>> equivalenceRelation = createEquivalenceRelation(); + + // Notes on ordering: + // - Digraph.getNodes() returns nodes in no particular order + // - CollectionUtils.partition inserts elements into unordered sets + // This means partitions may contain nodes in a different order than perhaps expected. + // Example (package //foo): + // some_rule( + // name = 'foo', + // srcs = ['a', 'b', 'c'], + // ) + // Querying for deps('foo') will return (among others) the 'foo' node with successors 'a', 'b' + // and 'c' (in this order), however when asking the Digraph for all of its nodes, the returned + // collection may be ordered differently. + Collection<Set<Node<Target>>> partition = + CollectionUtils.partition(result.getNodes(), equivalenceRelation); + + Digraph<Set<Node<Target>>> factoredGraph = result.createImageUnderPartition(partition); + + // Concatenate the labels of all topologically-equivalent nodes. + LabelSerializer<Set<Node<Target>>> labelSerializer = new LabelSerializer<Set<Node<Target>>>() { + @Override + public String serialize(Node<Set<Node<Target>>> node) { + int actualLimit = graphNodeStringLimit - RESERVED_LABEL_CHARS; + boolean firstItem = true; + StringBuffer buf = new StringBuffer(); + int count = 0; + for (Node<Target> eqNode : node.getLabel()) { + String labelString = eqNode.getLabel().getLabel().toString(); + if (!firstItem) { + buf.append("\\n"); + + // Use -1 to denote no limit, as it is easier than trying to pass MAX_INT on the cmdline + if (graphNodeStringLimit != -1 && (buf.length() + labelString.length() > actualLimit)) { + buf.append("...and "); + buf.append(node.getLabel().size() - count); + buf.append(" more items"); + break; + } + } + + buf.append(labelString); + count++; + firstItem = false; + } + return buf.toString(); + } + }; + + factoredGraph.visitNodesBeforeEdges( + new DotOutputVisitor<Set<Node<Target>>>(out, labelSerializer) { + @Override + public void beginVisit() { + super.beginVisit(); + // TODO(bazel-team): (2009) make this the default in Digraph. + out.println(" node [shape=box];"); + } + }); + } + + /** + * Returns an equivalence relation for nodes in the specified graph. + * + * <p>Two nodes are considered equal iff they have equal topology (predecessors and successors). + * + * TODO(bazel-team): Make this a method of Digraph. + */ + private static <LABEL> EquivalenceRelation<Node<LABEL>> createEquivalenceRelation() { + return new EquivalenceRelation<Node<LABEL>>() { + @Override + public int compare(Node<LABEL> x, Node<LABEL> y) { + if (x == y) { + return 0; + } + + if (x.numPredecessors() != y.numPredecessors() + || x.numSuccessors() != y.numSuccessors()) { + return -1; + } + + Set<Node<LABEL>> xpred = new HashSet<>(x.getPredecessors()); + Set<Node<LABEL>> ypred = new HashSet<>(y.getPredecessors()); + if (!xpred.equals(ypred)) { + return -1; + } + + Set<Node<LABEL>> xsucc = new HashSet<>(x.getSuccessors()); + Set<Node<LABEL>> ysucc = new HashSet<>(y.getSuccessors()); + if (!xsucc.equals(ysucc)) { + return -1; + } + + return 0; + } + }; + } + + private static final int RESERVED_LABEL_CHARS = "\\n...and 9999999 more items".length(); + + private static final LabelSerializer<Target> LABEL_STRINGIFIER = new LabelSerializer<Target>() { + @Override + public String serialize(Node<Target> node) { + return node.getLabel().getLabel().toString(); + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/OutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/output/OutputFormatter.java new file mode 100644 index 0000000..b7a9d64 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/output/OutputFormatter.java
@@ -0,0 +1,486 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.output; + +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.graph.Digraph; +import com.google.devtools.build.lib.graph.Node; +import com.google.devtools.build.lib.packages.AggregatingAttributeMapper; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.EvalUtils; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.BinaryPredicate; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.common.options.EnumConverter; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Interface for classes which order, format and print the result of a Blaze + * graph query. + */ +public abstract class OutputFormatter { + + /** + * Discriminator for different kinds of OutputFormatter. + */ + public enum Type { + LABEL, + LABEL_KIND, + BUILD, + MINRANK, + MAXRANK, + PACKAGE, + LOCATION, + GRAPH, + XML, + PROTO, + RECORD, + } + + /** + * Where the value of an attribute comes from + */ + protected enum AttributeValueSource { + RULE, // Explicitly specified on the rule + PACKAGE, // Package default + DEFAULT // Rule class default + } + + public static final Function<Node<Target>, Target> EXTRACT_NODE_LABEL = + new Function<Node<Target>, Target>() { + @Override + public Target apply(Node<Target> input) { + return input.getLabel(); + } + }; + + /** + * Converter from strings to OutputFormatter.Type. + */ + public static class Converter extends EnumConverter<Type> { + public Converter() { super(Type.class, "output formatter"); } + } + + public static ImmutableList<OutputFormatter> getDefaultFormatters() { + return ImmutableList.of( + new LabelOutputFormatter(false), + new LabelOutputFormatter(true), + new BuildOutputFormatter(), + new MinrankOutputFormatter(), + new MaxrankOutputFormatter(), + new PackageOutputFormatter(), + new LocationOutputFormatter(), + new GraphOutputFormatter(), + new XmlOutputFormatter(), + new ProtoOutputFormatter()); + } + + public static String formatterNames(Iterable<OutputFormatter> formatters) { + return Joiner.on(", ").join(Iterables.transform(formatters, + new Function<OutputFormatter, String>() { + @Override + public String apply(OutputFormatter input) { + return input.getName(); + } + })); + } + + /** + * Returns the output formatter for the specified command-line options. + */ + public static OutputFormatter getFormatter( + Iterable<OutputFormatter> formatters, String type) { + for (OutputFormatter formatter : formatters) { + if (formatter.getName().equals(type)) { + return formatter; + } + } + + return null; + } + + /** + * Given a set of query options, returns a BinaryPredicate suitable for + * passing to {@link Rule#getLabels()}, {@link XmlOutputFormatter}, etc. + */ + public static BinaryPredicate<Rule, Attribute> getDependencyFilter(QueryOptions queryOptions) { + // TODO(bazel-team): Optimize: and(ALL_DEPS, x) -> x, etc. + return Rule.and( + queryOptions.includeHostDeps ? Rule.ALL_DEPS : Rule.NO_HOST_DEPS, + queryOptions.includeImplicitDeps ? Rule.ALL_DEPS : Rule.NO_IMPLICIT_DEPS); + } + + /** + * Format the result (a set of target nodes implicitly ordered according to + * the graph maintained by the QueryEnvironment), and print it to "out". + */ + public abstract void output(QueryOptions options, Digraph<Target> result, PrintStream out) + throws IOException; + + /** + * Unordered output formatter (wrt. dependency ordering). + * + * <p>Formatters that support unordered output may be used when only the set of query results is + * requested but their ordering is irrelevant. + * + * <p>The benefit of using a unordered formatter is that we can save the potentially expensive + * subgraph extraction step before presenting the query results. + */ + public interface UnorderedFormatter { + void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out) + throws IOException; + } + + /** + * Returns the user-visible name of the output formatter. + */ + public abstract String getName(); + + /** + * An output formatter that prints the labels of the resulting target set in + * topological order, optionally with the target's kind. + */ + private static class LabelOutputFormatter extends OutputFormatter implements UnorderedFormatter{ + + private final boolean showKind; + + public LabelOutputFormatter(boolean showKind) { + this.showKind = showKind; + } + + @Override + public String getName() { + return showKind ? "label_kind" : "label"; + } + + @Override + public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out) { + for (Target target : result) { + if (showKind) { + out.print(target.getTargetKind()); + out.print(' '); + } + out.println(target.getLabel()); + } + } + + @Override + public void output(QueryOptions options, Digraph<Target> result, PrintStream out) { + Iterable<Target> ordered = Iterables.transform( + result.getTopologicalOrder(new TargetOrdering()), EXTRACT_NODE_LABEL); + outputUnordered(options, ordered, out); + } + } + + /** + * An ordering of Targets based on the ordering of their labels. + */ + static class TargetOrdering implements Comparator<Target> { + @Override + public int compare(Target o1, Target o2) { + return o1.getLabel().compareTo(o2.getLabel()); + } + } + + /** + * An output formatter that prints the names of the packages of the target + * set, in lexicographical order without duplicates. + */ + private static class PackageOutputFormatter extends OutputFormatter implements + UnorderedFormatter { + @Override + public String getName() { + return "package"; + } + + @Override + public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out) { + Set<String> packageNames = Sets.newTreeSet(); + for (Target target : result) { + packageNames.add(target.getLabel().getPackageName()); + } + for (String packageName : packageNames) { + out.println(packageName); + } + } + + @Override + public void output(QueryOptions options, Digraph<Target> result, PrintStream out) { + Iterable<Target> ordered = Iterables.transform( + result.getTopologicalOrder(new TargetOrdering()), EXTRACT_NODE_LABEL); + outputUnordered(options, ordered, out); + } + } + + /** + * An output formatter that prints the labels of the targets, preceded by + * their locations and kinds, in topological order. For output files, the + * location of the generating rule is given; for input files, the location of + * line 1 is given. + */ + private static class LocationOutputFormatter extends OutputFormatter implements + UnorderedFormatter { + @Override + public String getName() { + return "location"; + } + + @Override + public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out) { + for (Target target : result) { + Location location = target.getLocation(); + out.println(location.print() + ": " + target.getTargetKind() + " " + target.getLabel()); + } + } + + @Override + public void output(QueryOptions options, Digraph<Target> result, PrintStream out) { + Iterable<Target> ordered = Iterables.transform( + result.getTopologicalOrder(new TargetOrdering()), EXTRACT_NODE_LABEL); + outputUnordered(options, ordered, out); + } + } + + /** + * An output formatter that prints the generating rules using the syntax of + * the BUILD files. If multiple targets are generated by the same rule, it is + * printed only once. + */ + private static class BuildOutputFormatter extends OutputFormatter implements UnorderedFormatter { + @Override + public String getName() { + return "build"; + } + + private void outputRule(Rule rule, PrintStream out) { + out.println(String.format("# %s", rule.getLocation())); + out.println(String.format("%s(", rule.getRuleClass())); + out.println(String.format(" name = \"%s\",", rule.getName())); + + for (Attribute attr : rule.getAttributes()) { + Pair<Iterable<Object>, AttributeValueSource> values = getAttributeValues(rule, attr); + if (Iterables.size(values.first) != 1) { + continue; // TODO(bazel-team): handle configurable attributes. + } + if (values.second != AttributeValueSource.RULE) { + continue; // Don't print default values. + } + Object value = Iterables.getOnlyElement(values.first); + out.print(String.format(" %s = ", attr.getName())); + if (value instanceof Label) { + value = value.toString(); + } else if (value instanceof List<?> && EvalUtils.isImmutable(value)) { + // Display it as a list (and not as a tuple). Attributes can never be tuples. + value = new ArrayList<>((List<?>) value); + } + EvalUtils.prettyPrintValue(value, out); + out.println(","); + } + out.println(String.format(")\n")); + } + + @Override + public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out) { + Set<Label> printed = new HashSet<>(); + for (Target target : result) { + Rule rule = target.getAssociatedRule(); + if (rule == null || printed.contains(rule.getLabel())) { + continue; + } + outputRule(rule, out); + printed.add(rule.getLabel()); + } + } + + @Override + public void output(QueryOptions options, Digraph<Target> result, PrintStream out) { + Iterable<Target> ordered = Iterables.transform( + result.getTopologicalOrder(new TargetOrdering()), EXTRACT_NODE_LABEL); + outputUnordered(options, ordered, out); + } + } + + /** + * An output formatter that prints the labels in minimum rank order, preceded by + * their rank number. "Roots" have rank 0, their direct prerequisites have + * rank 1, etc. All nodes in a cycle are considered of equal rank. MINRANK + * shows the lowest rank for a given node, i.e. the length of the shortest + * path from a zero-rank node to it. + * + * If the result came from a <code>deps(x)</code> query, then the MINRANKs + * correspond to the shortest path from x to each of its prerequisites. + */ + private static class MinrankOutputFormatter extends OutputFormatter { + @Override + public String getName() { + return "minrank"; + } + + @Override + public void output(QueryOptions options, Digraph<Target> result, PrintStream out) { + // getRoots() isn't defined for cyclic graphs, so in order to handle + // cycles correctly, we need work on the strong component graph, as + // cycles should be treated a "clump" of nodes all on the same rank. + // Graphs may contain cycles because there are errors in BUILD files. + + Digraph<Set<Node<Target>>> scGraph = result.getStrongComponentGraph(); + Set<Node<Set<Node<Target>>>> rankNodes = scGraph.getRoots(); + Set<Node<Set<Node<Target>>>> seen = new HashSet<>(); + seen.addAll(rankNodes); + for (int rank = 0; !rankNodes.isEmpty(); rank++) { + // Print out this rank: + for (Node<Set<Node<Target>>> xScc : rankNodes) { + for (Node<Target> x : xScc.getLabel()) { + out.println(rank + " " + x.getLabel().getLabel()); + } + } + + // Find the next rank: + Set<Node<Set<Node<Target>>>> nextRankNodes = new LinkedHashSet<>(); + for (Node<Set<Node<Target>>> x : rankNodes) { + for (Node<Set<Node<Target>>> y : x.getSuccessors()) { + if (seen.add(y)) { + nextRankNodes.add(y); + } + } + } + rankNodes = nextRankNodes; + } + } + } + + /** + * An output formatter that prints the labels in maximum rank order, preceded + * by their rank number. "Roots" have rank 0, all other nodes have a rank + * which is one greater than the maximum rank of each of their predecessors. + * All nodes in a cycle are considered of equal rank. MAXRANK shows the + * highest rank for a given node, i.e. the length of the longest non-cyclic + * path from a zero-rank node to it. + * + * If the result came from a <code>deps(x)</code> query, then the MAXRANKs + * correspond to the longest path from x to each of its prerequisites. + */ + private static class MaxrankOutputFormatter extends OutputFormatter { + @Override + public String getName() { + return "maxrank"; + } + + @Override + public void output(QueryOptions options, Digraph<Target> result, PrintStream out) { + // In order to handle cycles correctly, we need work on the strong + // component graph, as cycles should be treated a "clump" of nodes all on + // the same rank. Graphs may contain cycles because there are errors in BUILD files. + + // Dynamic programming algorithm: + // rank(x) = max(rank(p)) + 1 foreach p in preds(x) + // TODO(bazel-team): Move to Digraph. + class DP { + final Map<Node<Set<Node<Target>>>, Integer> ranks = new HashMap<>(); + + int rank(Node<Set<Node<Target>>> node) { + Integer rank = ranks.get(node); + if (rank == null) { + int maxPredRank = -1; + for (Node<Set<Node<Target>>> p : node.getPredecessors()) { + maxPredRank = Math.max(maxPredRank, rank(p)); + } + rank = maxPredRank + 1; + ranks.put(node, rank); + } + return rank; + } + } + DP dp = new DP(); + + // Now sort by rank... + List<Pair<Integer, Label>> output = new ArrayList<>(); + for (Node<Set<Node<Target>>> x : result.getStrongComponentGraph().getNodes()) { + int rank = dp.rank(x); + for (Node<Target> y : x.getLabel()) { + output.add(Pair.of(rank, y.getLabel().getLabel())); + } + } + Collections.sort(output, new Comparator<Pair<Integer, Label>>() { + @Override + public int compare(Pair<Integer, Label> x, Pair<Integer, Label> y) { + return x.first - y.first; + } + }); + + for (Pair<Integer, Label> pair : output) { + out.println(pair.first + " " + pair.second); + } + } + } + + /** + * Returns the possible values of the specified attribute in the specified rule. For + * non-configured attributes, this is a single value. For configurable attributes, this + * may be multiple values. + * + * <p>This is needed because the visibility attribute is replaced with an empty list + * during package loading if it is public or private in order not to visit + * the package called 'visibility'. + * + * @return a pair, where the first value is the set of possible values and the + * second is an enum that tells where the values come from (declared on the + * rule, declared as a package level default or a + * global default) + */ + protected static Pair<Iterable<Object>, AttributeValueSource> getAttributeValues( + Rule rule, Attribute attr) { + List<Object> values = new LinkedList<>(); // Not an ImmutableList: may host null values. + AttributeValueSource source; + + if (attr.getName().equals("visibility")) { + values.add(rule.getVisibility().getDeclaredLabels()); + if (rule.isVisibilitySpecified()) { + source = AttributeValueSource.RULE; + } else if (rule.getPackage().isDefaultVisibilitySet()) { + source = AttributeValueSource.PACKAGE; + } else { + source = AttributeValueSource.DEFAULT; + } + } else { + for (Object o : + AggregatingAttributeMapper.of(rule).visitAttribute(attr.getName(), attr.getType())) { + values.add(o); + } + source = rule.isAttributeValueExplicitlySpecified(attr) + ? AttributeValueSource.RULE : AttributeValueSource.DEFAULT; + } + + return Pair.of((Iterable<Object>) values, source); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/ProtoOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/output/ProtoOutputFormatter.java new file mode 100644 index 0000000..53fbb21 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/output/ProtoOutputFormatter.java
@@ -0,0 +1,491 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.output; + +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.DISTRIBUTIONS; +import static com.google.devtools.build.lib.packages.Type.FILESET_ENTRY_LIST; +import static com.google.devtools.build.lib.packages.Type.INTEGER; +import static com.google.devtools.build.lib.packages.Type.INTEGER_LIST; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST_DICT; +import static com.google.devtools.build.lib.packages.Type.LICENSE; +import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL; +import static com.google.devtools.build.lib.packages.Type.NODEP_LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.OUTPUT; +import static com.google.devtools.build.lib.packages.Type.OUTPUT_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING; +import static com.google.devtools.build.lib.packages.Type.STRING_DICT; +import static com.google.devtools.build.lib.packages.Type.STRING_DICT_UNARY; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST_DICT; +import static com.google.devtools.build.lib.packages.Type.TRISTATE; +import static com.google.devtools.build.lib.query2.proto.proto2api.Build.Target.Discriminator.GENERATED_FILE; +import static com.google.devtools.build.lib.query2.proto.proto2api.Build.Target.Discriminator.PACKAGE_GROUP; +import static com.google.devtools.build.lib.query2.proto.proto2api.Build.Target.Discriminator.RULE; +import static com.google.devtools.build.lib.query2.proto.proto2api.Build.Target.Discriminator.SOURCE_FILE; + +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.graph.Digraph; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.License; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.PackageGroup; +import com.google.devtools.build.lib.packages.ProtoUtils; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TriState; +import com.google.devtools.build.lib.query2.FakeSubincludeTarget; +import com.google.devtools.build.lib.query2.output.OutputFormatter.UnorderedFormatter; +import com.google.devtools.build.lib.query2.proto.proto2api.Build; +import com.google.devtools.build.lib.syntax.FilesetEntry; +import com.google.devtools.build.lib.syntax.GlobCriteria; +import com.google.devtools.build.lib.syntax.GlobList; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.util.BinaryPredicate; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * An output formatter that outputs a protocol buffer representation + * of a query result and outputs the proto bytes to the output print stream. + * By taking the bytes and calling {@code mergeFrom()} on a + * {@code Build.QueryResult} object the full result can be reconstructed. + */ +public class ProtoOutputFormatter extends OutputFormatter implements UnorderedFormatter { + + /** + * A special attribute name for the rule implementation hash code. + */ + public static final String RULE_IMPLEMENTATION_HASH_ATTR_NAME = "$rule_implementation_hash"; + + private BinaryPredicate<Rule, Attribute> dependencyFilter; + + protected void setDependencyFilter(QueryOptions options) { + this.dependencyFilter = OutputFormatter.getDependencyFilter(options); + } + + @Override + public String getName() { + return "proto"; + } + + @Override + public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out) + throws IOException { + setDependencyFilter(options); + + Build.QueryResult.Builder queryResult = Build.QueryResult.newBuilder(); + for (Target target : result) { + addTarget(queryResult, target); + } + + queryResult.build().writeTo(out); + } + + @Override + public void output(QueryOptions options, Digraph<Target> result, PrintStream out) + throws IOException { + outputUnordered(options, result.getLabels(), out); + } + + /** + * Add the target to the query result. + * @param queryResult The query result that contains all rule, input and + * output targets. + * @param target The query target being converted to a protocol buffer. + */ + private void addTarget(Build.QueryResult.Builder queryResult, Target target) { + queryResult.addTarget(toTargetProtoBuffer(target)); + } + + /** + * Converts a logical Target object into a Target protobuffer. + */ + protected Build.Target toTargetProtoBuffer(Target target) { + Build.Target.Builder targetPb = Build.Target.newBuilder(); + + String location = target.getLocation().print(); + if (target instanceof Rule) { + Rule rule = (Rule) target; + Build.Rule.Builder rulePb = Build.Rule.newBuilder() + .setName(rule.getLabel().toString()) + .setRuleClass(rule.getRuleClass()) + .setLocation(location); + + for (Attribute attr : rule.getAttributes()) { + addAttributeToProto(rulePb, attr, getAttributeValues(rule, attr).first, null, + rule.isAttributeValueExplicitlySpecified(attr), false); + } + + SkylarkEnvironment env = rule.getRuleClassObject().getRuleDefinitionEnvironment(); + if (env != null) { + // The RuleDefinitionEnvironment is always defined for Skylark rules and + // always null for non Skylark rules. + rulePb.addAttribute( + Build.Attribute.newBuilder() + .setName(RULE_IMPLEMENTATION_HASH_ATTR_NAME) + .setType(ProtoUtils.getDiscriminatorFromType( + com.google.devtools.build.lib.packages.Type.STRING)) + .setStringValue(env.getTransitiveFileContentHashCode())); + } + + // Include explicit elements for all direct inputs and outputs of a rule; + // this goes beyond what is available from the attributes above, since it + // may also (depending on options) include implicit outputs, + // host-configuration outputs, and default values. + for (Label label : rule.getLabels(dependencyFilter)) { + rulePb.addRuleInput(label.toString()); + } + for (OutputFile outputFile : rule.getOutputFiles()) { + Label fileLabel = outputFile.getLabel(); + rulePb.addRuleOutput(fileLabel.toString()); + } + for (String feature : rule.getFeatures()) { + rulePb.addDefaultSetting(feature); + } + + targetPb.setType(RULE); + targetPb.setRule(rulePb); + } else if (target instanceof OutputFile) { + OutputFile outputFile = (OutputFile) target; + Label label = outputFile.getLabel(); + + Rule generatingRule = outputFile.getGeneratingRule(); + Build.GeneratedFile output = Build.GeneratedFile.newBuilder() + .setLocation(location) + .setGeneratingRule(generatingRule.getLabel().toString()) + .setName(label.toString()) + .build(); + + targetPb.setType(GENERATED_FILE); + targetPb.setGeneratedFile(output); + } else if (target instanceof InputFile) { + InputFile inputFile = (InputFile) target; + Label label = inputFile.getLabel(); + + Build.SourceFile.Builder input = Build.SourceFile.newBuilder() + .setLocation(location) + .setName(label.toString()); + + if (inputFile.getName().equals("BUILD")) { + for (Label subinclude : inputFile.getPackage().getSubincludeLabels()) { + input.addSubinclude(subinclude.toString()); + } + + for (Label skylarkFileDep : inputFile.getPackage().getSkylarkFileDependencies()) { + input.addSubinclude(skylarkFileDep.toString()); + } + + for (String feature : inputFile.getPackage().getFeatures()) { + input.addFeature(feature); + } + } + + for (Label visibilityDependency : target.getVisibility().getDependencyLabels()) { + input.addPackageGroup(visibilityDependency.toString()); + } + + for (Label visibilityDeclaration : target.getVisibility().getDeclaredLabels()) { + input.addVisibilityLabel(visibilityDeclaration.toString()); + } + + targetPb.setType(SOURCE_FILE); + targetPb.setSourceFile(input); + } else if (target instanceof FakeSubincludeTarget) { + Label label = target.getLabel(); + Build.SourceFile input = Build.SourceFile.newBuilder() + .setLocation(location) + .setName(label.toString()) + .build(); + + targetPb.setType(SOURCE_FILE); + targetPb.setSourceFile(input); + } else if (target instanceof PackageGroup) { + PackageGroup packageGroup = (PackageGroup) target; + Build.PackageGroup.Builder packageGroupPb = Build.PackageGroup.newBuilder() + .setName(packageGroup.getLabel().toString()); + for (String containedPackage : packageGroup.getContainedPackages()) { + packageGroupPb.addContainedPackage(containedPackage); + } + for (Label include : packageGroup.getIncludes()) { + packageGroupPb.addIncludedPackageGroup(include.toString()); + } + + targetPb.setType(PACKAGE_GROUP); + targetPb.setPackageGroup(packageGroupPb); + } else { + throw new IllegalArgumentException(target.toString()); + } + + return targetPb.build(); + } + + /** + * Adds the serialized version of the specified attribute to the specified message. + * + * @param rulePb the message to amend + * @param attr the attribute to add + * @param value the possible values of the attribute (can be a multi-value list for + * configurable attributes) + * @param location the location of the attribute in the source file + * @param explicitlySpecified whether the attribute was explicitly specified or not + * @param includeGlobs add glob expression for attributes that contain them + */ + @SuppressWarnings("unchecked") + public static void addAttributeToProto( + Build.Rule.Builder rulePb, Attribute attr, Iterable<Object> values, + Location location, Boolean explicitlySpecified, boolean includeGlobs) { + // Get the attribute type. We need to convert and add appropriately + com.google.devtools.build.lib.packages.Type<?> type = attr.getType(); + + Build.Attribute.Builder attrPb = Build.Attribute.newBuilder(); + + // Set the type, name and source + attrPb.setName(attr.getName()); + attrPb.setType(ProtoUtils.getDiscriminatorFromType(type)); + + if (location != null) { + attrPb.setParseableLocation(serialize(location)); + } + + if (explicitlySpecified != null) { + attrPb.setExplicitlySpecified(explicitlySpecified); + } + + // Convenience binding for single-value attributes. Because those attributes can only + // have a single value, when we encounter configurable versions of them we need to + // react somehow to having multiple possible values to report. We currently just + // refrain from setting *any* value in that scenario. This variable is set to null + // to indicate that scenario. + Object singleAttributeValue = Iterables.size(values) == 1 + ? Iterables.getOnlyElement(values) + : null; + + /* + * Set the appropriate type and value. Since string and string list store + * values for multiple types, use the toString() method on the objects + * instead of casting them. Note that Boolean and TriState attributes have + * both an integer and string representation. + */ + if (type == INTEGER) { + if (singleAttributeValue != null) { + attrPb.setIntValue((Integer) singleAttributeValue); + } + } else if (type == STRING || type == LABEL || type == NODEP_LABEL || type == OUTPUT) { + if (singleAttributeValue != null) { + attrPb.setStringValue(singleAttributeValue.toString()); + } + } else if (type == STRING_LIST || type == LABEL_LIST || type == NODEP_LABEL_LIST + || type == OUTPUT_LIST || type == DISTRIBUTIONS) { + for (Object value : values) { + for (Object entry : (Collection<?>) value) { + attrPb.addStringListValue(entry.toString()); + } + } + } else if (type == INTEGER_LIST) { + for (Object value : values) { + for (Integer entry : (Collection<Integer>) value) { + attrPb.addIntListValue(entry); + } + } + } else if (type == BOOLEAN) { + if (singleAttributeValue != null) { + if ((Boolean) singleAttributeValue) { + attrPb.setStringValue("true"); + attrPb.setBooleanValue(true); + } else { + attrPb.setStringValue("false"); + attrPb.setBooleanValue(false); + } + // This maintains partial backward compatibility for external users of the + // protobuf that were expecting an integer field and not a true boolean. + attrPb.setIntValue((Boolean) singleAttributeValue ? 1 : 0); + } + } else if (type == TRISTATE) { + if (singleAttributeValue != null) { + switch ((TriState) singleAttributeValue) { + case AUTO: + attrPb.setIntValue(-1); + attrPb.setStringValue("auto"); + attrPb.setTristateValue(Build.Attribute.Tristate.AUTO); + break; + case NO: + attrPb.setIntValue(0); + attrPb.setStringValue("no"); + attrPb.setTristateValue(Build.Attribute.Tristate.NO); + break; + case YES: + attrPb.setIntValue(1); + attrPb.setStringValue("yes"); + attrPb.setTristateValue(Build.Attribute.Tristate.YES); + break; + default: + throw new IllegalStateException("Execpted AUTO/NO/YES to cover all possible cases"); + } + } + } else if (type == LICENSE) { + if (singleAttributeValue != null) { + License license = (License) singleAttributeValue; + Build.License.Builder licensePb = Build.License.newBuilder(); + for (License.LicenseType licenseType : license.getLicenseTypes()) { + licensePb.addLicenseType(licenseType.toString()); + } + for (Label exception : license.getExceptions()) { + licensePb.addException(exception.toString()); + } + attrPb.setLicense(licensePb); + } + } else if (type == STRING_DICT) { + // TODO(bazel-team): support better de-duping here and in other dictionaries. + for (Object value : values) { + Map<String, String> dict = (Map<String, String>) value; + for (Map.Entry<String, String> keyValueList : dict.entrySet()) { + Build.StringDictEntry entry = Build.StringDictEntry.newBuilder() + .setKey(keyValueList.getKey()) + .setValue(keyValueList.getValue()) + .build(); + attrPb.addStringDictValue(entry); + } + } + } else if (type == STRING_DICT_UNARY) { + for (Object value : values) { + Map<String, String> dict = (Map<String, String>) value; + for (Map.Entry<String, String> dictEntry : dict.entrySet()) { + Build.StringDictUnaryEntry entry = Build.StringDictUnaryEntry.newBuilder() + .setKey(dictEntry.getKey()) + .setValue(dictEntry.getValue()) + .build(); + attrPb.addStringDictUnaryValue(entry); + } + } + } else if (type == STRING_LIST_DICT) { + for (Object value : values) { + Map<String, List<String>> dict = (Map<String, List<String>>) value; + for (Map.Entry<String, List<String>> dictEntry : dict.entrySet()) { + Build.StringListDictEntry.Builder entry = Build.StringListDictEntry.newBuilder() + .setKey(dictEntry.getKey()); + for (Object dictEntryValue : dictEntry.getValue()) { + entry.addValue(dictEntryValue.toString()); + } + attrPb.addStringListDictValue(entry); + } + } + } else if (type == LABEL_LIST_DICT) { + for (Object value : values) { + Map<String, List<Label>> dict = (Map<String, List<Label>>) value; + for (Map.Entry<String, List<Label>> dictEntry : dict.entrySet()) { + Build.LabelListDictEntry.Builder entry = Build.LabelListDictEntry.newBuilder() + .setKey(dictEntry.getKey()); + for (Object dictEntryValue : dictEntry.getValue()) { + entry.addValue(dictEntryValue.toString()); + } + attrPb.addLabelListDictValue(entry); + } + } + } else if (type == FILESET_ENTRY_LIST) { + for (Object value : values) { + List<FilesetEntry> filesetEntries = (List<FilesetEntry>) value; + for (FilesetEntry filesetEntry : filesetEntries) { + Build.FilesetEntry.Builder filesetEntryPb = Build.FilesetEntry.newBuilder() + .setSource(filesetEntry.getSrcLabel().toString()) + .setDestinationDirectory(filesetEntry.getDestDir().getPathString()) + .setSymlinkBehavior(symlinkBehaviorToPb(filesetEntry.getSymlinkBehavior())) + .setStripPrefix(filesetEntry.getStripPrefix()) + .setFilesPresent(filesetEntry.getFiles() != null); + + if (filesetEntry.getFiles() != null) { + for (Label file : filesetEntry.getFiles()) { + filesetEntryPb.addFile(file.toString()); + } + } + + if (filesetEntry.getExcludes() != null) { + for (String exclude : filesetEntry.getExcludes()) { + filesetEntryPb.addExclude(exclude); + } + } + + attrPb.addFilesetListValue(filesetEntryPb); + } + } + } else { + throw new IllegalStateException("Unknown type: " + type); + } + + if (includeGlobs) { + for (Object value : values) { + if (value instanceof GlobList<?>) { + GlobList<?> globList = (GlobList<?>) value; + + for (GlobCriteria criteria : globList.getCriteria()) { + Build.GlobCriteria.Builder criteriaPb = Build.GlobCriteria.newBuilder() + .setGlob(criteria.isGlob()); + for (String include : criteria.getIncludePatterns()) { + criteriaPb.addInclude(include); + } + for (String exclude : criteria.getExcludePatterns()) { + criteriaPb.addExclude(exclude); + } + + attrPb.addGlobCriteria(criteriaPb); + } + } + } + } + + rulePb.addAttribute(attrPb); + } + + // This is needed because I do not want to use the SymlinkBehavior from the + // protocol buffer all over the place, so there are two classes that do + // essentially the same thing. + private static Build.FilesetEntry.SymlinkBehavior symlinkBehaviorToPb( + FilesetEntry.SymlinkBehavior symlinkBehavior) { + switch (symlinkBehavior) { + case COPY: + return Build.FilesetEntry.SymlinkBehavior.COPY; + case DEREFERENCE: + return Build.FilesetEntry.SymlinkBehavior.DEREFERENCE; + default: + throw new AssertionError("Unhandled FilesetEntry.SymlinkBehavior"); + } + } + + private static Build.Location serialize(Location location) { + Build.Location.Builder result = Build.Location.newBuilder(); + + result.setStartOffset(location.getStartOffset()); + if (location.getStartLineAndColumn() != null) { + result.setStartLine(location.getStartLineAndColumn().getLine()); + result.setStartColumn(location.getStartLineAndColumn().getColumn()); + } + + result.setEndOffset(location.getEndOffset()); + if (location.getEndLineAndColumn() != null) { + result.setEndLine(location.getEndLineAndColumn().getLine()); + result.setEndColumn(location.getEndLineAndColumn().getColumn()); + } + + return result.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/QueryOptions.java b/src/main/java/com/google/devtools/build/lib/query2/output/QueryOptions.java new file mode 100644 index 0000000..810e8c7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/output/QueryOptions.java
@@ -0,0 +1,136 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.output; + +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Setting; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +import java.util.EnumSet; +import java.util.Set; + +/** + * Command-line options for the Blaze query language, revision 2. + */ +public class QueryOptions extends OptionsBase { + + @Option(name = "output", + defaultValue = "label", + category = "query", + help = "The format in which the query results should be printed." + + " Allowed values are: label, label_kind, minrank, maxrank, package, location, graph," + + " xml, proto, record.") + public String outputFormat; + + @Option(name = "order_results", + defaultValue = "true", + category = "query", + help = "Output the results in dependency-ordered (default) or unordered fashion. The" + + " unordered output is faster but only supported when --output is one of label," + + " label_kind, location, package, proto, record, xml.") + public boolean orderResults; + + @Option(name = "keep_going", + abbrev = 'k', + defaultValue = "false", + category = "strategy", + help = "Continue as much as possible after an error. While the " + + "target that failed, and those that depend on it, cannot be " + + "analyzed, other prerequisites of these " + + "targets can be.") + public boolean keepGoing; + + @Option(name = "loading_phase_threads", + defaultValue = "200", + category = "undocumented", + help = "Number of parallel threads to use for the loading phase.") + public int loadingPhaseThreads; + + @Option(name = "host_deps", + defaultValue = "true", + category = "query", + help = "If enabled, dependencies on 'host configuration' targets will be included in " + + "the dependency graph over which the query operates. A 'host configuration' " + + "dependency edge, such as the one from any 'proto_library' rule to the Protocol " + + "Compiler, usually points to a tool executed during the build (on the host machine) " + + "rather than a part of the same 'target' program. Queries whose purpose is to " + + "discover the set of things needed during a build will typically enable this option; " + + "queries aimed at revealing the structure of a single program will typically disable " + + "this option.") + public boolean includeHostDeps; + + @Option(name = "implicit_deps", + defaultValue = "true", + category = "query", + help = "If enabled, implicit dependencies will be included in the dependency graph over " + + "which the query operates. An implicit dependency is one that is not explicitly " + + "specified in the BUILD file but added by blaze.") + public boolean includeImplicitDeps; + + @Option(name = "graph:node_limit", + defaultValue = "512", + category = "query", + help = "The maximum length of the label string for a graph node in the output. Longer labels" + + " will be truncated; -1 means no truncation. This option is only applicable to" + + " --output=graph.") + public int graphNodeStringLimit; + + @Option(name = "graph:factored", + defaultValue = "true", + category = "query", + help = "If true, then the graph will be emitted 'factored', i.e. " + + "topologically-equivalent nodes will be merged together and their " + + "labels concatenated. This option is only applicable to " + + "--output=graph.") + public boolean graphFactored; + + @Option(name = "xml:line_numbers", + defaultValue = "true", + category = "query", + help = "If true, XML output contains line numbers. Disabling this option " + + "may make diffs easier to read. This option is only applicable to " + + "--output=xml.") + public boolean xmlLineNumbers; + + @Option(name = "xml:default_values", + defaultValue = "false", + category = "query", + help = "If true, rule attributes whose value is not explicitly specified " + + "in the BUILD file are printed; otherwise they are omitted.") + public boolean xmlShowDefaultValues; + + @Option(name = "strict_test_suite", + defaultValue = "false", + category = "query", + help = "If true, the tests() expression gives an error if it encounters a test_suite " + + "containing non-test targets.") + public boolean strictTestSuite; + + /** + * Return the current options as a set of QueryEnvironment settings. + */ + public Set<Setting> toSettings() { + Set<Setting> settings = EnumSet.noneOf(Setting.class); + if (strictTestSuite) { + settings.add(Setting.TESTS_EXPRESSION_STRICT); + } + if (!includeHostDeps) { + settings.add(Setting.NO_HOST_DEPS); + } + if (!includeImplicitDeps) { + settings.add(Setting.NO_IMPLICIT_DEPS); + } + return settings; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/query2/output/XmlOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/output/XmlOutputFormatter.java new file mode 100644 index 0000000..c01c90e4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/output/XmlOutputFormatter.java
@@ -0,0 +1,352 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2.output; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.graph.Digraph; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.License; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.PackageGroup; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.query2.FakeSubincludeTarget; +import com.google.devtools.build.lib.syntax.FilesetEntry; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.BinaryPredicate; +import com.google.devtools.build.lib.util.Pair; + +import org.w3c.dom.DOMException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import java.io.PrintStream; +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.TransformerFactoryConfigurationError; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +/** + * An output formatter that prints the result as XML. + */ +class XmlOutputFormatter extends OutputFormatter implements OutputFormatter.UnorderedFormatter { + + private boolean xmlLineNumbers; + private boolean showDefaultValues; + private BinaryPredicate<Rule, Attribute> dependencyFilter; + + @Override + public String getName() { + return "xml"; + } + + @Override + public void outputUnordered(QueryOptions options, Iterable<Target> result, PrintStream out) { + this.xmlLineNumbers = options.xmlLineNumbers; + this.showDefaultValues = options.xmlShowDefaultValues; + this.dependencyFilter = OutputFormatter.getDependencyFilter(options); + + Document doc; + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + doc = factory.newDocumentBuilder().newDocument(); + } catch (ParserConfigurationException e) { + // This shouldn't be possible: all the configuration is hard-coded. + throw new IllegalStateException("XML output failed", e); + } + doc.setXmlVersion("1.1"); + Element queryElem = doc.createElement("query"); + queryElem.setAttribute("version", "2"); + doc.appendChild(queryElem); + for (Target target : result) { + queryElem.appendChild(createTargetElement(doc, target)); + } + try { + Transformer transformer = TransformerFactory.newInstance().newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.transform(new DOMSource(doc), new StreamResult(out)); + } catch (TransformerFactoryConfigurationError | TransformerException e) { + // This shouldn't be possible: all the configuration is hard-coded. + throw new IllegalStateException("XML output failed", e); + } + } + + @Override + public void output(QueryOptions options, Digraph<Target> result, PrintStream out) { + Iterable<Target> ordered = Iterables.transform( + result.getTopologicalOrder(new TargetOrdering()), OutputFormatter.EXTRACT_NODE_LABEL); + outputUnordered(options, ordered, out); + } + + /** + * Creates and returns a new DOM tree for the specified build target. + * + * XML structure: + * - element tag is <source-file>, <generated-file> or <rule + * class="cc_library">, following the terminology of + * {@link Target#getTargetKind()}. + * - 'name' attribute is target's label. + * - 'location' attribute is consistent with output of --output location. + * - rule attributes are represented in the DOM structure. + */ + private Element createTargetElement(Document doc, Target target) { + Element elem; + if (target instanceof Rule) { + Rule rule = (Rule) target; + elem = doc.createElement("rule"); + elem.setAttribute("class", rule.getRuleClass()); + for (Attribute attr: rule.getAttributes()) { + Pair<Iterable<Object>, AttributeValueSource> values = getAttributeValues(rule, attr); + if (values.second == AttributeValueSource.RULE || showDefaultValues) { + Element attrElem = createValueElement(doc, attr.getType(), values.first); + attrElem.setAttribute("name", attr.getName()); + elem.appendChild(attrElem); + } + } + + // Include explicit elements for all direct inputs and outputs of a rule; + // this goes beyond what is available from the attributes above, since it + // may also (depending on options) include implicit outputs, + // host-configuration outputs, and default values. + for (Label label : rule.getLabels(dependencyFilter)) { + Element inputElem = doc.createElement("rule-input"); + inputElem.setAttribute("name", label.toString()); + elem.appendChild(inputElem); + } + for (OutputFile outputFile: rule.getOutputFiles()) { + Element outputElem = doc.createElement("rule-output"); + outputElem.setAttribute("name", outputFile.getLabel().toString()); + elem.appendChild(outputElem); + } + for (String feature : rule.getFeatures()) { + Element outputElem = doc.createElement("rule-default-setting"); + outputElem.setAttribute("name", feature); + elem.appendChild(outputElem); + } + } else if (target instanceof PackageGroup) { + PackageGroup packageGroup = (PackageGroup) target; + elem = doc.createElement("package-group"); + elem.setAttribute("name", packageGroup.getName()); + Element includes = createValueElement(doc, + com.google.devtools.build.lib.packages.Type.LABEL_LIST, + packageGroup.getIncludes()); + includes.setAttribute("name", "includes"); + elem.appendChild(includes); + Element packages = createValueElement(doc, + com.google.devtools.build.lib.packages.Type.STRING_LIST, + packageGroup.getContainedPackages()); + packages.setAttribute("name", "packages"); + elem.appendChild(packages); + } else if (target instanceof OutputFile) { + OutputFile outputFile = (OutputFile) target; + elem = doc.createElement("generated-file"); + elem.setAttribute("generating-rule", + outputFile.getGeneratingRule().getLabel().toString()); + } else if (target instanceof InputFile) { + elem = doc.createElement("source-file"); + InputFile inputFile = (InputFile) target; + if (inputFile.getName().equals("BUILD")) { + addSubincludedFilesToElement(doc, elem, inputFile); + addSkylarkFilesToElement(doc, elem, inputFile); + addFeaturesToElement(doc, elem, inputFile); + } + + addPackageGroupsToElement(doc, elem, inputFile); + } else if (target instanceof FakeSubincludeTarget) { + elem = doc.createElement("source-file"); + } else { + throw new IllegalArgumentException(target.toString()); + } + + elem.setAttribute("name", target.getLabel().toString()); + String location = target.getLocation().print(); + if (!xmlLineNumbers) { + int firstColon = location.indexOf(":"); + if (firstColon != -1) { + location = location.substring(0, firstColon); + } + } + + elem.setAttribute("location", location); + return elem; + } + + private void addPackageGroupsToElement(Document doc, Element parent, Target target) { + for (Label visibilityDependency : target.getVisibility().getDependencyLabels()) { + Element elem = doc.createElement("package-group"); + elem.setAttribute("name", visibilityDependency.toString()); + parent.appendChild(elem); + } + + for (Label visibilityDeclaration : target.getVisibility().getDeclaredLabels()) { + Element elem = doc.createElement("visibility-label"); + elem.setAttribute("name", visibilityDeclaration.toString()); + parent.appendChild(elem); + } + } + + private void addFeaturesToElement(Document doc, Element parent, InputFile inputFile) { + for (String feature : inputFile.getPackage().getFeatures()) { + Element elem = doc.createElement("feature"); + elem.setAttribute("name", feature); + parent.appendChild(elem); + } + } + + private void addSubincludedFilesToElement(Document doc, Element parent, InputFile inputFile) { + for (Label subinclude : inputFile.getPackage().getSubincludeLabels()) { + Element elem = doc.createElement("subinclude"); + elem.setAttribute("name", subinclude.toString()); + parent.appendChild(elem); + } + } + + private void addSkylarkFilesToElement(Document doc, Element parent, InputFile inputFile) { + for (Label skylarkFileDep : inputFile.getPackage().getSkylarkFileDependencies()) { + Element elem = doc.createElement("load"); + elem.setAttribute("name", skylarkFileDep.toString()); + parent.appendChild(elem); + } + } + + /** + * Creates and returns a new DOM tree for the specified attribute values. + * For non-configurable attributes, this is a single value. For configurable + * attributes, this contains one value for each configuration. + * (Only toplevel values are named attributes; list elements are unnamed.) + * + * <p>In the case of configurable attributes, multi-value attributes (e.g. lists) + * merge all configured lists into an aggregate flattened list. Single-value attributes + * simply refrain to set a value and annotate the DOM element as configurable. + * + * <P>(The ungainly qualified class name is required to avoid ambiguity with + * OutputFormatter.Type.) + */ + private static Element createValueElement(Document doc, + com.google.devtools.build.lib.packages.Type<?> type, Iterable<Object> values) { + // "Import static" with method scope: + com.google.devtools.build.lib.packages.Type<?> + FILESET_ENTRY = com.google.devtools.build.lib.packages.Type.FILESET_ENTRY, + LABEL_LIST = com.google.devtools.build.lib.packages.Type.LABEL_LIST, + LICENSE = com.google.devtools.build.lib.packages.Type.LICENSE, + STRING_LIST = com.google.devtools.build.lib.packages.Type.STRING_LIST; + + final Element elem; + final boolean hasMultipleValues = Iterables.size(values) > 1; + com.google.devtools.build.lib.packages.Type<?> elemType = type.getListElementType(); + if (elemType != null) { // it's a list (includes "distribs") + elem = doc.createElement("list"); + for (Object value : values) { + for (Object elemValue : (Collection<?>) value) { + elem.appendChild(createValueElement(doc, elemType, elemValue)); + } + } + } else if (type instanceof com.google.devtools.build.lib.packages.Type.DictType) { + Set<Object> visitedValues = new HashSet<>(); + elem = doc.createElement("dict"); + com.google.devtools.build.lib.packages.Type.DictType<?, ?> dictType = + (com.google.devtools.build.lib.packages.Type.DictType<?, ?>) type; + for (Object value : values) { + for (Map.Entry<?, ?> entry : ((Map<?, ?>) value).entrySet()) { + if (visitedValues.add(entry.getKey())) { + Element pairElem = doc.createElement("pair"); + elem.appendChild(pairElem); + pairElem.appendChild(createValueElement(doc, + dictType.getKeyType(), entry.getKey())); + pairElem.appendChild(createValueElement(doc, + dictType.getValueType(), entry.getValue())); + } + } + } + } else if (type == LICENSE) { + elem = createSingleValueElement(doc, "license", hasMultipleValues); + if (!hasMultipleValues) { + License license = (License) Iterables.getOnlyElement(values); + + Element exceptions = createValueElement(doc, LABEL_LIST, license.getExceptions()); + exceptions.setAttribute("name", "exceptions"); + elem.appendChild(exceptions); + + Element licenseTypes = createValueElement(doc, STRING_LIST, license.getLicenseTypes()); + licenseTypes.setAttribute("name", "license-types"); + elem.appendChild(licenseTypes); + } + } else if (type == FILESET_ENTRY) { + // Fileset entries: not configurable. + FilesetEntry filesetEntry = (FilesetEntry) Iterables.getOnlyElement(values); + elem = doc.createElement("fileset-entry"); + elem.setAttribute("srcdir", filesetEntry.getSrcLabel().toString()); + elem.setAttribute("destdir", filesetEntry.getDestDir().toString()); + elem.setAttribute("symlinks", filesetEntry.getSymlinkBehavior().toString()); + elem.setAttribute("strip_prefix", filesetEntry.getStripPrefix()); + + if (filesetEntry.getExcludes() != null) { + Element excludes = + createValueElement(doc, LABEL_LIST, filesetEntry.getExcludes()); + excludes.setAttribute("name", "excludes"); + elem.appendChild(excludes); + } + if (filesetEntry.getFiles() != null) { + Element files = createValueElement(doc, LABEL_LIST, filesetEntry.getFiles()); + files.setAttribute("name", "files"); + elem.appendChild(files); + } + } else { // INTEGER STRING LABEL DISTRIBUTION OUTPUT + elem = createSingleValueElement(doc, type.toString(), hasMultipleValues); + if (!hasMultipleValues && !Iterables.isEmpty(values)) { + Object value = Iterables.getOnlyElement(values); + // Values such as those of attribute "linkstamp" may be null. + if (value != null) { + try { + elem.setAttribute("value", value.toString()); + } catch (DOMException e) { + elem.setAttribute("value", "[[[ERROR: could not be encoded as XML]]]"); + } + } + } + } + return elem; + } + + private static Element createValueElement(Document doc, + com.google.devtools.build.lib.packages.Type<?> type, Object value) { + return createValueElement(doc, type, ImmutableList.of(value)); + } + + /** + * Creates the given DOM element, adding <code>configurable="yes"</code> if it represents + * a configurable single-value attribute (configurable list attributes simply have their + * lists merged into an aggregate flat list). + */ + private static Element createSingleValueElement(Document doc, String name, + boolean configurable) { + Element elem = doc.createElement(name); + if (configurable) { + elem.setAttribute("configurable", "yes"); + } + return elem; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/RuleConfiguredTargetFactory.java b/src/main/java/com/google/devtools/build/lib/rules/RuleConfiguredTargetFactory.java new file mode 100644 index 0000000..45df124 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/RuleConfiguredTargetFactory.java
@@ -0,0 +1,25 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules; + +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.packages.RuleClass; + +/** + * A shortcut class to the appropriate specialization of {@code RuleClass.ConfiguredTargetFactory}. + */ +public interface RuleConfiguredTargetFactory + extends RuleClass.ConfiguredTargetFactory<ConfiguredTarget, RuleContext> { +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkAttr.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkAttr.java new file mode 100644 index 0000000..dfd6a92 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkAttr.java
@@ -0,0 +1,350 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules; + +import static com.google.devtools.build.lib.syntax.SkylarkFunction.castList; + +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition; +import com.google.devtools.build.lib.packages.Attribute.SkylarkLateBound; +import com.google.devtools.build.lib.packages.SkylarkFileType; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.packages.Type.ConversionException; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param; +import com.google.devtools.build.lib.syntax.SkylarkCallbackFunction; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.SkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.syntax.UserDefinedFunction; +import com.google.devtools.build.lib.util.FileTypeSet; + +import java.util.Map; + +/** + * A helper class to provide Attr module in Skylark. + */ +@SkylarkModule(name = "attr", namespace = true, onlyLoadingPhase = true, + doc = "Module for creating new attributes. " + + "They are only for use with the <code>rule</code> function.") +public final class SkylarkAttr { + + private static final String MANDATORY_DOC = + "set to true if users have to explicitely specify the value"; + + private static final String ALLOW_FILES_DOC = + "whether File targets are allowed. Can be True, False (default), or " + + "a FileType filter."; + + private static final String ALLOW_RULES_DOC = + "which rule targets (name of the classes) are allowed." + + "This is deprecated (kept only for compatiblity), use providers instead."; + + private static final String FLAGS_DOC = + "deprecated, will be removed"; + + private static final String DEFAULT_DOC = + "sets the default value of the attribute."; + + private static final String CONFIGURATION_DOC = + "configuration of the attribute. " + + "For example, use DATA_CFG or HOST_CFG."; + + private static final String EXECUTABLE_DOC = + "set to True if the labels have to be executable. Access the labels with " + + "ctx.executable.<attribute_name>"; + + private static Attribute.Builder<?> createAttribute(Type<?> type, Map<String, Object> arguments, + FuncallExpression ast, SkylarkEnvironment env) throws EvalException, ConversionException { + final Location loc = ast.getLocation(); + // We use an empty name now so that we can set it later. + // This trick makes sense only in the context of Skylark (builtin rules should not use it). + Attribute.Builder<?> builder = Attribute.attr("", type); + + Object defaultValue = arguments.get("default"); + if (defaultValue != null) { + if (defaultValue instanceof UserDefinedFunction) { + // Late bound attribute. Non label type attributes already caused a type check error. + builder.value(new SkylarkLateBound( + new SkylarkCallbackFunction((UserDefinedFunction) defaultValue, ast, env))); + } else { + builder.defaultValue(defaultValue); + } + } + + for (String flag : castList(arguments.get("flags"), String.class)) { + builder.setPropertyFlag(flag); + } + + if (arguments.containsKey("mandatory") && (Boolean) arguments.get("mandatory")) { + builder.setPropertyFlag("MANDATORY"); + } + + if (arguments.containsKey("executable") && (Boolean) arguments.get("executable")) { + builder.setPropertyFlag("EXECUTABLE"); + } + + if (arguments.containsKey("single_file") && (Boolean) arguments.get("single_file")) { + builder.setPropertyFlag("SINGLE_ARTIFACT"); + } + + if (arguments.containsKey("allow_files")) { + Object fileTypesObj = arguments.get("allow_files"); + if (fileTypesObj == Boolean.TRUE) { + builder.allowedFileTypes(FileTypeSet.ANY_FILE); + } else if (fileTypesObj == Boolean.FALSE) { + builder.allowedFileTypes(FileTypeSet.NO_FILE); + } else if (fileTypesObj instanceof SkylarkFileType) { + builder.allowedFileTypes(((SkylarkFileType) fileTypesObj).getFileTypeSet()); + } else { + throw new EvalException(loc, "allow_files should be a boolean or a filetype object."); + } + } else if (type.equals(Type.LABEL) || type.equals(Type.LABEL_LIST)) { + builder.allowedFileTypes(FileTypeSet.NO_FILE); + } + + Object ruleClassesObj = arguments.get("allow_rules"); + if (ruleClassesObj != null) { + builder.allowedRuleClasses(castList(ruleClassesObj, String.class, + "allowed rule classes for attribute definition")); + } + + if (arguments.containsKey("providers")) { + builder.mandatoryProviders(castList(arguments.get("providers"), String.class)); + } + + if (arguments.containsKey("cfg")) { + builder.cfg((ConfigurationTransition) arguments.get("cfg")); + } + return builder; + } + + private static Object createAttribute(Map<String, Object> kwargs, Type<?> type, + FuncallExpression ast, Environment env) throws EvalException { + try { + return createAttribute(type, kwargs, ast, (SkylarkEnvironment) env); + } catch (ConversionException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + } + + @SkylarkBuiltin(name = "int", doc = + "Creates an attribute of type int.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = Integer.class, + doc = DEFAULT_DOC + " If not specified, default is 0."), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction integer = new SkylarkFunction("int") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.INTEGER, ast, env); + } + }; + + @SkylarkBuiltin(name = "string", doc = + "Creates an attribute of type string.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = String.class, + doc = DEFAULT_DOC + " If not specified, default is \"\"."), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction string = new SkylarkFunction("string") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.STRING, ast, env); + } + }; + + @SkylarkBuiltin(name = "label", doc = + "Creates an attribute of type Label. " + + "It is the only way to specify a dependency to another target. " + + "If you need a dependency that the user cannot overwrite, make the attribute " + + "private (starts with <code>_</code>).", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = Label.class, callbackEnabled = true, + doc = DEFAULT_DOC + " If not specified, default is None. " + + "Use the <code>Label</code> function to specify a default value."), + @Param(name = "executable", type = Boolean.class, doc = EXECUTABLE_DOC), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "allow_files", doc = ALLOW_FILES_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "providers", type = SkylarkList.class, generic1 = String.class, + doc = "mandatory providers every dependency has to have"), + @Param(name = "allow_rules", type = SkylarkList.class, generic1 = String.class, + doc = ALLOW_RULES_DOC), + @Param(name = "single_file", doc = + "if true, the label must correspond to a single File. " + + "Access it through ctx.file.<attribute_name>."), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction label = new SkylarkFunction("label") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.LABEL, ast, env); + } + }; + + @SkylarkBuiltin(name = "string_list", doc = + "Creates an attribute of type list of strings", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = SkylarkList.class, generic1 = String.class, + doc = DEFAULT_DOC + " If not specified, default is []."), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, + doc = CONFIGURATION_DOC)}) + private static SkylarkFunction stringList = new SkylarkFunction("string_list") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.STRING_LIST, ast, env); + } + }; + + @SkylarkBuiltin(name = "label_list", doc = + "Creates an attribute of type list of labels. " + + "See <code>label</code> for more information.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = SkylarkList.class, generic1 = Label.class, + callbackEnabled = true, + doc = DEFAULT_DOC + " If not specified, default is []. " + + "Use the <code>Label</code> function to specify a default value."), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "allow_files", doc = ALLOW_FILES_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "allow_rules", type = SkylarkList.class, generic1 = String.class, + doc = ALLOW_RULES_DOC), + @Param(name = "providers", type = SkylarkList.class, generic1 = String.class, + doc = "mandatory providers every dependency has to have"), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction labelList = new SkylarkFunction("label_list") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.LABEL_LIST, ast, env); + } + }; + + @SkylarkBuiltin(name = "bool", doc = + "Creates an attribute of type bool. Its default value is False.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = Boolean.class, doc = DEFAULT_DOC), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction bool = new SkylarkFunction("bool") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.BOOLEAN, ast, env); + } + }; + + @SkylarkBuiltin(name = "output", doc = + "Creates an attribute of type output. Its default value is None. " + + "The user provides a file name (string) and the rule must create an action that " + + "generates the file.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = Label.class, doc = DEFAULT_DOC), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction output = new SkylarkFunction("output") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.OUTPUT, ast, env); + } + }; + + @SkylarkBuiltin(name = "output_list", doc = + "Creates an attribute of type list of outputs. Its default value is []. " + + "See <code>output</code> above for more information.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = SkylarkList.class, generic1 = Label.class, doc = DEFAULT_DOC), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction outputList = new SkylarkFunction("output_list") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.OUTPUT_LIST, ast, env); + } + }; + + @SkylarkBuiltin(name = "string_dict", doc = + "Creates an attribute of type dictionary, mapping from string to string. " + + "Its default value is {}.", + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", type = Map.class, doc = DEFAULT_DOC), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction stringDict = new SkylarkFunction("string_dict") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.STRING_DICT, ast, env); + } + }; + + @SkylarkBuiltin(name = "license", doc = + "Creates an attribute of type license. Its default value is NO_LICENSE.", + // TODO(bazel-team): Implement proper license support for Skylark. + objectType = SkylarkAttr.class, + returnType = Attribute.class, + optionalParams = { + @Param(name = "default", doc = DEFAULT_DOC), + @Param(name = "flags", type = SkylarkList.class, generic1 = String.class, doc = FLAGS_DOC), + @Param(name = "mandatory", type = Boolean.class, doc = MANDATORY_DOC), + @Param(name = "cfg", type = ConfigurationTransition.class, doc = CONFIGURATION_DOC)}) + private static SkylarkFunction license = new SkylarkFunction("license") { + @Override + public Object call(Map<String, Object> kwargs, FuncallExpression ast, Environment env) + throws EvalException { + return createAttribute(kwargs, Type.LICENSE, ast, env); + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkCommandLine.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkCommandLine.java new file mode 100644 index 0000000..e51805e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkCommandLine.java
@@ -0,0 +1,89 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules; + +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param; +import com.google.devtools.build.lib.syntax.SkylarkFunction.SimpleSkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; + +import java.util.Map; + +/** + * A Skylark module class to create memory efficient command lines. + */ +@SkylarkModule(name = "cmd_helper", namespace = true, + doc = "Module for creating memory efficient command lines.") +public class SkylarkCommandLine { + + @SkylarkBuiltin(name = "join_paths", + doc = "Creates a single command line argument joining the paths of a set " + + "of files on the separator string.", + objectType = SkylarkCommandLine.class, + returnType = String.class, + mandatoryParams = { + @Param(name = "separator", type = String.class, doc = "the separator string to join on"), + @Param(name = "files", type = SkylarkNestedSet.class, generic1 = Artifact.class, + doc = "the files to concatenate")}) + private static SimpleSkylarkFunction joinPaths = + new SimpleSkylarkFunction("join_paths") { + @Override + public Object call(Map<String, Object> params, Location loc) + throws EvalException { + final String separator = (String) params.get("separator"); + final NestedSet<Artifact> artifacts = + ((SkylarkNestedSet) params.get("files")).getSet(Artifact.class); + // TODO(bazel-team): lazy evaluate + return Artifact.joinExecPaths(separator, artifacts); + } + }; + + // TODO(bazel-team): this method should support sets of objects and substitute all struct fields. + @SkylarkBuiltin(name = "template", + doc = "Transforms a set of files to a list of strings using the template string.", + objectType = SkylarkCommandLine.class, + returnType = SkylarkList.class, + mandatoryParams = { + @Param(name = "items", type = SkylarkNestedSet.class, generic1 = Artifact.class, + doc = "The set of structs to transform."), + @Param(name = "template", type = String.class, + doc = "The template to use for the transformation, %{path} and %{short_path} " + + "being substituted with the corresponding fields of each file.")}) + private static SimpleSkylarkFunction template = new SimpleSkylarkFunction("template") { + @Override + public Object call(Map<String, Object> params, Location loc) + throws EvalException { + final String template = (String) params.get("template"); + SkylarkNestedSet items = (SkylarkNestedSet) params.get("items"); + return SkylarkList.lazyList(Iterables.transform(items, new Function<Object, String>() { + @Override + public String apply(Object input) { + Artifact artifact = (Artifact) input; + return template + .replace("%{path}", artifact.getExecPathString()) + .replace("%{short_path}", artifact.getRootRelativePathString()); + } + }), String.class); + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkModules.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkModules.java new file mode 100644 index 0000000..ae81f81 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkModules.java
@@ -0,0 +1,198 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.MethodLibrary; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.Function; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.SkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.syntax.SkylarkType; +import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType; +import com.google.devtools.build.lib.syntax.ValidationEnvironment; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +/** + * A class to handle all Skylark modules, to create and setup Validation and regular Environments. + */ +public class SkylarkModules { + + public static final ImmutableList<Class<?>> MODULES = ImmutableList.of( + SkylarkAttr.class, + SkylarkCommandLine.class, + SkylarkRuleClassFunctions.class, + SkylarkRuleImplementationFunctions.class); + + private static final ImmutableMap<Class<?>, ImmutableList<Function>> FUNCTION_MAP; + private static final ImmutableMap<String, Object> OBJECTS; + + static { + try { + ImmutableMap.Builder<Class<?>, ImmutableList<Function>> functionMap = ImmutableMap.builder(); + ImmutableMap.Builder<String, Object> objects = ImmutableMap.builder(); + for (Class<?> moduleClass : MODULES) { + if (moduleClass.isAnnotationPresent(SkylarkModule.class)) { + objects.put(moduleClass.getAnnotation(SkylarkModule.class).name(), + moduleClass.newInstance()); + } + ImmutableList.Builder<Function> functions = ImmutableList.builder(); + collectSkylarkFunctionsAndObjectsFromFields(moduleClass, functions, objects); + functionMap.put(moduleClass, functions.build()); + } + FUNCTION_MAP = functionMap.build(); + OBJECTS = objects.build(); + } catch (InstantiationException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + /** + * Returns a new SkylarkEnvironment with the elements of the Skylark modules. + */ + public static SkylarkEnvironment getNewEnvironment( + EventHandler eventHandler, String astFileContentHashCode) { + SkylarkEnvironment env = new SkylarkEnvironment(eventHandler, astFileContentHashCode); + setupEnvironment(env); + return env; + } + + @VisibleForTesting + public static SkylarkEnvironment getNewEnvironment(EventHandler eventHandler) { + return getNewEnvironment(eventHandler, null); + } + + private static void setupEnvironment(Environment env) { + MethodLibrary.setupMethodEnvironment(env); + for (Map.Entry<Class<?>, ImmutableList<Function>> entry : FUNCTION_MAP.entrySet()) { + for (Function function : entry.getValue()) { + if (function.getObjectType() != null) { + env.registerFunction(function.getObjectType(), function.getName(), function); + } else { + env.update(function.getName(), function); + } + } + } + for (Map.Entry<String, Object> entry : OBJECTS.entrySet()) { + env.update(entry.getKey(), entry.getValue()); + } + } + + /** + * Returns a new ValidationEnvironment with the elements of the Skylark modules. + */ + public static ValidationEnvironment getValidationEnvironment() { + return getValidationEnvironment(ImmutableMap.<String, SkylarkType>of()); + } + + /** + * Returns a new ValidationEnvironment with the elements of the Skylark modules and extraObjects. + */ + public static ValidationEnvironment getValidationEnvironment( + ImmutableMap<String, SkylarkType> extraObjects) { + Map<SkylarkType, Map<String, SkylarkType>> builtIn = new HashMap<>(); + Map<String, SkylarkType> global = new HashMap<>(); + builtIn.put(SkylarkType.GLOBAL, global); + collectSkylarkTypesFromFields(Environment.class, builtIn); + for (Class<?> moduleClass : MODULES) { + if (moduleClass.isAnnotationPresent(SkylarkModule.class)) { + global.put(moduleClass.getAnnotation(SkylarkModule.class).name(), + SkylarkType.of(moduleClass)); + } + } + global.put("native", SkylarkType.UNKNOWN); + MethodLibrary.setupValidationEnvironment(builtIn); + for (Class<?> module : MODULES) { + collectSkylarkTypesFromFields(module, builtIn); + } + global.putAll(extraObjects); + return new ValidationEnvironment(CollectionUtils.toImmutable(builtIn)); + } + + /** + * Collects the SkylarkFunctions from the fields of the class of the object parameter + * and adds them into the builder. + */ + private static void collectSkylarkFunctionsAndObjectsFromFields(Class<?> type, + ImmutableList.Builder<Function> functions, ImmutableMap.Builder<String, Object> objects) { + try { + for (Field field : type.getDeclaredFields()) { + if (field.isAnnotationPresent(SkylarkBuiltin.class)) { + // Fields in Skylark modules are sometimes private. Nevertheless they have to + // be annotated with SkylarkBuiltin. + field.setAccessible(true); + SkylarkBuiltin annotation = field.getAnnotation(SkylarkBuiltin.class); + if (SkylarkFunction.class.isAssignableFrom(field.getType())) { + SkylarkFunction function = (SkylarkFunction) field.get(null); + if (!function.isConfigured()) { + function.configure(annotation); + } + functions.add(function); + } else { + objects.put(annotation.name(), field.get(null)); + } + } + } + } catch (IllegalArgumentException | IllegalAccessException e) { + // This should never happen. + throw new RuntimeException(e); + } + } + + /** + * Collects the SkylarkFunctions from the fields of the class of the object parameter + * and adds their class and their corresponding return value to the builder. + */ + private static void collectSkylarkTypesFromFields(Class<?> classObject, + Map<SkylarkType, Map<String, SkylarkType>> builtIn) { + for (Field field : classObject.getDeclaredFields()) { + if (field.isAnnotationPresent(SkylarkBuiltin.class)) { + SkylarkBuiltin annotation = field.getAnnotation(SkylarkBuiltin.class); + if (SkylarkFunction.class.isAssignableFrom(field.getType())) { + try { + // TODO(bazel-team): infer the correct types. + SkylarkType objectType = annotation.objectType().equals(Object.class) + ? SkylarkType.GLOBAL + : SkylarkType.of(annotation.objectType()); + if (!builtIn.containsKey(objectType)) { + builtIn.put(objectType, new HashMap<String, SkylarkType>()); + } + // TODO(bazel-team): add parameters to SkylarkFunctionType + SkylarkType returnType = SkylarkType.getReturnType(annotation); + builtIn.get(objectType).put(annotation.name(), + SkylarkFunctionType.of(annotation.name(), returnType)); + } catch (IllegalArgumentException e) { + // This should never happen. + throw new RuntimeException(e); + } + } else if (Function.class.isAssignableFrom(field.getType())) { + builtIn.get(SkylarkType.GLOBAL).put(annotation.name(), + SkylarkFunctionType.of(annotation.name(), SkylarkType.UNKNOWN)); + } else { + builtIn.get(SkylarkType.GLOBAL).put(annotation.name(), SkylarkType.of(field.getType())); + } + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleClassFunctions.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleClassFunctions.java new file mode 100644 index 0000000..39b8836 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleClassFunctions.java
@@ -0,0 +1,430 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.DATA; +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.INTEGER; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.RunUnder; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition; +import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SkylarkImplicitOutputsFunctionWithCallback; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SkylarkImplicitOutputsFunctionWithMap; +import com.google.devtools.build.lib.packages.Package.NameConflictException; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.PackageFactory.PackageContext; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.packages.RuleFactory; +import com.google.devtools.build.lib.packages.RuleFactory.InvalidRuleException; +import com.google.devtools.build.lib.packages.SkylarkFileType; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.TestSize; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.packages.Type.ConversionException; +import com.google.devtools.build.lib.syntax.AbstractFunction; +import com.google.devtools.build.lib.syntax.ClassObject; +import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.Environment.NoSuchVariableException; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.EvalUtils; +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.Function; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param; +import com.google.devtools.build.lib.syntax.SkylarkCallbackFunction; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.SkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkFunction.SimpleSkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.syntax.UserDefinedFunction; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +/** + * A helper class to provide an easier API for Skylark rule definitions. + * This is experimental code. + */ +public class SkylarkRuleClassFunctions { + + //TODO(bazel-team): proper enum support + @SkylarkBuiltin(name = "DATA_CFG", returnType = ConfigurationTransition.class, + doc = "The default runfiles collection state.") + private static final Object dataTransition = ConfigurationTransition.DATA; + + @SkylarkBuiltin(name = "HOST_CFG", returnType = ConfigurationTransition.class, + doc = "The default runfiles collection state.") + private static final Object hostTransition = ConfigurationTransition.HOST; + + private static final Attribute.ComputedDefault DEPRECATION = + new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + return rule.getPackageDefaultDeprecation(); + } + }; + + private static final Attribute.ComputedDefault TEST_ONLY = + new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + return rule.getPackageDefaultTestOnly(); + } + }; + + private static final LateBoundLabel<BuildConfiguration> RUN_UNDER = + new LateBoundLabel<BuildConfiguration>() { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + RunUnder runUnder = configuration.getRunUnder(); + return runUnder == null ? null : runUnder.getLabel(); + } + }; + + // TODO(bazel-team): Copied from ConfiguredRuleClassProvider for the transition from built-in + // rules to skylark extensions. Using the same instance would require a large refactoring. + // If we don't want to support old built-in rules and Skylark simultaneously + // (except for transition phase) it's probably OK. + private static LoadingCache<String, Label> labelCache = + CacheBuilder.newBuilder().build(new CacheLoader<String, Label>() { + @Override + public Label load(String from) throws Exception { + try { + return Label.parseAbsolute(from); + } catch (Label.SyntaxException e) { + throw new Exception(from); + } + } + }); + + // TODO(bazel-team): Remove the code duplication (BaseRuleClasses and this class). + private static final RuleClass baseRule = + BaseRuleClasses.commonCoreAndSkylarkAttributes( + new RuleClass.Builder("$base_rule", RuleClassType.ABSTRACT, true)) + .add(attr("expect_failure", STRING)) + .build(); + + private static final RuleClass testBaseRule = + new RuleClass.Builder("$test_base_rule", RuleClassType.ABSTRACT, true, baseRule) + .add(attr("size", STRING).value("medium").taggable() + .nonconfigurable("used in loading phase rule validation logic")) + .add(attr("timeout", STRING).taggable() + .nonconfigurable("used in loading phase rule validation logic").value( + new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + TestSize size = TestSize.getTestSize(rule.get("size", Type.STRING)); + if (size != null) { + String timeout = size.getDefaultTimeout().toString(); + if (timeout != null) { + return timeout; + } + } + return "illegal"; + } + })) + .add(attr("flaky", BOOLEAN).value(false).taggable() + .nonconfigurable("taggable - called in Rule.getRuleTags")) + .add(attr("shard_count", INTEGER).value(-1)) + .add(attr("local", BOOLEAN).value(false).taggable() + .nonconfigurable("policy decision: this should be consistent across configurations")) + .add(attr("$test_runtime", LABEL_LIST).cfg(HOST).value(ImmutableList.of( + labelCache.getUnchecked("//tools/test:runtime")))) + .add(attr(":run_under", LABEL).cfg(DATA).value(RUN_UNDER)) + .build(); + + /** + * In native code, private values start with $. + * In Skylark, private values start with _, because of the grammar. + */ + private static String attributeToNative(String oldName, Location loc, boolean isLateBound) + throws EvalException { + if (oldName.isEmpty()) { + throw new EvalException(loc, "Attribute name cannot be empty"); + } + if (isLateBound) { + if (oldName.charAt(0) != '_') { + throw new EvalException(loc, "When an attribute value is a function, " + + "the attribute must be private (start with '_')"); + } + return ":" + oldName.substring(1); + } + if (oldName.charAt(0) == '_') { + return "$" + oldName.substring(1); + } + return oldName; + } + + // TODO(bazel-team): implement attribute copy and other rule properties + + @SkylarkBuiltin(name = "rule", doc = + "Creates a new rule. Store it in a global value, so that it can be loaded and called " + + "from BUILD files.", + onlyLoadingPhase = true, + returnType = Function.class, + mandatoryParams = { + @Param(name = "implementation", type = UserDefinedFunction.class, + doc = "the function implementing this rule, has to have exactly one parameter: " + + "<code>ctx</code>. The function is called during analysis phase for each " + + "instance of the rule. It can access the attributes provided by the user. " + + "It must create actions to generate all the declared outputs.") + }, + optionalParams = { + @Param(name = "test", type = Boolean.class, doc = "Whether this rule is a test rule. " + + "If True, the rule must end with <code>_test</code> (otherwise it cannot)."), + @Param(name = "attrs", doc = + "dictionary to declare all the attributes of the rule. It maps from an attribute name " + + "to an attribute object (see 'attr' module). Attributes starting with <code>_</code> " + + "are private, and can be used to add an implicit dependency on a label."), + @Param(name = "outputs", doc = "outputs of this rule. " + + "It is a dictionary mapping from string to a template name. For example: " + + "<code>{\"ext\": \"${name}.ext\"}</code>. <br>" + // TODO(bazel-team): Make doc more clear, wrt late-bound attributes. + + "It may also be a function (which receives <code>ctx.attr</code> as argument) " + + "returning such a dictionary."), + @Param(name = "executable", type = Boolean.class, + doc = "whether this rule always outputs an executable of the same name or not. If True, " + + "there must be an action that generates <code>ctx.outputs.executable</code>.")}) + private static final SkylarkFunction rule = new SkylarkFunction("rule") { + + @Override + public Object call(Map<String, Object> arguments, FuncallExpression ast, + Environment funcallEnv) throws EvalException, ConversionException { + final Location loc = ast.getLocation(); + + RuleClassType type = RuleClassType.NORMAL; + if (arguments.containsKey("test") && EvalUtils.toBoolean(arguments.get("test"))) { + type = RuleClassType.TEST; + } + + // We'll set the name later, pass the empty string for now. + final RuleClass.Builder builder = type == RuleClassType.TEST + ? new RuleClass.Builder("", type, true, testBaseRule) + : new RuleClass.Builder("", type, true, baseRule); + + for (Map.Entry<String, Attribute.Builder> attr : castMap( + arguments.get("attrs"), String.class, Attribute.Builder.class, "attrs")) { + Attribute.Builder<?> attrBuilder = attr.getValue(); + String attrName = attributeToNative(attr.getKey(), loc, + attrBuilder.hasLateBoundValue()); + builder.addOrOverrideAttribute(attrBuilder.build(attrName)); + } + if (arguments.containsKey("executable") && (Boolean) arguments.get("executable")) { + builder.addOrOverrideAttribute( + attr("$is_executable", BOOLEAN).value(true) + .nonconfigurable("Called from RunCommand.isExecutable, which takes a Target") + .build()); + builder.setOutputsDefaultExecutable(); + } + + if (arguments.containsKey("outputs")) { + final Object implicitOutputs = arguments.get("outputs"); + if (implicitOutputs instanceof UserDefinedFunction) { + UserDefinedFunction func = (UserDefinedFunction) implicitOutputs; + final SkylarkCallbackFunction callback = + new SkylarkCallbackFunction(func, ast, (SkylarkEnvironment) funcallEnv); + builder.setImplicitOutputsFunction( + new SkylarkImplicitOutputsFunctionWithCallback(callback, loc)); + } else { + builder.setImplicitOutputsFunction(new SkylarkImplicitOutputsFunctionWithMap( + toMap(castMap(arguments.get("outputs"), String.class, String.class, + "implicit outputs of the rule class")))); + } + } + + builder.setConfiguredTargetFunction( + (UserDefinedFunction) arguments.get("implementation")); + builder.setRuleDefinitionEnvironment((SkylarkEnvironment) funcallEnv); + return new RuleFunction(builder, type); + } + }; + + // This class is needed for testing + static final class RuleFunction extends AbstractFunction { + // Note that this means that we can reuse the same builder. + // This is fine since we don't modify the builder from here. + private final RuleClass.Builder builder; + private final RuleClassType type; + + public RuleFunction(Builder builder, RuleClassType type) { + super("rule"); + this.builder = builder; + this.type = type; + } + + @Override + public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast, + Environment env) throws EvalException, InterruptedException { + try { + String ruleClassName = ast.getFunction().getName(); + if (ruleClassName.startsWith("_")) { + throw new EvalException(ast.getLocation(), "Invalid rule class name '" + ruleClassName + + "', cannot be private"); + } + if (type == RuleClassType.TEST != TargetUtils.isTestRuleName(ruleClassName)) { + throw new EvalException(ast.getLocation(), "Invalid rule class name '" + ruleClassName + + "', test rule class names must end with '_test' and other rule classes must not"); + } + RuleClass ruleClass = builder.build(ruleClassName); + PackageContext pkgContext = (PackageContext) env.lookup(PackageFactory.PKG_CONTEXT); + return RuleFactory.createAndAddRule(pkgContext, ruleClass, kwargs, ast); + } catch (InvalidRuleException | NameConflictException | NoSuchVariableException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + } + + @VisibleForTesting + RuleClass.Builder getBuilder() { + return builder; + } + } + + @SkylarkBuiltin(name = "Label", doc = "Creates a Label referring to a BUILD target. Use " + + "this function only when you want to give a default value for the label attributes. " + + "Example: <br><pre class=language-python>Label(\"//tools:default\")</pre>", + returnType = Label.class, + mandatoryParams = {@Param(name = "label_string", type = String.class, + doc = "the label string")}) + private static final SkylarkFunction label = new SimpleSkylarkFunction("Label") { + @Override + public Object call(Map<String, Object> arguments, Location loc) throws EvalException, + ConversionException { + String labelString = (String) arguments.get("label_string"); + try { + return labelCache.get(labelString); + } catch (ExecutionException e) { + throw new EvalException(loc, "Illegal absolute label syntax: " + labelString); + } + } + }; + + @SkylarkBuiltin(name = "FileType", + doc = "Creates a file filter from a list of strings. For example, to match files ending " + + "with .cc or .cpp, use: <pre class=language-python>FileType([\".cc\", \".cpp\"])</pre>", + returnType = SkylarkFileType.class, + mandatoryParams = { + @Param(name = "types", type = SkylarkList.class, generic1 = String.class, + doc = "a list of the accepted file extensions")}) + private static final SkylarkFunction fileType = new SimpleSkylarkFunction("FileType") { + @Override + public Object call(Map<String, Object> arguments, Location loc) throws EvalException, + ConversionException { + return SkylarkFileType.of(castList(arguments.get("types"), String.class)); + } + }; + + @SkylarkBuiltin(name = "to_proto", + doc = "Creates a text message from the struct parameter. This method only works if all " + + "struct elements (recursively) are strings, ints, booleans, other structs or a " + + "list of these types. Quotes and new lines in strings are escaped. " + + "Examples:<br><pre class=language-python>" + + "struct(key=123).to_proto()\n# key: 123\n\n" + + "struct(key=True).to_proto()\n# key: true\n\n" + + "struct(key=[1, 2, 3]).to_proto()\n# key: 1\n# key: 2\n# key: 3\n\n" + + "struct(key='text').to_proto()\n# key: \"text\"\n\n" + + "struct(key=struct(inner_key='text')).to_proto()\n" + + "# key {\n# inner_key: \"text\"\n# }\n\n" + + "struct(key=[struct(inner_key=1), struct(inner_key=2)]).to_proto()\n" + + "# key {\n# inner_key: 1\n# }\n# key {\n# inner_key: 2\n# }\n\n" + + "struct(key=struct(inner_key=struct(inner_inner_key='text'))).to_proto()\n" + + "# key {\n# inner_key {\n# inner_inner_key: \"text\"\n# }\n# }\n</pre>", + objectType = SkylarkClassObject.class, returnType = String.class) + private static final SkylarkFunction toProto = new SimpleSkylarkFunction("to_proto") { + @Override + public Object call(Map<String, Object> arguments, Location loc) throws EvalException, + ConversionException { + ClassObject object = (ClassObject) arguments.get("self"); + StringBuilder sb = new StringBuilder(); + printTextMessage(object, sb, 0, loc); + return sb.toString(); + } + + private void printTextMessage(ClassObject object, StringBuilder sb, + int indent, Location loc) throws EvalException { + for (String key : object.getKeys()) { + printTextMessage(key, object.getValue(key), sb, indent, loc); + } + } + + private void printSimpleTextMessage(String key, Object value, StringBuilder sb, + int indent, Location loc, String container) throws EvalException { + if (value instanceof ClassObject) { + print(sb, key + " {", indent); + printTextMessage((ClassObject) value, sb, indent + 1, loc); + print(sb, "}", indent); + } else if (value instanceof String) { + print(sb, key + ": \"" + escape((String) value) + "\"", indent); + } else if (value instanceof Integer) { + print(sb, key + ": " + value, indent); + } else if (value instanceof Boolean) { + // We're relying on the fact that Java converts Booleans to Strings in the same way + // as the protocol buffers do. + print(sb, key + ": " + value, indent); + } else { + throw new EvalException(loc, + "Invalid text format, expected a struct, a string, a bool, or an int but got a " + + EvalUtils.getDatatypeName(value) + " for " + container + " '" + key + "'"); + } + } + + private void printTextMessage(String key, Object value, StringBuilder sb, + int indent, Location loc) throws EvalException { + if (value instanceof SkylarkList) { + for (Object item : ((SkylarkList) value)) { + // TODO(bazel-team): There should be some constraint on the fields of the structs + // in the same list but we ignore that for now. + printSimpleTextMessage(key, item, sb, indent, loc, "list element in struct field"); + } + } else { + printSimpleTextMessage(key, value, sb, indent, loc, "struct field"); + } + } + + private String escape(String string) { + // TODO(bazel-team): use guava's SourceCodeEscapers when it's released. + return string.replace("\"", "\\\"").replace("\n", "\\n"); + } + + private void print(StringBuilder sb, String text, int indent) { + for (int i = 0; i < indent; i++) { + sb.append(" "); + } + sb.append(text); + sb.append("\n"); + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleConfiguredTargetBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleConfiguredTargetBuilder.java new file mode 100644 index 0000000..528e0f1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleConfiguredTargetBuilder.java
@@ -0,0 +1,213 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules; + +import static com.google.devtools.build.lib.syntax.SkylarkFunction.cast; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Function; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; + +/** + * A helper class to build Rule Configured Targets via runtime loaded rule implementations + * defined using the Skylark Build Extension Language. This is experimental code. + */ +public final class SkylarkRuleConfiguredTargetBuilder { + + /** + * Create a Rule Configured Target from the ruleContext and the ruleImplementation. + */ + public static ConfiguredTarget buildRule(RuleContext ruleContext, + Function ruleImplementation) { + String expectError = ruleContext.attributes().get("expect_failure", Type.STRING); + try { + SkylarkRuleContext skylarkRuleContext = new SkylarkRuleContext(ruleContext); + SkylarkEnvironment env = ruleContext.getRule().getRuleClassObject() + .getRuleDefinitionEnvironment().cloneEnv( + ruleContext.getAnalysisEnvironment().getEventHandler()); + // Collect the symbols to disable statically and pass at the next call, so we don't need to + // clone the RuleDefinitionEnvironment. + env.disableOnlyLoadingPhaseObjects(); + Object target = ruleImplementation.call(ImmutableList.<Object>of(skylarkRuleContext), + ImmutableMap.<String, Object>of(), null, env); + + if (ruleContext.hasErrors()) { + return null; + } else if (!(target instanceof SkylarkClassObject) && target != Environment.NONE) { + ruleContext.ruleError("Rule implementation doesn't return a struct"); + return null; + } else if (!expectError.isEmpty()) { + ruleContext.ruleError("Expected error not found: " + expectError); + return null; + } + ConfiguredTarget configuredTarget = createTarget(ruleContext, target); + checkOrphanArtifacts(ruleContext); + return configuredTarget; + + } catch (InterruptedException e) { + ruleContext.ruleError(e.getMessage()); + return null; + } catch (EvalException e) { + // If the error was expected, return an empty target. + if (!expectError.isEmpty() && e.getMessage().matches(expectError)) { + return new com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder(ruleContext) + .add(RunfilesProvider.class, RunfilesProvider.EMPTY) + .build(); + } + ruleContext.ruleError("\n" + e.print()); + return null; + } + } + + private static void checkOrphanArtifacts(RuleContext ruleContext) throws EvalException { + ImmutableSet<Artifact> orphanArtifacts = + ruleContext.getAnalysisEnvironment().getOrphanArtifacts(); + if (!orphanArtifacts.isEmpty()) { + throw new EvalException(null, "The following files have no generating action:\n" + + Joiner.on("\n").join(Iterables.transform(orphanArtifacts, + new com.google.common.base.Function<Artifact, String>() { + @Override + public String apply(Artifact artifact) { + return artifact.getRootRelativePathString(); + } + }))); + } + } + + // TODO(bazel-team): this whole defaulting - overriding executable, runfiles and files_to_build + // is getting out of hand. Clean this whole mess up. + private static ConfiguredTarget createTarget(RuleContext ruleContext, Object target) + throws EvalException { + Artifact executable = getExecutable(ruleContext, target); + RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(ruleContext); + // Set the default files to build. + NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.<Artifact>stableOrder() + .addAll(ruleContext.getOutputArtifacts()); + if (executable != null) { + filesToBuild.add(executable); + } + builder.setFilesToBuild(filesToBuild.build()); + return addStructFields(ruleContext, builder, target, executable); + } + + private static Artifact getExecutable(RuleContext ruleContext, Object target) + throws EvalException { + Artifact executable = ruleContext.getRule().getRuleClassObject().outputsDefaultExecutable() + // This doesn't actually create a new Artifact just returns the one + // created in SkylarkruleContext. + ? ruleContext.createOutputArtifact() : null; + if (target instanceof SkylarkClassObject) { + SkylarkClassObject struct = (SkylarkClassObject) target; + if (struct.getValue("executable") != null) { + // We need this because of genrule.bzl. This overrides the default executable. + executable = cast( + struct.getValue("executable"), Artifact.class, "executable", struct.getCreationLoc()); + } + } + return executable; + } + + private static ConfiguredTarget addStructFields(RuleContext ruleContext, + RuleConfiguredTargetBuilder builder, Object target, Artifact executable) + throws EvalException { + Location loc = null; + Runfiles statelessRunfiles = null; + Runfiles dataRunfiles = null; + Runfiles defaultRunfiles = null; + if (target instanceof SkylarkClassObject) { + SkylarkClassObject struct = (SkylarkClassObject) target; + loc = struct.getCreationLoc(); + for (String key : struct.getKeys()) { + if (key.equals("files")) { + // If we specify files_to_build we don't have the executable in it by default. + builder.setFilesToBuild(cast(struct.getValue("files"), + SkylarkNestedSet.class, "files", loc).getSet(Artifact.class)); + } else if (key.equals("runfiles")) { + statelessRunfiles = cast(struct.getValue("runfiles"), Runfiles.class, "runfiles", loc); + } else if (key.equals("data_runfiles")) { + dataRunfiles = + cast(struct.getValue("data_runfiles"), Runfiles.class, "data_runfiles", loc); + } else if (key.equals("default_runfiles")) { + defaultRunfiles = + cast(struct.getValue("default_runfiles"), Runfiles.class, "default_runfiles", loc); + } else if (!key.equals("executable")) { + // We handled executable already. + builder.addSkylarkTransitiveInfo(key, struct.getValue(key), loc); + } + } + } + + if ((statelessRunfiles != null) && (dataRunfiles != null || defaultRunfiles != null)) { + throw new EvalException(loc, "Cannot specify the provider 'runfiles' " + + "together with 'data_runfiles' or 'default_runfiles'"); + } + + if (statelessRunfiles == null && dataRunfiles == null && defaultRunfiles == null) { + // No runfiles specified, set default + statelessRunfiles = Runfiles.EMPTY; + } + + RunfilesProvider runfilesProvider = statelessRunfiles != null + ? RunfilesProvider.simple(merge(statelessRunfiles, executable)) + : RunfilesProvider.withData( + // The executable doesn't get into the default runfiles if we have runfiles states. + // This is to keep skylark genrule consistent with the original genrule. + defaultRunfiles != null ? defaultRunfiles : Runfiles.EMPTY, + dataRunfiles != null ? dataRunfiles : Runfiles.EMPTY); + builder.addProvider(RunfilesProvider.class, runfilesProvider); + + Runfiles computedDefaultRunfiles = runfilesProvider.getDefaultRunfiles(); + // This works because we only allowed to call a rule *_test iff it's a test type rule. + boolean testRule = TargetUtils.isTestRuleName(ruleContext.getRule().getRuleClass()); + if (testRule && computedDefaultRunfiles.isEmpty()) { + throw new EvalException(loc, "Test rules have to define runfiles"); + } + if (executable != null || testRule) { + RunfilesSupport runfilesSupport = computedDefaultRunfiles.isEmpty() + ? null : RunfilesSupport.withExecutable(ruleContext, computedDefaultRunfiles, executable); + builder.setRunfilesSupport(runfilesSupport, executable); + } + try { + return builder.build(); + } catch (IllegalArgumentException e) { + throw new EvalException(loc, e.getMessage()); + } + } + + private static Runfiles merge(Runfiles runfiles, Artifact executable) { + if (executable == null) { + return runfiles; + } + return new Runfiles.Builder().addArtifact(executable).merge(runfiles).build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleContext.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleContext.java new file mode 100644 index 0000000..fc06677 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleContext.java
@@ -0,0 +1,484 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules; + +import com.google.common.base.Preconditions; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.ConfigurationMakeVariableContext; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.LabelExpander; +import com.google.devtools.build.lib.analysis.LabelExpander.NotUniqueExpansionException; +import com.google.devtools.build.lib.analysis.MakeVariableExpander.ExpansionException; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SkylarkImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.RawAttributeMapper; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.shell.ShellUtils.TokenizationException; +import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.FuncallExpression.FuncallException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.syntax.SkylarkType; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A Skylark API for the ruleContext. + */ +@SkylarkModule(name = "ctx", doc = "The context of the rule containing helper functions and " + + "information about attributes, depending targets and outputs. " + + "You get a ctx object as an argument to the <code>implementation</code> function when " + + "you create a rule.") +public final class SkylarkRuleContext { + + public static final String PROVIDER_CLASS_PREFIX = "com.google.devtools.build.lib."; + + static final LoadingCache<String, Class<?>> classCache = CacheBuilder.newBuilder() + .initialCapacity(10) + .maximumSize(100) + .build(new CacheLoader<String, Class<?>>() { + + @Override + public Class<?> load(String key) throws Exception { + String classPath = SkylarkRuleContext.PROVIDER_CLASS_PREFIX + key; + return Class.forName(classPath); + } + }); + + private final RuleContext ruleContext; + + // TODO(bazel-team): support configurable attributes. + private final SkylarkClassObject attrObject; + + private final SkylarkClassObject outputsObject; + + private final SkylarkClassObject executableObject; + + private final SkylarkClassObject fileObject; + + private final SkylarkClassObject filesObject; + + private final SkylarkClassObject targetsObject; + + private final SkylarkClassObject targetObject; + + // TODO(bazel-team): we only need this because of the css_binary rule. + private final ImmutableMap<Artifact, Label> artifactLabelMap; + + private final ImmutableMap<Artifact, FilesToRunProvider> executableRunfilesMap; + + /** + * In native code, private values start with $. + * In Skylark, private values start with _, because of the grammar. + */ + private String attributeToSkylark(String oldName) { + if (!oldName.isEmpty() && (oldName.charAt(0) == '$' || oldName.charAt(0) == ':')) { + return "_" + oldName.substring(1); + } + return oldName; + } + + /** + * Creates a new SkylarkRuleContext using ruleContext. + */ + public SkylarkRuleContext(RuleContext ruleContext) throws EvalException { + this.ruleContext = Preconditions.checkNotNull(ruleContext); + + HashMap<String, Object> outputsBuilder = new HashMap<>(); + if (ruleContext.getRule().getRuleClassObject().outputsDefaultExecutable()) { + addOutput(outputsBuilder, "executable", ruleContext.createOutputArtifact()); + } + ImplicitOutputsFunction implicitOutputsFunction = + ruleContext.getRule().getRuleClassObject().getImplicitOutputsFunction(); + + if (implicitOutputsFunction instanceof SkylarkImplicitOutputsFunction) { + SkylarkImplicitOutputsFunction func = (SkylarkImplicitOutputsFunction) + ruleContext.getRule().getRuleClassObject().getImplicitOutputsFunction(); + for (Map.Entry<String, String> entry : func.calculateOutputs( + RawAttributeMapper.of(ruleContext.getRule())).entrySet()) { + addOutput(outputsBuilder, entry.getKey(), + ruleContext.getImplicitOutputArtifact(entry.getValue())); + } + } + + ImmutableMap.Builder<Artifact, Label> artifactLabelMapBuilder = + ImmutableMap.builder(); + for (Attribute a : ruleContext.getRule().getAttributes()) { + String attrName = a.getName(); + Type<?> type = a.getType(); + if (type != Type.OUTPUT && type != Type.OUTPUT_LIST) { + continue; + } + ImmutableList.Builder<Artifact> artifactsBuilder = ImmutableList.builder(); + for (OutputFile outputFile : ruleContext.getRule().getOutputFileMap().get(attrName)) { + Artifact artifact = ruleContext.createOutputArtifact(outputFile); + artifactsBuilder.add(artifact); + artifactLabelMapBuilder.put(artifact, outputFile.getLabel()); + } + ImmutableList<Artifact> artifacts = artifactsBuilder.build(); + + if (type == Type.OUTPUT) { + if (artifacts.size() == 1) { + addOutput(outputsBuilder, attrName, Iterables.getOnlyElement(artifacts)); + } else { + addOutput(outputsBuilder, attrName, Environment.NONE); + } + } else if (type == Type.OUTPUT_LIST) { + addOutput(outputsBuilder, attrName, + SkylarkList.list(artifacts, Artifact.class)); + } else { + throw new IllegalArgumentException( + "Type of " + attrName + "(" + type + ") is not output type "); + } + } + artifactLabelMap = artifactLabelMapBuilder.build(); + outputsObject = new SkylarkClassObject(outputsBuilder, "No such output '%s'"); + + ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>(); + ImmutableMap.Builder<String, Object> executableBuilder = new ImmutableMap.Builder<>(); + ImmutableMap.Builder<Artifact, FilesToRunProvider> executableRunfilesbuilder = + new ImmutableMap.Builder<>(); + ImmutableMap.Builder<String, Object> fileBuilder = new ImmutableMap.Builder<>(); + ImmutableMap.Builder<String, Object> filesBuilder = new ImmutableMap.Builder<>(); + ImmutableMap.Builder<String, Object> targetBuilder = new ImmutableMap.Builder<>(); + ImmutableMap.Builder<String, Object> targetsBuilder = new ImmutableMap.Builder<>(); + for (Attribute a : ruleContext.getRule().getAttributes()) { + Type<?> type = a.getType(); + Object val = ruleContext.attributes().get(a.getName(), type); + builder.put(attributeToSkylark(a.getName()), val == null ? Environment.NONE + // Attribute values should be type safe + : SkylarkType.convertToSkylark(val, null)); + if (type != Type.LABEL && type != Type.LABEL_LIST) { + continue; + } + String skyname = attributeToSkylark(a.getName()); + Mode mode = getMode(a.getName()); + if (a.isExecutable()) { + // In Skylark only label (not label list) type attributes can have the Executable flag. + FilesToRunProvider provider = ruleContext.getExecutablePrerequisite(a.getName(), mode); + if (provider != null && provider.getExecutable() != null) { + Artifact executable = provider.getExecutable(); + executableBuilder.put(skyname, executable); + executableRunfilesbuilder.put(executable, provider); + } else { + executableBuilder.put(skyname, Environment.NONE); + } + } + if (a.isSingleArtifact()) { + // In Skylark only label (not label list) type attributes can have the SingleArtifact flag. + Artifact artifact = ruleContext.getPrerequisiteArtifact(a.getName(), mode); + if (artifact != null) { + fileBuilder.put(skyname, artifact); + } else { + fileBuilder.put(skyname, Environment.NONE); + } + } + filesBuilder.put(skyname, ruleContext.getPrerequisiteArtifacts(a.getName(), mode).list()); + targetsBuilder.put(skyname, SkylarkList.list( + ruleContext.getPrerequisites(a.getName(), mode), TransitiveInfoCollection.class)); + if (type == Type.LABEL) { + Object prereq = ruleContext.getPrerequisite(a.getName(), mode); + if (prereq != null) { + targetBuilder.put(skyname, prereq); + } else { + targetBuilder.put(skyname, Environment.NONE); + } + } + } + attrObject = new SkylarkClassObject(builder.build(), "No such attribute '%s'"); + executableObject = new SkylarkClassObject(executableBuilder.build(), "No such executable. " + + "Make sure there is a '%s' label type attribute marked as 'executable'"); + fileObject = new SkylarkClassObject(fileBuilder.build(), + "No such file. Make sure there is a '%s' label type attribute marked as 'single_file'"); + filesObject = new SkylarkClassObject(filesBuilder.build(), + "No such files. Make sure there is a '%s' label or label_list type attribute"); + targetObject = new SkylarkClassObject(targetBuilder.build(), + "No such target. Make sure there is a '%s' label type attribute"); + targetsObject = new SkylarkClassObject(targetsBuilder.build(), + "No such targets. Make sure there is a '%s' label or label_list type attribute"); + executableRunfilesMap = executableRunfilesbuilder.build(); + } + + private void addOutput(HashMap<String, Object> outputsBuilder, String key, Object value) + throws EvalException { + if (outputsBuilder.containsKey(key)) { + throw new EvalException(null, "Multiple outputs with the same key: " + key); + } + outputsBuilder.put(key, value); + } + + /** + * Returns the original ruleContext. + */ + public RuleContext getRuleContext() { + return ruleContext; + } + + private Mode getMode(String attributeName) { + return ruleContext.getAttributeMode(attributeName); + } + + @SkylarkCallable(name = "attr", structField = true, + doc = "A struct to access the values of the attributes. The values are provided by " + + "the user (if not, a default value is used).") + public SkylarkClassObject getAttr() { + return attrObject; + } + + /** + * <p>See {@link RuleContext#getExecutablePrerequisite(String, Mode)}. + */ + @SkylarkCallable(name = "executable", structField = true, + doc = "A <code>struct</code> containing executable files defined in label type " + + "attributes marked as <code>executable=True</code>. The struct fields correspond " + + "to the attribute names. The struct value is always a <code>file</code>s or " + + "<code>None</code>. If a non-mandatory attribute is not specified in the rule " + + "the corresponding struct value is <code>None</code>. If a label type is not " + + "marked as <code>executable=True</code>, no corresponding struct field is generated.") + public SkylarkClassObject getExecutable() { + return executableObject; + } + + /** + * See {@link RuleContext#getPrerequisiteArtifact(String, Mode)}. + */ + @SkylarkCallable(name = "file", structField = true, + doc = "A <code>struct</code> containing files defined in label type " + + "attributes marked as <code>single_file=True</code>. The struct fields correspond " + + "to the attribute names. The struct value is always a <code>file</code> or " + + "<code>None</code>. If a non-mandatory attribute is not specified in the rule " + + "the corresponding struct value is <code>None</code>. If a label type is not " + + "marked as <code>single_file=True</code>, no corresponding struct field is generated.") + public SkylarkClassObject getFile() { + return fileObject; + } + + /** + * See {@link RuleContext#getPrerequisiteArtifacts(String, Mode)}. + */ + @SkylarkCallable(name = "files", structField = true, + doc = "A <code>struct</code> containing files defined in label or label list " + + "type attributes. The struct fields correspond to the attribute names. The struct " + + "values are <code>list</code> of <code>file</code>s. If a non-mandatory attribute is " + + "not specified in the rule, an empty list is generated.") + public SkylarkClassObject getFiles() { + return filesObject; + } + + /** + * See {@link RuleContext#getPrerequisite(String, Mode)}. + */ + @SkylarkCallable(name = "target", structField = true, + doc = "A <code>struct</code> containing prerequisite targets defined in label type " + + "attributes. The struct fields correspond to the attribute names. The struct value " + + "is always a <code>target</code> or <code>None</code>. If a non-mandatory attribute " + + "is not specified in the rule, the corresponding struct value is <code>None</code>.") + public SkylarkClassObject getTarget() { + return targetObject; + } + + /** + * See {@link RuleContext#getPrerequisites(String, Mode)}. + */ + @SkylarkCallable(name = "targets", structField = true, + doc = "A <code>struct</code> containing prerequisite targets defined in label or label list " + + "type attributes. The struct fields correspond to the attribute names. The struct " + + "values are <code>list</code> of <code>target</code>s. If a non-mandatory attribute is " + + "not specified in the rule, an empty list is generated.") + public SkylarkClassObject getTargets() { + return targetsObject; + } + + @SkylarkCallable(name = "label", structField = true, doc = "The label of this rule.") + public Label getLabel() { + return ruleContext.getLabel(); + } + + @SkylarkCallable(name = "configuration", structField = true, + doc = "Returns the default configuration. See the <code>configuration</code> type for " + + "more details.") + public BuildConfiguration getConfiguration() { + return ruleContext.getConfiguration(); + } + + @SkylarkCallable(name = "host_configuration", structField = true, + doc = "Returns the host configuration. See the <code>configuration</code> type for " + + "more details.") + public BuildConfiguration getHostConfiguration() { + return ruleContext.getHostConfiguration(); + } + + @SkylarkCallable(name = "data_configuration", structField = true, + doc = "Returns the data configuration. See the <code>configuration</code> type for " + + "more details.") + public BuildConfiguration getDataConfiguration() { + return ruleContext.getConfiguration().getConfiguration(ConfigurationTransition.DATA); + } + + @SkylarkCallable(structField = true, + doc = "A <code>struct</code> containing all the output files." + + " The struct is generated the following way:<br>" + + "<ul><li>If the rule is marked as <code>executable=True</code> the struct has an " + + "\"executable\" field with the rules default executable <code>file</code> value." + + "<li>For every entry in the rule's <code>outputs</code> dict a field is generated with " + + "the same name and the corresponding <code>file</code> value." + + "<li>For every output type attribute a struct field is generated with the " + + "same name and the corresponding <code>file</code> value or <code>None</code>, " + + "if no value is specified in the rule." + + "<li>For every output list type attribute a struct field is generated with the " + + "same name and corresponding <code>list</code> of <code>file</code>s value " + + "(an empty list if no value is specified in the rule.</ul>") + public SkylarkClassObject outputs() { + return outputsObject; + } + + @Override + public String toString() { + return ruleContext.getLabel().toString(); + } + + @SkylarkCallable(doc = "Splits a shell command to a list of tokens.", hidden = true) + public List<String> tokenize(String optionString) throws FuncallException { + List<String> options = new ArrayList<String>(); + try { + ShellUtils.tokenize(options, optionString); + } catch (TokenizationException e) { + throw new FuncallException(e.getMessage() + " while tokenizing '" + optionString + "'"); + } + return ImmutableList.copyOf(options); + } + + @SkylarkCallable(doc = + "Expands all references to labels embedded within a string for all files using a mapping " + + "from definition labels (i.e. the label in the output type attribute) to files. Deprecated.", + hidden = true) + public String expand(@Nullable String expression, + List<Artifact> artifacts, Label labelResolver) throws FuncallException { + try { + Map<Label, Iterable<Artifact>> labelMap = new HashMap<>(); + for (Artifact artifact : artifacts) { + labelMap.put(artifactLabelMap.get(artifact), ImmutableList.of(artifact)); + } + return LabelExpander.expand(expression, labelMap, labelResolver); + } catch (NotUniqueExpansionException e) { + throw new FuncallException(e.getMessage() + " while expanding '" + expression + "'"); + } + } + + @SkylarkCallable(doc = + "Creates a file with the given filename. You must create an action that generates " + + "the file. If the file should be publicly visible, declare a rule " + + "output instead when possible.") + public Artifact newFile(Root root, String filename) { + PathFragment fragment = ruleContext.getLabel().getPackageFragment(); + for (String pathFragmentString : filename.split("/")) { + fragment = fragment.getRelative(pathFragmentString); + } + return ruleContext.getAnalysisEnvironment().getDerivedArtifact(fragment, root); + } + + @SkylarkCallable(doc = + "Creates a new file, derived from the given file and suffix. " + + "You must create an action that generates " + + "the file. If the file should be publicly visible, declare a rule " + + "output instead when possible.") + public Artifact newFile(Root root, Artifact baseArtifact, String suffix) { + PathFragment original = baseArtifact.getRootRelativePath(); + PathFragment fragment = original.replaceName(original.getBaseName() + suffix); + return ruleContext.getAnalysisEnvironment().getDerivedArtifact(fragment, root); + } + + @SkylarkCallable(doc = "", hidden = true) + public NestedSet<Artifact> middleMan(String attribute) { + return AnalysisUtils.getMiddlemanFor(ruleContext, attribute); + } + + @SkylarkCallable(doc = "", hidden = true) + public boolean checkPlaceholders(String template, List<String> allowedPlaceholders) { + List<String> actualPlaceHolders = new LinkedList<>(); + Set<String> allowedPlaceholderSet = ImmutableSet.copyOf(allowedPlaceholders); + ImplicitOutputsFunction.createPlaceholderSubstitutionFormatString(template, actualPlaceHolders); + for (String placeholder : actualPlaceHolders) { + if (!allowedPlaceholderSet.contains(placeholder)) { + return false; + } + } + return true; + } + + @SkylarkCallable(doc = "") + public String expandMakeVariables(String attributeName, String command, + final Map<String, String> additionalSubstitutions) { + return ruleContext.expandMakeVariables(attributeName, + command, new ConfigurationMakeVariableContext(ruleContext.getRule().getPackage(), + ruleContext.getConfiguration()) { + @Override + public String lookupMakeVariable(String name) throws ExpansionException { + if (additionalSubstitutions.containsKey(name)) { + return additionalSubstitutions.get(name); + } else { + return super.lookupMakeVariable(name); + } + } + }); + } + + FilesToRunProvider getExecutableRunfiles(Artifact executable) { + return executableRunfilesMap.get(executable); + } + + @SkylarkCallable(name = "info_file", structField = true, hidden = true, + doc = "Returns the file that is used to hold the non-volatile workspace status for the " + + "current build request.") + public Artifact getStableWorkspaceStatus() { + return ruleContext.getAnalysisEnvironment().getStableWorkspaceStatusArtifact(); + } + + @SkylarkCallable(name = "version_file", structField = true, hidden = true, + doc = "Returns the file that is used to hold the volatile workspace status for the " + + "current build request.") + public Artifact getVolatileWorkspaceStatus() { + return ruleContext.getAnalysisEnvironment().getVolatileWorkspaceStatusArtifact(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java new file mode 100644 index 0000000..1f7d160 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkRuleImplementationFunctions.java
@@ -0,0 +1,367 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.CommandHelper; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.MakeVariableExpander; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction; +import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.Substitution; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.Type.ConversionException; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.EvalUtils; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin; +import com.google.devtools.build.lib.syntax.SkylarkBuiltin.Param; +import com.google.devtools.build.lib.syntax.SkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkFunction.SimpleSkylarkFunction; +import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Map; +import java.util.concurrent.ExecutionException; + +// TODO(bazel-team): function argument names are often duplicated, +// figure out a nicely readable way to get rid of the duplications. +/** + * A helper class to provide an easier API for Skylark rule implementations + * and hide the original Java API. This is experimental code. + */ +public class SkylarkRuleImplementationFunctions { + + // TODO(bazel-team): add all the remaining parameters + // TODO(bazel-team): merge executable and arguments + /** + * A Skylark built-in function to create and register a SpawnAction using a + * dictionary of parameters: + * createSpawnAction( + * inputs = [input1, input2, ...], + * outputs = [output1, output2, ...], + * executable = executable, + * arguments = [argument1, argument2, ...], + * mnemonic = 'mnemonic', + * command = 'command', + * register = 1 + * ) + */ + @SkylarkBuiltin(name = "action", + doc = "Creates an action that runs an executable or a shell command.", + objectType = SkylarkRuleContext.class, + returnType = Environment.NoneType.class, + mandatoryParams = { + @Param(name = "outputs", type = SkylarkList.class, generic1 = Artifact.class, + doc = "list of the output files of the action")}, + optionalParams = { + @Param(name = "inputs", type = SkylarkList.class, generic1 = Artifact.class, + doc = "list of the input files of the action"), + @Param(name = "executable", doc = "the executable file to be called by the action"), + @Param(name = "arguments", type = SkylarkList.class, generic1 = String.class, + doc = "command line arguments of the action"), + @Param(name = "mnemonic", type = String.class, doc = "mnemonic"), + @Param(name = "command", doc = "shell command to execute"), + @Param(name = "command_line", doc = "a command line to execute"), + @Param(name = "progress_message", type = String.class, + doc = "progress message to show to the user during the build"), + @Param(name = "use_default_shell_env", type = Boolean.class, + doc = "whether the action should use the built in shell environment or not"), + @Param(name = "env", type = Map.class, doc = "sets the dictionary of environment variables"), + @Param(name = "execution_requirements", type = Map.class, + doc = "information for scheduling the action"), + @Param(name = "input_manifests", type = Map.class, + doc = "sets the map of input manifests files; " + + "they are typicially generated by the command_helper")}) + private static final SkylarkFunction createSpawnAction = + new SimpleSkylarkFunction("action") { + + @Override + public Object call(Map<String, Object> params, Location loc) throws EvalException, + ConversionException { + SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self"); + SpawnAction.Builder builder = new SpawnAction.Builder(); + // TODO(bazel-team): builder still makes unnecessary copies of inputs, outputs and args. + builder.addInputs(castList(params.get("inputs"), Artifact.class)); + builder.addOutputs(castList(params.get("outputs"), Artifact.class)); + builder.addArguments(castList(params.get("arguments"), String.class)); + if (params.containsKey("executable")) { + Object exe = params.get("executable"); + if (exe instanceof Artifact) { + Artifact executable = (Artifact) exe; + builder.addInput(executable); + FilesToRunProvider provider = ctx.getExecutableRunfiles(executable); + if (provider == null) { + builder.setExecutable((Artifact) exe); + } else { + builder.setExecutable(provider); + } + } else if (exe instanceof PathFragment) { + builder.setExecutable((PathFragment) exe); + } else { + throw new EvalException(loc, "expected file or PathFragment for " + + "executable but got " + EvalUtils.getDatatypeName(exe) + " instead"); + } + } + if (params.containsKey("command") == params.containsKey("executable")) { + throw new EvalException(loc, "You must specify either 'command' or 'executable' argument"); + } + if (params.containsKey("command")) { + Object command = params.get("command"); + if (command instanceof String) { + builder.setShellCommand((String) command); + } else if (command instanceof SkylarkList) { + SkylarkList commandList = (SkylarkList) command; + if (commandList.size() < 3) { + throw new EvalException(loc, "'command' list has to be of size at least 3"); + } + builder.setShellCommand(castList(commandList, String.class, "command")); + } else { + throw new EvalException(loc, "expected string or list of strings for " + + "command instead of " + EvalUtils.getDatatypeName(command)); + } + } + if (params.containsKey("command_line")) { + builder.setCommandLine(CommandLine.ofCharSequences(ImmutableList.copyOf(castList( + params.get("command_line"), CharSequence.class, "command line")))); + } + if (params.containsKey("mnemonic")) { + builder.setMnemonic((String) params.get("mnemonic")); + } + if (params.containsKey("env")) { + builder.setEnvironment( + toMap(castMap(params.get("env"), String.class, String.class, "env"))); + } + if (params.containsKey("progress_message")) { + builder.setProgressMessage((String) params.get("progress_message")); + } + if (params.containsKey("use_default_shell_env") + && EvalUtils.toBoolean(params.get("use_default_shell_env"))) { + builder.useDefaultShellEnvironment(); + } + if (params.containsKey("execution_requirements")) { + builder.setExecutionInfo(toMap(castMap(params.get("execution_requirements"), + String.class, String.class, "execution_requirements"))); + } + if (params.containsKey("input_manifests")) { + for (Map.Entry<PathFragment, Artifact> entry : castMap(params.get("input_manifests"), + PathFragment.class, Artifact.class, "input manifest file map")) { + builder.addInputManifest(entry.getValue(), entry.getKey()); + } + } + // Always register the action + ctx.getRuleContext().registerAction(builder.build(ctx.getRuleContext())); + return Environment.NONE; + } + }; + + // TODO(bazel-team): improve this method to be more memory friendly + @SkylarkBuiltin(name = "file_action", + doc = "Creates a file write action.", + objectType = SkylarkRuleContext.class, + returnType = Environment.NoneType.class, + optionalParams = { + @Param(name = "executable", type = Boolean.class, + doc = "whether the output file should be executable (default is False)"), + }, + mandatoryParams = { + @Param(name = "output", type = Artifact.class, doc = "the output file"), + @Param(name = "content", type = String.class, doc = "the contents of the file")}) + private static final SkylarkFunction createFileWriteAction = + new SimpleSkylarkFunction("file_action") { + + @Override + public Object call(Map<String, Object> params, Location loc) throws EvalException, + ConversionException { + SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self"); + boolean executable = params.containsKey("executable") && (Boolean) params.get("executable"); + FileWriteAction action = new FileWriteAction( + ctx.getRuleContext().getActionOwner(), + (Artifact) params.get("output"), + (String) params.get("content"), + executable); + ctx.getRuleContext().registerAction(action); + return action; + } + }; + + @SkylarkBuiltin(name = "template_action", + doc = "Creates a template expansion action.", + objectType = SkylarkRuleContext.class, + returnType = Environment.NoneType.class, + mandatoryParams = { + @Param(name = "template", type = Artifact.class, doc = "the template file"), + @Param(name = "output", type = Artifact.class, doc = "the output file"), + @Param(name = "substitutions", type = Map.class, + doc = "substitutions to make when expanding the template")}, + optionalParams = { + @Param(name = "executable", type = Boolean.class, + doc = "whether the output file should be executable (default is False)")}) + private static final SkylarkFunction createTemplateAction = + new SimpleSkylarkFunction("template_action") { + + @Override + public Object call(Map<String, Object> params, Location loc) throws EvalException, + ConversionException { + SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self"); + ImmutableList.Builder<Substitution> substitutions = ImmutableList.builder(); + for (Map.Entry<String, String> substitution + : castMap(params.get("substitutions"), String.class, String.class, "substitutions")) { + substitutions.add(Substitution.of(substitution.getKey(), substitution.getValue())); + } + + boolean executable = params.containsKey("executable") && (Boolean) params.get("executable"); + TemplateExpansionAction action = new TemplateExpansionAction( + ctx.getRuleContext().getActionOwner(), + (Artifact) params.get("template"), + (Artifact) params.get("output"), + substitutions.build(), + executable); + ctx.getRuleContext().registerAction(action); + return action; + } + }; + + /** + * A built in Skylark helper function to access the + * Transitive info providers of Transitive info collections. + */ + @SkylarkBuiltin(name = "provider", + doc = "Returns the transitive info provider provided by the target.", + mandatoryParams = { + @Param(name = "target", type = TransitiveInfoCollection.class, + doc = "the configured target which provides the provider"), + @Param(name = "type", type = String.class, doc = "the class type of the provider")}) + private static final SkylarkFunction provider = new SimpleSkylarkFunction("provider") { + @Override + public Object call(Map<String, Object> params, Location loc) throws EvalException { + TransitiveInfoCollection target = (TransitiveInfoCollection) params.get("target"); + String type = (String) params.get("type"); + try { + Class<?> classType = SkylarkRuleContext.classCache.get(type); + Class<? extends TransitiveInfoProvider> convertedClass = + classType.asSubclass(TransitiveInfoProvider.class); + Object result = target.getProvider(convertedClass); + return result == null ? Environment.NONE : result; + } catch (ExecutionException e) { + throw new EvalException(loc, "Unknown class type " + type); + } catch (ClassCastException e) { + throw new EvalException(loc, "Not a TransitiveInfoProvider " + type); + } + } + }; + + // TODO(bazel-team): Remove runfile states from Skylark. + @SkylarkBuiltin(name = "runfiles", + doc = "Creates a runfiles object.", + objectType = SkylarkRuleContext.class, + returnType = Runfiles.class, + optionalParams = { + @Param(name = "files", type = SkylarkList.class, generic1 = Artifact.class, + doc = "The list of files to be added to the runfiles."), + // TODO(bazel-team): If we have a memory efficient support for lazy list containing NestedSets + // we can remove this and just use files = [file] + list(set) + @Param(name = "transitive_files", type = SkylarkNestedSet.class, generic1 = Artifact.class, + doc = "The (transitive) set of files to be added to the runfiles."), + @Param(name = "collect_data", type = Boolean.class, doc = "Whether to collect the data " + + "runfiles from the dependencies in srcs, data and deps attributes."), + @Param(name = "collect_default", type = Boolean.class, doc = "Whether to collect the default " + + "runfiles from the dependencies in srcs, data and deps attributes.")}) + private static final SkylarkFunction runfiles = new SimpleSkylarkFunction("runfiles") { + @Override + public Object call(Map<String, Object> params, Location loc) throws EvalException, + ConversionException { + SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self"); + Runfiles.Builder builder = new Runfiles.Builder(); + if (params.containsKey("collect_data") && (Boolean) params.get("collect_data")) { + builder.addRunfiles(ctx.getRuleContext(), RunfilesProvider.DATA_RUNFILES); + } + if (params.containsKey("collect_default") && (Boolean) params.get("collect_default")) { + builder.addRunfiles(ctx.getRuleContext(), RunfilesProvider.DEFAULT_RUNFILES); + } + if (params.containsKey("files")) { + builder.addArtifacts(castList(params.get("files"), Artifact.class)); + } + if (params.containsKey("transitive_files")) { + builder.addTransitiveArtifacts(cast(params.get("transitive_files"), + SkylarkNestedSet.class, "files", loc).getSet(Artifact.class)); + } + return builder.build(); + } + }; + + @SkylarkBuiltin(name = "command_helper", doc = "Creates a command helper class.", + objectType = SkylarkRuleContext.class, + returnType = CommandHelper.class, + mandatoryParams = { + @Param(name = "tools", type = SkylarkList.class, generic1 = TransitiveInfoCollection.class, + doc = "list of tools (list of targets)"), + @Param(name = "label_dict", type = Map.class, + doc = "dictionary of resolved labels and the corresponding list of artifacts " + + "(a dict of Label : list of files)")}) + private static final SkylarkFunction createCommandHelper = + new SimpleSkylarkFunction("command_helper") { + @SuppressWarnings("unchecked") + @Override + protected Object call(Map<String, Object> params, Location loc) + throws ConversionException, EvalException { + SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self"); + return new CommandHelper(ctx.getRuleContext(), + AnalysisUtils.getProviders( + castList(params.get("tools"), TransitiveInfoCollection.class), + FilesToRunProvider.class), + // TODO(bazel-team): this cast to Map is unchecked and is not safe. + // The best way to fix this probably is to convert CommandHelper to Skylark. + ImmutableMap.copyOf((Map<Label, Iterable<Artifact>>) params.get("label_dict"))); + } + }; + + + @SkylarkBuiltin(name = "var", + doc = "get the value bound to a configuration variable in the context", + objectType = SkylarkRuleContext.class, + mandatoryParams = { + @Param(name = "name", type = String.class, doc = "the name of the variable") + }, + returnType = String.class) + private static final SkylarkFunction configurationMakeVariableContext = + new SimpleSkylarkFunction("var") { + @SuppressWarnings("unchecked") + @Override + protected Object call(Map<String, Object> params, Location loc) + throws ConversionException, EvalException { + SkylarkRuleContext ctx = (SkylarkRuleContext) params.get("self"); + String name = (String) params.get("name"); + try { + return ctx.getRuleContext().getConfigurationMakeVariableContext() + .lookupMakeVariable(name); + } catch (MakeVariableExpander.ExpansionException e) { + throw new EvalException(loc, "configuration variable " + + ShellEscaper.escapeString(name) + " not defined"); + } + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcBinary.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcBinary.java new file mode 100644 index 0000000..6efcd9d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcBinary.java
@@ -0,0 +1,635 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ParameterFile; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.DynamicMode; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.rules.test.BaselineCoverageAction; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.util.OsUtils; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * A ConfiguredTarget for <code>cc_binary</code> rules. + */ +public abstract class CcBinary implements RuleConfiguredTargetFactory { + + private final CppSemantics semantics; + + protected CcBinary(CppSemantics semantics) { + this.semantics = semantics; + } + + // TODO(bazel-team): should this use Link.SHARED_LIBRARY_FILETYPES? + private static final FileTypeSet SHARED_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY); + + /** + * The maximum number of inputs for any single .dwp generating action. For cases where + * this value is exceeded, the action is split up into "batches" that fall under the limit. + * See {@link #createDebugPackagerActions} for details. + */ + @VisibleForTesting + public static final int MAX_INPUTS_PER_DWP_ACTION = 100; + + /** + * Intermediate dwps are written to this subdirectory under the main dwp's output path. + */ + @VisibleForTesting + public static final String INTERMEDIATE_DWP_DIR = "_dwps"; + + private static Runfiles collectRunfiles(RuleContext context, + CcCommon common, + CcLinkingOutputs linkingOutputs, + CppCompilationContext cppCompilationContext, + LinkStaticness linkStaticness, + NestedSet<Artifact> filesToBuild, + Iterable<Artifact> fakeLinkerInputs, + boolean fake) { + Runfiles.Builder builder = new Runfiles.Builder(); + Function<TransitiveInfoCollection, Runfiles> runfilesMapping = + CppRunfilesProvider.runfilesFunction(linkStaticness != LinkStaticness.DYNAMIC); + boolean linkshared = isLinkShared(context); + builder.addTransitiveArtifacts(filesToBuild); + // Add the shared libraries to the runfiles. This adds any shared libraries that are in the + // srcs of this target. + builder.addArtifacts(linkingOutputs.getLibrariesForRunfiles(true)); + builder.addRunfiles(context, RunfilesProvider.DEFAULT_RUNFILES); + builder.add(context, runfilesMapping); + CcToolchainProvider toolchain = CppHelper.getToolchain(context); + // Add the C++ runtime libraries if linking them dynamically. + if (linkStaticness == LinkStaticness.DYNAMIC) { + builder.addTransitiveArtifacts(toolchain.getDynamicRuntimeLinkInputs()); + } + // For cc_binary and cc_test rules, there is an implicit dependency on + // the malloc library package, which is specified by the "malloc" attribute. + // As the BUILD encyclopedia says, the "malloc" attribute should be ignored + // if linkshared=1. + if (!linkshared) { + TransitiveInfoCollection malloc = CppHelper.mallocForTarget(context); + builder.addTarget(malloc, RunfilesProvider.DEFAULT_RUNFILES); + builder.addTarget(malloc, runfilesMapping); + } + + if (fake) { + // Add the object files, libraries, and linker scripts that are used to + // link this executable. + builder.addSymlinksToArtifacts(Iterables.filter(fakeLinkerInputs, Artifact.MIDDLEMAN_FILTER)); + // The crosstool inputs for the link action are not sufficient; we also need the crosstool + // inputs for compilation. Node that these cannot be middlemen because Runfiles does not + // know how to expand them. + builder.addTransitiveArtifacts(toolchain.getCrosstool()); + builder.addTransitiveArtifacts(toolchain.getLibcLink()); + // Add the sources files that are used to compile the object files. + // We add the headers in the transitive closure and our own sources in the srcs + // attribute. We do not provide the auxiliary inputs, because they are only used when we + // do FDO compilation, and cc_fake_binary does not support FDO. + builder.addSymlinksToArtifacts( + Iterables.transform(common.getCAndCppSources(), Pair.<Artifact, Label>firstFunction())); + builder.addSymlinksToArtifacts(cppCompilationContext.getDeclaredIncludeSrcs()); + } + return builder.build(); + } + + @Override + public ConfiguredTarget create(RuleContext context) { + return CcBinary.init(semantics, context, /*fake =*/ false, /*useTestOnlyFlags =*/ false); + } + + public static ConfiguredTarget init(CppSemantics semantics, RuleContext ruleContext, boolean fake, + boolean useTestOnlyFlags) { + ruleContext.checkSrcsSamePackage(true); + CcCommon common = new CcCommon(ruleContext); + CppConfiguration cppConfiguration = ruleContext.getFragment(CppConfiguration.class); + + LinkTargetType linkType = + isLinkShared(ruleContext) ? LinkTargetType.DYNAMIC_LIBRARY : LinkTargetType.EXECUTABLE; + + CcLibraryHelper helper = new CcLibraryHelper(ruleContext, semantics) + .setLinkType(linkType) + .setHeadersCheckingMode(common.determineHeadersCheckingMode()) + .addCopts(common.getCopts()) + .setNoCopts(common.getNoCopts()) + .addLinkopts(common.getLinkopts()) + .addDefines(common.getDefines()) + .addCompilationPrerequisites(common.getSharedLibrariesFromSrcs()) + .addCompilationPrerequisites(common.getStaticLibrariesFromSrcs()) + .addSources(common.getCAndCppSources()) + .addPrivateHeaders(FileType.filter( + ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list(), + CppFileTypes.CPP_HEADER)) + .addObjectFiles(common.getObjectFilesFromSrcs(false)) + .addPicObjectFiles(common.getObjectFilesFromSrcs(true)) + .addPicIndependentObjectFiles(common.getLinkerScripts()) + .addDeps(ruleContext.getPrerequisites("deps", Mode.TARGET)) + .addDeps(ImmutableList.of(CppHelper.mallocForTarget(ruleContext))) + .setEnableLayeringCheck(ruleContext.getFeatures().contains(CppRuleClasses.LAYERING_CHECK)) + .addSystemIncludeDirs(common.getSystemIncludeDirs()) + .addIncludeDirs(common.getIncludeDirs()) + .addLooseIncludeDirs(common.getLooseIncludeDirs()) + .setFake(fake); + + CcLibraryHelper.Info info = helper.build(); + CppCompilationContext cppCompilationContext = info.getCppCompilationContext(); + CcCompilationOutputs ccCompilationOutputs = info.getCcCompilationOutputs(); + + // if cc_binary includes "linkshared=1", then gcc will be invoked with + // linkopt "-shared", which causes the result of linking to be a shared + // library. In this case, the name of the executable target should end + // in ".so". + PathFragment executableName = Util.getWorkspaceRelativePath( + ruleContext.getTarget(), "", OsUtils.executableExtension()); + CppLinkAction.Builder linkActionBuilder = determineLinkerArguments( + ruleContext, common, cppConfiguration, ccCompilationOutputs, + cppCompilationContext.getCompilationPrerequisites(), fake, executableName); + linkActionBuilder.setUseTestOnlyFlags(useTestOnlyFlags); + linkActionBuilder.addNonLibraryInputs(ccCompilationOutputs.getHeaderTokenFiles()); + + CcToolchainProvider ccToolchain = CppHelper.getToolchain(ruleContext); + LinkStaticness linkStaticness = getLinkStaticness(ruleContext, common, cppConfiguration); + if (linkStaticness == LinkStaticness.DYNAMIC) { + linkActionBuilder.setRuntimeInputs( + ccToolchain.getDynamicRuntimeLinkMiddleman(), + ccToolchain.getDynamicRuntimeLinkInputs()); + } else { + linkActionBuilder.setRuntimeInputs( + ccToolchain.getStaticRuntimeLinkMiddleman(), + ccToolchain.getStaticRuntimeLinkInputs()); + // Only force a static link of libgcc if static runtime linking is enabled (which + // can't be true if runtimeInputs is empty). + // TODO(bazel-team): Move this to CcToolchain. + if (!ccToolchain.getStaticRuntimeLinkInputs().isEmpty()) { + linkActionBuilder.addLinkopt("-static-libgcc"); + } + } + + linkActionBuilder.setLinkType(linkType); + linkActionBuilder.setLinkStaticness(linkStaticness); + linkActionBuilder.setFake(fake); + + // store immutable context now, recreate builder later + CppLinkAction.Context linkContext = new CppLinkAction.Context(linkActionBuilder); + + CppLinkAction linkAction = linkActionBuilder.build(); + ruleContext.registerAction(linkAction); + LibraryToLink outputLibrary = linkAction.getOutputLibrary(); + Iterable<Artifact> fakeLinkerInputs = + fake ? linkAction.getInputs() : ImmutableList.<Artifact>of(); + Artifact executable = outputLibrary.getArtifact(); + CcLinkingOutputs.Builder linkingOutputsBuilder = new CcLinkingOutputs.Builder(); + if (isLinkShared(ruleContext)) { + if (CppFileTypes.SHARED_LIBRARY.matches(executableName)) { + linkingOutputsBuilder.addDynamicLibrary(outputLibrary); + linkingOutputsBuilder.addExecutionDynamicLibrary(outputLibrary); + } else { + ruleContext.attributeError("linkshared", "'linkshared' used in non-shared library"); + } + } + // Also add all shared libraries from srcs. + for (Artifact library : common.getSharedLibrariesFromSrcs()) { + LibraryToLink symlink = common.getDynamicLibrarySymlink(library, true); + linkingOutputsBuilder.addDynamicLibrary(symlink); + linkingOutputsBuilder.addExecutionDynamicLibrary(symlink); + } + CcLinkingOutputs linkingOutputs = linkingOutputsBuilder.build(); + NestedSet<Artifact> filesToBuild = NestedSetBuilder.create(Order.STABLE_ORDER, executable); + + // Create the stripped binary, but don't add it to filesToBuild; it's only built when requested. + Artifact strippedFile = ruleContext.getImplicitOutputArtifact( + CppRuleClasses.CC_BINARY_STRIPPED); + createStripAction(ruleContext, cppConfiguration, executable, strippedFile); + + DwoArtifactsCollector dwoArtifacts = + collectTransitiveDwoArtifacts(ruleContext, common, cppConfiguration, ccCompilationOutputs); + Artifact dwpFile = + ruleContext.getImplicitOutputArtifact(CppRuleClasses.CC_BINARY_DEBUG_PACKAGE); + createDebugPackagerActions(ruleContext, cppConfiguration, dwpFile, dwoArtifacts); + + // The debug package should include the dwp file only if it was explicitly requested. + Artifact explicitDwpFile = dwpFile; + if (!cppConfiguration.useFission()) { + explicitDwpFile = null; + } + + // TODO(bazel-team): Do we need to put original shared libraries (along with + // mangled symlinks) into the RunfilesSupport object? It does not seem + // logical since all symlinked libraries will be linked anyway and would + // not require manual loading but if we do, then we would need to collect + // their names and use a different constructor below. + Runfiles runfiles = collectRunfiles(ruleContext, common, linkingOutputs, + cppCompilationContext, linkStaticness, filesToBuild, fakeLinkerInputs, fake); + RunfilesSupport runfilesSupport = RunfilesSupport.withExecutable( + ruleContext, runfiles, executable, ruleContext.getConfiguration().buildRunfiles()); + + TransitiveLipoInfoProvider transitiveLipoInfo; + if (cppConfiguration.isLipoContextCollector()) { + transitiveLipoInfo = common.collectTransitiveLipoLabels(ccCompilationOutputs); + } else { + transitiveLipoInfo = TransitiveLipoInfoProvider.EMPTY; + } + + RuleConfiguredTargetBuilder ruleBuilder = new RuleConfiguredTargetBuilder(ruleContext); + common.addTransitiveInfoProviders( + ruleBuilder, filesToBuild, ccCompilationOutputs, cppCompilationContext, linkingOutputs, + dwoArtifacts, transitiveLipoInfo); + + Map<Artifact, IncludeScannable> scannableMap = new LinkedHashMap<>(); + if (cppConfiguration.isLipoContextCollector()) { + for (IncludeScannable scannable : transitiveLipoInfo.getTransitiveIncludeScannables()) { + // These should all be CppCompileActions, which should have only one source file. + // This is also checked when they are put into the nested set. + Artifact source = + Iterables.getOnlyElement(scannable.getIncludeScannerSources()); + scannableMap.put(source, scannable); + } + } + + return ruleBuilder + .add(RunfilesProvider.class, RunfilesProvider.simple(runfiles)) + .add( + CppDebugPackageProvider.class, + new CppDebugPackageProvider(strippedFile, executable, explicitDwpFile)) + .setRunfilesSupport(runfilesSupport, executable) + .setBaselineCoverageArtifacts(createBaselineCoverageArtifacts( + ruleContext, common, ccCompilationOutputs, fake)) + .addProvider(LipoContextProvider.class, new LipoContextProvider( + cppCompilationContext, ImmutableMap.copyOf(scannableMap))) + .addProvider(CppLinkAction.Context.class, linkContext) + .build(); + } + + /** + * Creates an action to strip an executable. + */ + private static void createStripAction(RuleContext context, + CppConfiguration cppConfiguration, Artifact input, Artifact output) { + context.registerAction(new SpawnAction.Builder() + .addInput(input) + .addTransitiveInputs(CppHelper.getToolchain(context).getStrip()) + .addOutput(output) + .useDefaultShellEnvironment() + .setExecutable(cppConfiguration.getStripExecutable()) + .addArguments("-S", "-p", "-o", output.getExecPathString()) + .addArguments("-R", ".gnu.switches.text.quote_paths") + .addArguments("-R", ".gnu.switches.text.bracket_paths") + .addArguments("-R", ".gnu.switches.text.system_paths") + .addArguments("-R", ".gnu.switches.text.cpp_defines") + .addArguments("-R", ".gnu.switches.text.cpp_includes") + .addArguments("-R", ".gnu.switches.text.cl_args") + .addArguments("-R", ".gnu.switches.text.lipo_info") + .addArguments("-R", ".gnu.switches.text.annotation") + .addArguments(cppConfiguration.getStripOpts()) + .addArgument(input.getExecPathString()) + .setProgressMessage("Stripping " + output.prettyPrint() + " for " + context.getLabel()) + .setMnemonic("CcStrip") + .build(context)); + } + + /** + * Given 'temps', traverse this target and its dependencies and collect up all + * the object files, libraries, linker options, linkstamps attributes and linker scripts. + */ + private static CppLinkAction.Builder determineLinkerArguments(RuleContext context, + CcCommon common, CppConfiguration cppConfiguration, CcCompilationOutputs compilationOutputs, + ImmutableSet<Artifact> compilationPrerequisites, + boolean fake, PathFragment executableName) { + CppLinkAction.Builder builder = new CppLinkAction.Builder(context, executableName) + .setCrosstoolInputs(CppHelper.getToolchain(context).getLink()) + .addNonLibraryInputs(compilationPrerequisites); + + // Determine the object files to link in. + boolean usePic = CppHelper.usePic(context, !isLinkShared(context)) && !fake; + Iterable<Artifact> compiledObjectFiles = compilationOutputs.getObjectFiles(usePic); + + if (fake) { + builder.addFakeNonLibraryInputs(compiledObjectFiles); + } else { + builder.addNonLibraryInputs(compiledObjectFiles); + } + + builder.addNonLibraryInputs(common.getObjectFilesFromSrcs(usePic)); + builder.addNonLibraryInputs(common.getLinkerScripts()); + + // Determine the libraries to link in. + // First libraries from srcs. Shared library artifacts here are substituted with mangled symlink + // artifacts generated by getDynamicLibraryLink(). This is done to minimize number of -rpath + // entries during linking process. + for (Artifact library : common.getLibrariesFromSrcs()) { + if (SHARED_LIBRARY_FILETYPES.matches(library.getFilename())) { + builder.addLibrary(common.getDynamicLibrarySymlink(library, true)); + } else { + builder.addLibrary(LinkerInputs.opaqueLibraryToLink(library)); + } + } + + // Then libraries from the closure of deps. + List<String> linkopts = new ArrayList<>(); + Map<Artifact, ImmutableList<Artifact>> linkstamps = new LinkedHashMap<>(); + + NestedSet<LibraryToLink> librariesInDepsClosure = + findLibrariesToLinkInDepsClosure(context, common, cppConfiguration, linkopts, linkstamps); + builder.addLinkopts(linkopts); + builder.addLinkstamps(linkstamps); + + builder.addLibraries(librariesInDepsClosure); + return builder; + } + + /** + * Explore the transitive closure of our deps to collect linking information. + */ + private static NestedSet<LibraryToLink> findLibrariesToLinkInDepsClosure( + RuleContext context, + CcCommon common, + CppConfiguration cppConfiguration, + List<String> linkopts, + Map<Artifact, + ImmutableList<Artifact>> linkstamps) { + // This is true for both FULLY STATIC and MOSTLY STATIC linking. + boolean linkingStatically = + getLinkStaticness(context, common, cppConfiguration) != LinkStaticness.DYNAMIC; + + CcLinkParams linkParams = collectCcLinkParams( + context, common, linkingStatically, isLinkShared(context)); + linkopts.addAll(linkParams.flattenedLinkopts()); + linkstamps.putAll(CppHelper.resolveLinkstamps(context, linkParams)); + return linkParams.getLibraries(); + } + + /** + * Gets the linkopts to use for this binary. These options are NOT used when + * linking other binaries that depend on this binary. + * + * @return a new List instance that contains the linkopts for this binary + * target. + */ + private static ImmutableList<String> getBinaryLinkopts(RuleContext context, + CcCommon common) { + List<String> linkopts = new ArrayList<>(); + if (isLinkShared(context)) { + linkopts.add("-shared"); + } + linkopts.addAll(common.getLinkopts()); + return ImmutableList.copyOf(linkopts); + } + + private static boolean linkstaticAttribute(RuleContext context) { + return context.attributes().get("linkstatic", Type.BOOLEAN); + } + + /** + * Returns "true" if the {@code linkshared} attribute exists and is set. + */ + private static final boolean isLinkShared(RuleContext context) { + return context.getRule().getRuleClassObject().hasAttr("linkshared", Type.BOOLEAN) + && context.attributes().get("linkshared", Type.BOOLEAN); + } + + private static final boolean dashStaticInLinkopts(CcCommon common, + CppConfiguration cppConfiguration) { + return common.getLinkopts().contains("-static") + || cppConfiguration.getLinkOptions().contains("-static"); + } + + private static final LinkStaticness getLinkStaticness(RuleContext context, + CcCommon common, CppConfiguration cppConfiguration) { + if (cppConfiguration.getDynamicMode() == DynamicMode.FULLY) { + return LinkStaticness.DYNAMIC; + } else if (dashStaticInLinkopts(common, cppConfiguration)) { + return LinkStaticness.FULLY_STATIC; + } else if (cppConfiguration.getDynamicMode() == DynamicMode.OFF + || linkstaticAttribute(context)) { + return LinkStaticness.MOSTLY_STATIC; + } else { + return LinkStaticness.DYNAMIC; + } + } + + /** + * Collects .dwo artifacts either transitively or directly, depending on the link type. + * + * <p>For a cc_binary, we only include the .dwo files corresponding to the .o files that are + * passed into the link. For static linking, this includes all transitive dependencies. But + * for dynamic linking, dependencies are separately linked into their own shared libraries, + * so we don't need them here. + */ + private static DwoArtifactsCollector collectTransitiveDwoArtifacts(RuleContext context, + CcCommon common, CppConfiguration cppConfiguration, CcCompilationOutputs compilationOutputs) { + if (getLinkStaticness(context, common, cppConfiguration) == LinkStaticness.DYNAMIC) { + return DwoArtifactsCollector.directCollector(compilationOutputs); + } else { + return CcCommon.collectTransitiveDwoArtifacts(context, compilationOutputs); + } + } + + @VisibleForTesting + public static Iterable<Artifact> getDwpInputs( + RuleContext context, NestedSet<Artifact> picDwoArtifacts, NestedSet<Artifact> dwoArtifacts) { + return CppHelper.usePic(context, !isLinkShared(context)) ? picDwoArtifacts : dwoArtifacts; + } + + /** + * Creates the actions needed to generate this target's "debug info package" + * (i.e. its .dwp file). + */ + private static void createDebugPackagerActions(RuleContext context, + CppConfiguration cppConfiguration, Artifact dwpOutput, + DwoArtifactsCollector dwoArtifactsCollector) { + Iterable<Artifact> allInputs = getDwpInputs(context, + dwoArtifactsCollector.getPicDwoArtifacts(), + dwoArtifactsCollector.getDwoArtifacts()); + + // No inputs? Just generate a trivially empty .dwp. + // + // Note this condition automatically triggers for any build where fission is disabled. + // Because rules referencing .dwp targets may be invoked with or without fission, we need + // to support .dwp generation even when fission is disabled. Since no actual functionality + // is expected then, an empty file is appropriate. + if (Iterables.isEmpty(allInputs)) { + context.registerAction( + new FileWriteAction(context.getActionOwner(), dwpOutput, "", false)); + return; + } + + // Get the tool inputs necessary to run the dwp command. + NestedSet<Artifact> dwpTools = CppHelper.getToolchain(context).getDwp(); + Preconditions.checkState(!dwpTools.isEmpty()); + + // We apply a hierarchical action structure to limit the maximum number of inputs to any + // single action. + // + // While the dwp tools consumes .dwo files, it can also consume intermediate .dwp files, + // allowing us to split a large input set into smaller batches of arbitrary size and order. + // Aside from the parallelism performance benefits this offers, this also reduces input + // size requirements: if a.dwo, b.dwo, c.dwo, and e.dwo are each 1 KB files, we can apply + // two intermediate actions DWP(a.dwo, b.dwo) --> i1.dwp and DWP(c.dwo, e.dwo) --> i2.dwp. + // When we then apply the final action DWP(i1.dwp, i2.dwp) --> finalOutput.dwp, the inputs + // to this action will usually total far less than 4 KB. + // + // This list tracks every action we'll need to generate the output .dwp with batching. + List<SpawnAction.Builder> packagers = new ArrayList<>(); + + // Step 1: generate our batches. We currently break into arbitrary batches of fixed maximum + // input counts, but we can always apply more intelligent heuristics if the need arises. + SpawnAction.Builder currentPackager = newDwpAction(cppConfiguration, dwpTools); + int inputsForCurrentPackager = 0; + + for (Artifact dwoInput : allInputs) { + if (inputsForCurrentPackager == MAX_INPUTS_PER_DWP_ACTION) { + packagers.add(currentPackager); + currentPackager = newDwpAction(cppConfiguration, dwpTools); + inputsForCurrentPackager = 0; + } + currentPackager.addInputArgument(dwoInput); + inputsForCurrentPackager++; + } + packagers.add(currentPackager); + + // Step 2: given the batches, create the actions. + if (packagers.size() == 1) { + // If we only have one batch, make a single "original inputs --> final output" action. + context.registerAction(Iterables.getOnlyElement(packagers) + .addArgument("-o") + .addOutputArgument(dwpOutput) + .setMnemonic("CcGenerateDwp") + .build(context)); + } else { + // If we have multiple batches, make them all intermediate actions, then pipe their outputs + // into an additional action that outputs the final artifact. + // + // Note this only creates a hierarchy one level deep (i.e. we don't check if the number of + // intermediate outputs exceeds the maximum batch size). This is okay for current needs, + // which shouldn't stress those limits. + List<Artifact> intermediateOutputs = new ArrayList<>(); + + int count = 1; + for (SpawnAction.Builder packager : packagers) { + Artifact intermediateOutput = + getIntermediateDwpFile(context.getAnalysisEnvironment(), dwpOutput, count++); + context.registerAction(packager + .addArgument("-o") + .addOutputArgument(intermediateOutput) + .setMnemonic("CcGenerateIntermediateDwp") + .build(context)); + intermediateOutputs.add(intermediateOutput); + } + + // Now create the final action. + context.registerAction(newDwpAction(cppConfiguration, dwpTools) + .addInputArguments(intermediateOutputs) + .addArgument("-o") + .addOutputArgument(dwpOutput) + .setMnemonic("CcGenerateDwp") + .build(context)); + } + } + + /** + * Returns a new SpawnAction builder for generating dwp files, pre-initialized with + * standard settings. + */ + private static SpawnAction.Builder newDwpAction(CppConfiguration cppConfiguration, + NestedSet<Artifact> dwpTools) { + return new SpawnAction.Builder() + .addTransitiveInputs(dwpTools) + .setExecutable(cppConfiguration.getDwpExecutable()) + .useParameterFile(ParameterFile.ParameterFileType.UNQUOTED); + } + + /** + * Creates an intermediate dwp file keyed off the name and path of the final output. + */ + private static Artifact getIntermediateDwpFile(AnalysisEnvironment env, Artifact dwpOutput, + int orderNumber) { + PathFragment outputPath = dwpOutput.getRootRelativePath(); + PathFragment intermediatePath = + FileSystemUtils.appendWithoutExtension(outputPath, "-" + orderNumber); + return env.getDerivedArtifact( + outputPath.getParentDirectory().getRelative( + INTERMEDIATE_DWP_DIR + "/" + intermediatePath.getPathString()), + dwpOutput.getRoot()); + } + + /** + * Collect link parameters from the transitive closure. + */ + private static CcLinkParams collectCcLinkParams(RuleContext context, CcCommon common, + boolean linkingStatically, boolean linkShared) { + CcLinkParams.Builder builder = CcLinkParams.builder(linkingStatically, linkShared); + + if (isLinkShared(context)) { + // CcLinkingOutputs is empty because this target is not configured yet + builder.addCcLibrary(context, common, false, CcLinkingOutputs.EMPTY); + } else { + builder.addTransitiveTargets( + context.getPrerequisites("deps", Mode.TARGET), + CcLinkParamsProvider.TO_LINK_PARAMS, CcSpecificLinkParamsProvider.TO_LINK_PARAMS); + builder.addTransitiveTarget(CppHelper.mallocForTarget(context)); + builder.addLinkOpts(getBinaryLinkopts(context, common)); + } + return builder.build(); + } + + private static ImmutableList<Artifact> createBaselineCoverageArtifacts( + RuleContext context, CcCommon common, CcCompilationOutputs compilationOutputs, + boolean fake) { + if (!TargetUtils.isTestRule(context.getRule()) && !fake) { + Iterable<Artifact> objectFiles = compilationOutputs.getObjectFiles( + CppHelper.usePic(context, !isLinkShared(context))); + return BaselineCoverageAction.getBaselineCoverageArtifacts(context, + common.getInstrumentedFilesProvider(objectFiles).getInstrumentedFiles()); + } else { + return ImmutableList.of(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java new file mode 100644 index 0000000..3f5ff76 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java
@@ -0,0 +1,678 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.CompilationPrerequisitesProvider; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToCompileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TempsProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.DynamicMode; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.HeadersCheckingMode; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.LocalMetadataCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProviderImpl; +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * Common parts of the implementation of cc rules. + */ +public final class CcCommon { + + private static final String NO_COPTS_ATTRIBUTE = "nocopts"; + + private static final FileTypeSet SOURCE_TYPES = FileTypeSet.of( + CppFileTypes.CPP_SOURCE, + CppFileTypes.CPP_HEADER, + CppFileTypes.C_SOURCE, + CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR); + + /** + * Collects all metadata files generated by C++ compilation actions that output the .o files + * on the input. + */ + private static final LocalMetadataCollector CC_METADATA_COLLECTOR = + new LocalMetadataCollector() { + @Override + public void collectMetadataArtifacts(Iterable<Artifact> objectFiles, + AnalysisEnvironment analysisEnvironment, NestedSetBuilder<Artifact> metadataFilesBuilder) { + for (Artifact artifact : objectFiles) { + Action action = analysisEnvironment.getLocalGeneratingAction(artifact); + if (action instanceof CppCompileAction) { + addOutputs(metadataFilesBuilder, action, CppFileTypes.COVERAGE_NOTES); + } + } + } + }; + + /** C++ configuration */ + private final CppConfiguration cppConfiguration; + + /** The Artifacts from srcs. */ + private final ImmutableList<Artifact> sources; + + private final ImmutableList<Pair<Artifact, Label>> cAndCppSources; + + /** Expanded and tokenized copts attribute. Set by initCopts(). */ + private final ImmutableList<String> copts; + + /** + * The expanded linkopts for this rule. + */ + private final ImmutableList<String> linkopts; + + private final RuleContext ruleContext; + + public CcCommon(RuleContext ruleContext) { + this.ruleContext = ruleContext; + this.cppConfiguration = ruleContext.getFragment(CppConfiguration.class); + this.sources = hasAttribute("srcs", Type.LABEL_LIST) + ? ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list() + : ImmutableList.<Artifact>of(); + + this.cAndCppSources = collectCAndCppSources(); + copts = initCopts(); + linkopts = initLinkopts(); + } + + ImmutableList<Artifact> getTemps(CcCompilationOutputs compilationOutputs) { + return cppConfiguration.isLipoContextCollector() + ? ImmutableList.<Artifact>of() + : compilationOutputs.getTemps(); + } + + /** + * Returns our own linkopts from the rule attribute. This determines linker + * options to use when building this target and anything that depends on it. + */ + public ImmutableList<String> getLinkopts() { + return linkopts; + } + + public ImmutableList<String> getCopts() { + return copts; + } + + private boolean hasAttribute(String name, Type<?> type) { + return ruleContext.getRule().getRuleClassObject().hasAttr(name, type); + } + + private static NestedSet<Artifact> collectExecutionDynamicLibraryArtifacts( + RuleContext ruleContext, + List<LibraryToLink> executionDynamicLibraries) { + Iterable<Artifact> artifacts = LinkerInputs.toLibraryArtifacts(executionDynamicLibraries); + if (!Iterables.isEmpty(artifacts)) { + return NestedSetBuilder.wrap(Order.STABLE_ORDER, artifacts); + } + + Iterable<CcExecutionDynamicLibrariesProvider> deps = ruleContext + .getPrerequisites("deps", Mode.TARGET, CcExecutionDynamicLibrariesProvider.class); + + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (CcExecutionDynamicLibrariesProvider dep : deps) { + builder.addTransitive(dep.getExecutionDynamicLibraryArtifacts()); + } + return builder.build(); + } + + /** + * Collects all .dwo artifacts in this target's transitive closure. + */ + public static DwoArtifactsCollector collectTransitiveDwoArtifacts( + RuleContext ruleContext, + CcCompilationOutputs compilationOutputs) { + ImmutableList.Builder<TransitiveInfoCollection> deps = + ImmutableList.<TransitiveInfoCollection>builder(); + + deps.addAll(ruleContext.getPrerequisites("deps", Mode.TARGET)); + + if (ruleContext.getRule().getRuleClassObject().hasAttr("malloc", Type.LABEL)) { + deps.add(CppHelper.mallocForTarget(ruleContext)); + } + if (ruleContext.getRule().getRuleClassObject().hasAttr("implementation", Type.LABEL_LIST)) { + deps.addAll(ruleContext.getPrerequisites("implementation", Mode.TARGET)); + } + + return compilationOutputs == null // Possible in LIPO collection mode (see initializationHook). + ? DwoArtifactsCollector.emptyCollector() + : DwoArtifactsCollector.transitiveCollector(compilationOutputs, deps.build()); + } + + public TransitiveLipoInfoProvider collectTransitiveLipoLabels(CcCompilationOutputs outputs) { + if (cppConfiguration.getFdoSupport().getFdoRoot() == null + || !cppConfiguration.isLipoContextCollector()) { + return TransitiveLipoInfoProvider.EMPTY; + } + + NestedSetBuilder<IncludeScannable> scannableBuilder = NestedSetBuilder.stableOrder(); + CppHelper.addTransitiveLipoInfoForCommonAttributes(ruleContext, outputs, scannableBuilder); + if (hasAttribute("implementation", Type.LABEL_LIST)) { + for (TransitiveLipoInfoProvider impl : AnalysisUtils.getProviders( + ruleContext.getPrerequisites("implementation", Mode.TARGET), + TransitiveLipoInfoProvider.class)) { + scannableBuilder.addTransitive(impl.getTransitiveIncludeScannables()); + } + } + + return new TransitiveLipoInfoProvider(scannableBuilder.build()); + } + + private NestedSet<LinkerInput> collectTransitiveCcNativeLibraries( + RuleContext ruleContext, + List<? extends LinkerInput> dynamicLibraries) { + NestedSetBuilder<LinkerInput> builder = NestedSetBuilder.linkOrder(); + builder.addAll(dynamicLibraries); + for (CcNativeLibraryProvider dep : + ruleContext.getPrerequisites("deps", Mode.TARGET, CcNativeLibraryProvider.class)) { + builder.addTransitive(dep.getTransitiveCcNativeLibraries()); + } + return builder.build(); + } + + /** + * Returns a list of ({@link Artifact}, {@link Label}) pairs. Each pair represents an input + * source file and the label of the rule that generates it (or the label of the source file + * itself if it is an input file) + */ + ImmutableList<Pair<Artifact, Label>> getCAndCppSources() { + return cAndCppSources; + } + + private boolean shouldProcessHeaders() { + boolean crosstoolSupportsHeaderParsing = + CppHelper.getToolchain(ruleContext).supportsHeaderParsing(); + return crosstoolSupportsHeaderParsing && ( + ruleContext.getFeatures().contains(CppRuleClasses.PREPROCESS_HEADERS) + || ruleContext.getFeatures().contains(CppRuleClasses.PARSE_HEADERS)); + } + + private ImmutableList<Pair<Artifact, Label>> collectCAndCppSources() { + Map<Artifact, Label> map = Maps.newLinkedHashMap(); + if (!hasAttribute("srcs", Type.LABEL_LIST)) { + return ImmutableList.<Pair<Artifact, Label>>of(); + } + Iterable<FileProvider> providers = + ruleContext.getPrerequisites("srcs", Mode.TARGET, FileProvider.class); + // TODO(bazel-team): Move header processing logic down in the stack (to CcLibraryHelper or + // such). + boolean processHeaders = shouldProcessHeaders(); + if (processHeaders && hasAttribute("hdrs", Type.LABEL_LIST)) { + providers = Iterables.concat(providers, + ruleContext.getPrerequisites("hdrs", Mode.TARGET, FileProvider.class)); + } + for (FileProvider provider : providers) { + for (Artifact artifact : FileType.filter(provider.getFilesToBuild(), SOURCE_TYPES)) { + boolean isHeader = CppFileTypes.CPP_HEADER.matches(artifact.getExecPath()); + if ((isHeader && !processHeaders) + || CppFileTypes.CPP_TEXTUAL_INCLUDE.matches(artifact.getExecPath())) { + continue; + } + Label oldLabel = map.put(artifact, provider.getLabel()); + // TODO(bazel-team): We currently do not warn for duplicate headers with + // different labels, as that would require cleaning up the code base + // without significant benefit; we should eventually make this + // consistent one way or the other. + if (!isHeader && oldLabel != null && !oldLabel.equals(provider.getLabel())) { + ruleContext.attributeError("srcs", String.format( + "Artifact '%s' is duplicated (through '%s' and '%s')", + artifact.getExecPathString(), oldLabel, provider.getLabel())); + } + } + } + + ImmutableList.Builder<Pair<Artifact, Label>> result = ImmutableList.builder(); + for (Map.Entry<Artifact, Label> entry : map.entrySet()) { + result.add(Pair.of(entry.getKey(), entry.getValue())); + } + + return result.build(); + } + + Iterable<Artifact> getLibrariesFromSrcs() { + return FileType.filter(sources, CppFileTypes.ARCHIVE, CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_LIBRARY, CppFileTypes.ALWAYS_LINK_PIC_LIBRARY, + CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY); + } + + Iterable<Artifact> getSharedLibrariesFromSrcs() { + return getSharedLibrariesFrom(sources); + } + + static Iterable<Artifact> getSharedLibrariesFrom(Iterable<Artifact> collection) { + return FileType.filter(collection, CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY); + } + + Iterable<Artifact> getStaticLibrariesFromSrcs() { + return FileType.filter(sources, CppFileTypes.ARCHIVE, CppFileTypes.ALWAYS_LINK_LIBRARY); + } + + Iterable<LibraryToLink> getPicStaticLibrariesFromSrcs() { + return LinkerInputs.opaqueLibrariesToLink( + FileType.filter(sources, CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_PIC_LIBRARY)); + } + + Iterable<Artifact> getObjectFilesFromSrcs(final boolean usePic) { + if (usePic) { + return Iterables.filter(sources, new Predicate<Artifact>() { + @Override + public boolean apply(Artifact artifact) { + String filename = artifact.getExecPathString(); + + // For compatibility with existing BUILD files, any ".o" files listed + // in srcs are assumed to be position-independent code, or + // at least suitable for inclusion in shared libraries, unless they + // end with ".nopic.o". (The ".nopic.o" extension is an undocumented + // feature to give users at least some control over this.) Note that + // some target platforms do not require shared library code to be PIC. + return CppFileTypes.PIC_OBJECT_FILE.matches(filename) || + (CppFileTypes.OBJECT_FILE.matches(filename) && !filename.endsWith(".nopic.o")); + } + }); + } else { + return FileType.filter(sources, CppFileTypes.OBJECT_FILE); + } + } + + /** + * Returns the files from headers and does some sanity checks. Note that this method reports + * warnings to the {@link RuleContext} as a side effect, and so should only be called once for any + * given rule. + */ + public static List<Artifact> getHeaders(RuleContext ruleContext) { + List<Artifact> hdrs = new ArrayList<>(); + for (TransitiveInfoCollection target : + ruleContext.getPrerequisitesIf("hdrs", Mode.TARGET, FileProvider.class)) { + FileProvider provider = target.getProvider(FileProvider.class); + for (Artifact artifact : provider.getFilesToBuild()) { + if (!CppRuleClasses.DISALLOWED_HDRS_FILES.matches(artifact.getFilename())) { + hdrs.add(artifact); + } else { + ruleContext.attributeWarning("hdrs", "file '" + artifact.getFilename() + + "' from target '" + target.getLabel() + "' is not allowed in hdrs"); + } + } + } + return hdrs; + } + + /** + * Uses {@link #getHeaders(RuleContext)} to get the {@code hdrs} on this target. This method will + * return an empty list if there is no {@code hdrs} attribute on this rule type. + */ + List<Artifact> getHeaders() { + if (!hasAttribute("hdrs", Type.LABEL_LIST)) { + return ImmutableList.of(); + } + return getHeaders(ruleContext); + } + + HeadersCheckingMode determineHeadersCheckingMode() { + HeadersCheckingMode headersCheckingMode = cppConfiguration.getHeadersCheckingMode(); + + // Package default overrides command line option. + if (ruleContext.getRule().getPackage().isDefaultHdrsCheckSet()) { + String value = + ruleContext.getRule().getPackage().getDefaultHdrsCheck().toUpperCase(Locale.ENGLISH); + headersCheckingMode = HeadersCheckingMode.valueOf(value); + } + + // 'hdrs_check' attribute overrides package default. + if (hasAttribute("hdrs_check", Type.STRING) + && ruleContext.getRule().isAttributeValueExplicitlySpecified("hdrs_check")) { + try { + String value = ruleContext.attributes().get("hdrs_check", Type.STRING) + .toUpperCase(Locale.ENGLISH); + headersCheckingMode = HeadersCheckingMode.valueOf(value); + } catch (IllegalArgumentException e) { + ruleContext.attributeError("hdrs_check", "must be one of: 'loose', 'warn' or 'strict'"); + } + } + + return headersCheckingMode; + } + + /** + * Expand and tokenize the copts and nocopts attributes. + */ + private ImmutableList<String> initCopts() { + if (!hasAttribute("copts", Type.STRING_LIST)) { + return ImmutableList.<String>of(); + } + // TODO(bazel-team): getAttributeCopts should not tokenize the strings. + // Make a warning for now. + List<String> tokens = new ArrayList<>(); + for (String str : ruleContext.attributes().get("copts", Type.STRING_LIST)) { + tokens.clear(); + try { + ShellUtils.tokenize(tokens, str); + if (tokens.size() > 1) { + ruleContext.attributeWarning("copts", + "each item in the list should contain only one option"); + } + } catch (ShellUtils.TokenizationException e) { + // ignore, the error is reported in the getAttributeCopts call + } + } + + Pattern nocopts = getNoCopts(ruleContext); + if (nocopts != null && nocopts.matcher("-Wno-future-warnings").matches()) { + ruleContext.attributeWarning("nocopts", + "Regular expression '" + nocopts.pattern() + "' is too general; for example, it matches " + + "'-Wno-future-warnings'. Thus it might *re-enable* compiler warnings we wish to " + + "disable globally. To disable all compiler warnings, add '-w' to copts instead"); + } + + return ImmutableList.<String>builder() + .addAll(getPackageCopts(ruleContext)) + .addAll(CppHelper.getAttributeCopts(ruleContext, "copts")) + .build(); + } + + private static ImmutableList<String> getPackageCopts(RuleContext ruleContext) { + List<String> unexpanded = ruleContext.getRule().getPackage().getDefaultCopts(); + return ImmutableList.copyOf(CppHelper.expandMakeVariables(ruleContext, "copts", unexpanded)); + } + + Pattern getNoCopts() { + return getNoCopts(ruleContext); + } + + /** + * Returns nocopts pattern built from the make variable expanded nocopts + * attribute. + */ + private static Pattern getNoCopts(RuleContext ruleContext) { + Pattern nocopts = null; + if (ruleContext.getRule().isAttrDefined(NO_COPTS_ATTRIBUTE, Type.STRING)) { + String nocoptsAttr = ruleContext.expandMakeVariables(NO_COPTS_ATTRIBUTE, + ruleContext.attributes().get(NO_COPTS_ATTRIBUTE, Type.STRING)); + try { + nocopts = Pattern.compile(nocoptsAttr); + } catch (PatternSyntaxException e) { + ruleContext.attributeError(NO_COPTS_ATTRIBUTE, + "invalid regular expression '" + nocoptsAttr + "': " + e.getMessage()); + } + } + return nocopts; + } + + // TODO(bazel-team): calculating nocopts every time is not very efficient, + // fix this after the rule migration. The problem is that in some cases we call this after + // the RCT is created (so RuleContext is not accessible), in some cases during the creation. + // It would probably make more sense to use TransitiveInfoProviders. + /** + * Returns true if the rule context has a nocopts regex that matches the given value, false + * otherwise. + */ + static boolean noCoptsMatches(String option, RuleContext ruleContext) { + Pattern nocopts = getNoCopts(ruleContext); + return nocopts == null ? false : nocopts.matcher(option).matches(); + } + + private static final String DEFINES_ATTRIBUTE = "defines"; + + /** + * Returns a list of define tokens from "defines" attribute. + * + * <p>We tokenize the "defines" attribute, to ensure that the handling of + * quotes and backslash escapes is consistent Bazel's treatment of the "copts" attribute. + * + * <p>But we require that the "defines" attribute consists of a single token. + */ + public List<String> getDefines() { + List<String> defines = new ArrayList<>(); + for (String define : + ruleContext.attributes().get(DEFINES_ATTRIBUTE, Type.STRING_LIST)) { + List<String> tokens = new ArrayList<>(); + try { + ShellUtils.tokenize(tokens, ruleContext.expandMakeVariables(DEFINES_ATTRIBUTE, define)); + if (tokens.size() == 1) { + defines.add(tokens.get(0)); + } else if (tokens.isEmpty()) { + ruleContext.attributeError(DEFINES_ATTRIBUTE, "empty definition not allowed"); + } else { + ruleContext.attributeError(DEFINES_ATTRIBUTE, + "definition contains too many tokens (found " + tokens.size() + + ", expecting exactly one)"); + } + } catch (ShellUtils.TokenizationException e) { + ruleContext.attributeError(DEFINES_ATTRIBUTE, e.getMessage()); + } + } + return defines; + } + + /** + * Collects our own linkopts from the rule attribute. This determines linker + * options to use when building this library and anything that depends on it. + */ + private final ImmutableList<String> initLinkopts() { + if (!hasAttribute("linkopts", Type.STRING_LIST)) { + return ImmutableList.<String>of(); + } + List<String> ourLinkopts = ruleContext.attributes().get("linkopts", Type.STRING_LIST); + List<String> result = new ArrayList<>(); + if (ourLinkopts != null) { + boolean allowDashStatic = !cppConfiguration.forceIgnoreDashStatic() + && (cppConfiguration.getDynamicMode() != DynamicMode.FULLY); + for (String linkopt : ourLinkopts) { + if (linkopt.equals("-static") && !allowDashStatic) { + continue; + } + CppHelper.expandAttribute(ruleContext, result, "linkopts", linkopt, true); + } + } + return ImmutableList.copyOf(result); + } + + /** + * Determines a list of loose include directories that are only allowed to be referenced when + * headers checking is {@link HeadersCheckingMode#LOOSE} or {@link HeadersCheckingMode#WARN}. + */ + List<PathFragment> getLooseIncludeDirs() { + List<PathFragment> result = new ArrayList<>(); + // The package directory of the rule contributes includes. Note that this also covers all + // non-subpackage sub-directories. + PathFragment rulePackage = ruleContext.getLabel().getPackageFragment(); + result.add(rulePackage); + + // Gather up all the dirs from the rule's srcs as well as any of the srcs outputs. + if (hasAttribute("srcs", Type.LABEL_LIST)) { + for (FileProvider src : + ruleContext.getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) { + PathFragment packageDir = src.getLabel().getPackageFragment(); + for (Artifact a : src.getFilesToBuild()) { + result.add(packageDir); + // Attempt to gather subdirectories that might contain include files. + result.add(a.getRootRelativePath().getParentDirectory()); + } + } + } + + // Add in any 'includes' attribute values as relative path fragments + if (ruleContext.getRule().isAttributeValueExplicitlySpecified("includes")) { + PathFragment packageFragment = ruleContext.getLabel().getPackageFragment(); + // For now, anything with an 'includes' needs a blanket declaration + result.add(packageFragment.getRelative("**")); + } + return result; + } + + List<PathFragment> getSystemIncludeDirs() { + // Add in any 'includes' attribute values as relative path fragments + if (!ruleContext.getRule().isAttributeValueExplicitlySpecified("includes") + || !cppConfiguration.useIsystemForIncludes()) { + return ImmutableList.of(); + } + return getIncludeDirsFromIncludesAttribute(); + } + + List<PathFragment> getIncludeDirs() { + if (!ruleContext.getRule().isAttributeValueExplicitlySpecified("includes") + || cppConfiguration.useIsystemForIncludes()) { + return ImmutableList.of(); + } + return getIncludeDirsFromIncludesAttribute(); + } + + private List<PathFragment> getIncludeDirsFromIncludesAttribute() { + List<PathFragment> result = new ArrayList<>(); + PathFragment packageFragment = ruleContext.getLabel().getPackageFragment(); + for (String includesAttr : ruleContext.attributes().get("includes", Type.STRING_LIST)) { + includesAttr = ruleContext.expandMakeVariables("includes", includesAttr); + if (includesAttr.startsWith("/")) { + ruleContext.attributeWarning("includes", + "ignoring invalid absolute path '" + includesAttr + "'"); + continue; + } + PathFragment includesPath = packageFragment.getRelative(includesAttr).normalize(); + if (!includesPath.isNormalized()) { + ruleContext.attributeError("includes", + "Path references a path above the execution root."); + } + result.add(includesPath); + result.add(ruleContext.getConfiguration().getGenfilesFragment().getRelative(includesPath)); + } + return result; + } + + /** + * Collects compilation prerequisite artifacts. + */ + static CompilationPrerequisitesProvider collectCompilationPrerequisites( + RuleContext ruleContext, CppCompilationContext context) { + // TODO(bazel-team): Use context.getCompilationPrerequisites() instead. + NestedSetBuilder<Artifact> prerequisites = NestedSetBuilder.stableOrder(); + if (ruleContext.getRule().getRuleClassObject().hasAttr("srcs", Type.LABEL_LIST)) { + for (FileProvider provider : ruleContext + .getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) { + prerequisites.addAll(FileType.filter(provider.getFilesToBuild(), SOURCE_TYPES)); + } + } + prerequisites.addTransitive(context.getDeclaredIncludeSrcs()); + return new CompilationPrerequisitesProvider(prerequisites.build()); + } + + /** + * Replaces shared library artifact with mangled symlink and creates related + * symlink action. For artifacts that should retain filename (e.g. libraries + * with SONAME tag), link is created to the parent directory instead. + * + * This action is performed to minimize number of -rpath entries used during + * linking process (by essentially "collecting" as many shared libraries as + * possible in the single directory), since we will be paying quadratic price + * for each additional entry on the -rpath. + * + * @param library Shared library artifact that needs to be mangled + * @param preserveName true if filename should be preserved, false - mangled. + * @return mangled symlink artifact. + */ + public LibraryToLink getDynamicLibrarySymlink(Artifact library, boolean preserveName) { + return SolibSymlinkAction.getDynamicLibrarySymlink( + ruleContext, library, preserveName, true, ruleContext.getConfiguration()); + } + + /** + * Returns any linker scripts found in the dependencies of the rule. + */ + Iterable<Artifact> getLinkerScripts() { + return FileType.filter(ruleContext.getPrerequisiteArtifacts("deps", Mode.TARGET).list(), + CppFileTypes.LINKER_SCRIPT); + } + + ImmutableList<Artifact> getFilesToCompile(CcCompilationOutputs compilationOutputs) { + return cppConfiguration.isLipoContextCollector() + ? ImmutableList.<Artifact>of() + : compilationOutputs.getObjectFiles(CppHelper.usePic(ruleContext, false)); + } + + InstrumentedFilesProvider getInstrumentedFilesProvider(Iterable<Artifact> files) { + return cppConfiguration.isLipoContextCollector() + ? InstrumentedFilesProviderImpl.EMPTY + : new InstrumentedFilesProviderImpl(new InstrumentedFilesCollector( + ruleContext, CppRuleClasses.INSTRUMENTATION_SPEC, CC_METADATA_COLLECTOR, files)); + } + + public static FeatureConfiguration configureFeatures(RuleContext ruleContext) { + CcToolchainProvider toolchain = CppHelper.getToolchain(ruleContext); + Set<String> requestedFeatures = ImmutableSet.of(CppRuleClasses.MODULE_MAP_HOME_CWD); + return toolchain.getFeatures().getFeatureConfiguration(requestedFeatures); + } + + public void addTransitiveInfoProviders(RuleConfiguredTargetBuilder builder, + NestedSet<Artifact> filesToBuild, + CcCompilationOutputs ccCompilationOutputs, + CppCompilationContext cppCompilationContext, + CcLinkingOutputs linkingOutputs, + DwoArtifactsCollector dwoArtifacts, + TransitiveLipoInfoProvider transitiveLipoInfo) { + List<Artifact> instrumentedObjectFiles = new ArrayList<>(); + instrumentedObjectFiles.addAll(ccCompilationOutputs.getObjectFiles(false)); + instrumentedObjectFiles.addAll(ccCompilationOutputs.getObjectFiles(true)); + builder + .setFilesToBuild(filesToBuild) + .add(CppCompilationContext.class, cppCompilationContext) + .add(TransitiveLipoInfoProvider.class, transitiveLipoInfo) + .add(CcExecutionDynamicLibrariesProvider.class, + new CcExecutionDynamicLibrariesProvider(collectExecutionDynamicLibraryArtifacts( + ruleContext, linkingOutputs.getExecutionDynamicLibraries()))) + .add(CcNativeLibraryProvider.class, new CcNativeLibraryProvider( + collectTransitiveCcNativeLibraries(ruleContext, linkingOutputs.getDynamicLibraries()))) + .add(InstrumentedFilesProvider.class, getInstrumentedFilesProvider( + instrumentedObjectFiles)) + .add(FilesToCompileProvider.class, new FilesToCompileProvider( + getFilesToCompile(ccCompilationOutputs))) + .add(CompilationPrerequisitesProvider.class, + collectCompilationPrerequisites(ruleContext, cppCompilationContext)) + .add(TempsProvider.class, new TempsProvider(getTemps(ccCompilationOutputs))) + .add(CppDebugFileProvider.class, new CppDebugFileProvider( + dwoArtifacts.getDwoArtifacts(), + dwoArtifacts.getPicDwoArtifacts())); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java new file mode 100644 index 0000000..b9fa4e8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java
@@ -0,0 +1,207 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * A structured representation of the compilation outputs of a C++ rule. + */ +public class CcCompilationOutputs { + /** + * All .o files built by the target. + */ + private final ImmutableList<Artifact> objectFiles; + + /** + * All .pic.o files built by the target. + */ + private final ImmutableList<Artifact> picObjectFiles; + + /** + * All .dwo files built by the target, corresponding to .o outputs. + */ + private final ImmutableList<Artifact> dwoFiles; + + /** + * All .pic.dwo files built by the target, corresponding to .pic.o outputs. + */ + private final ImmutableList<Artifact> picDwoFiles; + + /** + * All artifacts that are created if "--save_temps" is true. + */ + private final ImmutableList<Artifact> temps; + + /** + * All token .h.processed files created when preprocessing or parsing headers. + */ + private final ImmutableList<Artifact> headerTokenFiles; + + private final List<IncludeScannable> lipoScannables; + + private CcCompilationOutputs(ImmutableList<Artifact> objectFiles, + ImmutableList<Artifact> picObjectFiles, ImmutableList<Artifact> dwoFiles, + ImmutableList<Artifact> picDwoFiles, ImmutableList<Artifact> temps, + ImmutableList<Artifact> headerTokenFiles, + ImmutableList<IncludeScannable> lipoScannables) { + this.objectFiles = objectFiles; + this.picObjectFiles = picObjectFiles; + this.dwoFiles = dwoFiles; + this.picDwoFiles = picDwoFiles; + this.temps = temps; + this.headerTokenFiles = headerTokenFiles; + this.lipoScannables = lipoScannables; + } + + /** + * Returns an unmodifiable view of the .o or .pic.o files set. + * + * @param usePic whether to return .pic.o files + */ + public ImmutableList<Artifact> getObjectFiles(boolean usePic) { + return usePic ? picObjectFiles : objectFiles; + } + + /** + * Returns an unmodifiable view of the .dwo files set. + */ + public ImmutableList<Artifact> getDwoFiles() { + return dwoFiles; + } + + /** + * Returns an unmodifiable view of the .pic.dwo files set. + */ + public ImmutableList<Artifact> getPicDwoFiles() { + return picDwoFiles; + } + + /** + * Returns an unmodifiable view of the temp files set. + */ + public ImmutableList<Artifact> getTemps() { + return temps; + } + + /** + * Returns an unmodifiable view of the .h.processed files. + */ + public Iterable<Artifact> getHeaderTokenFiles() { + return headerTokenFiles; + } + + /** + * Returns the {@link IncludeScannable} objects this C++ compile action contributes to a + * LIPO context collector. + */ + public List<IncludeScannable> getLipoScannables() { + return lipoScannables; + } + + public static final class Builder { + private final Set<Artifact> objectFiles = new LinkedHashSet<>(); + private final Set<Artifact> picObjectFiles = new LinkedHashSet<>(); + private final Set<Artifact> dwoFiles = new LinkedHashSet<>(); + private final Set<Artifact> picDwoFiles = new LinkedHashSet<>(); + private final Set<Artifact> temps = new LinkedHashSet<>(); + private final Set<Artifact> headerTokenFiles = new LinkedHashSet<>(); + private final List<IncludeScannable> lipoScannables = new ArrayList<>(); + + public CcCompilationOutputs build() { + return new CcCompilationOutputs(ImmutableList.copyOf(objectFiles), + ImmutableList.copyOf(picObjectFiles), ImmutableList.copyOf(dwoFiles), + ImmutableList.copyOf(picDwoFiles), ImmutableList.copyOf(temps), + ImmutableList.copyOf(headerTokenFiles), + ImmutableList.copyOf(lipoScannables)); + } + + public Builder merge(CcCompilationOutputs outputs) { + this.objectFiles.addAll(outputs.objectFiles); + this.picObjectFiles.addAll(outputs.picObjectFiles); + this.dwoFiles.addAll(outputs.dwoFiles); + this.picDwoFiles.addAll(outputs.picDwoFiles); + this.temps.addAll(outputs.temps); + this.headerTokenFiles.addAll(outputs.headerTokenFiles); + this.lipoScannables.addAll(outputs.lipoScannables); + return this; + } + + /** + * Adds an .o file. + */ + public Builder addObjectFile(Artifact artifact) { + objectFiles.add(artifact); + return this; + } + + public Builder addObjectFiles(Iterable<Artifact> artifacts) { + Iterables.addAll(objectFiles, artifacts); + return this; + } + + /** + * Adds a .pic.o file. + */ + public Builder addPicObjectFile(Artifact artifact) { + picObjectFiles.add(artifact); + return this; + } + + public Builder addPicObjectFiles(Iterable<Artifact> artifacts) { + Iterables.addAll(picObjectFiles, artifacts); + return this; + } + + public Builder addDwoFile(Artifact artifact) { + dwoFiles.add(artifact); + return this; + } + + public Builder addPicDwoFile(Artifact artifact) { + picDwoFiles.add(artifact); + return this; + } + + /** + * Adds temp files. + */ + public Builder addTemps(Iterable<Artifact> artifacts) { + Iterables.addAll(temps, artifacts); + return this; + } + + public Builder addHeaderTokenFile(Artifact artifact) { + headerTokenFiles.add(artifact); + return this; + } + + /** + * Adds an {@link IncludeScannable} that this compilation output object contributes to a + * LIPO context collector. + */ + public Builder addLipoScannable(IncludeScannable scannable) { + lipoScannables.add(scannable); + return this; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcExecutionDynamicLibrariesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcExecutionDynamicLibrariesProvider.java new file mode 100644 index 0000000..39ce942 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcExecutionDynamicLibrariesProvider.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A target that provides the execution-time dynamic libraries of a C++ rule. + */ +@Immutable +public final class CcExecutionDynamicLibrariesProvider implements TransitiveInfoProvider { + public static final CcExecutionDynamicLibrariesProvider EMPTY = + new CcExecutionDynamicLibrariesProvider( + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER)); + + private final NestedSet<Artifact> ccExecutionDynamicLibraries; + + public CcExecutionDynamicLibrariesProvider(NestedSet<Artifact> ccExecutionDynamicLibraries) { + this.ccExecutionDynamicLibraries = ccExecutionDynamicLibraries; + } + + /** + * Returns the execution-time dynamic libraries. + * + * <p>This normally returns the dynamic library created by the rule itself. However, if the rule + * does not create any dynamic libraries, then it returns the combined results of calling + * getExecutionDynamicLibraryArtifacts on all the rule's deps. This behaviour is so that this + * method is useful for a cc_library with deps but no srcs. + */ + public NestedSet<Artifact> getExecutionDynamicLibraryArtifacts() { + return ccExecutionDynamicLibraries; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibrary.java new file mode 100644 index 0000000..428eedb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibrary.java
@@ -0,0 +1,395 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AlwaysBuiltArtifactsProvider; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.rules.test.BaselineCoverageAction; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.List; + +/** + * A ConfiguredTarget for <code>cc_library</code> rules. + */ +public abstract class CcLibrary implements RuleConfiguredTargetFactory { + + private final CppSemantics semantics; + + protected CcLibrary(CppSemantics semantics) { + this.semantics = semantics; + } + + // These file extensions don't generate object files. + private static final FileTypeSet NO_OBJECT_GENERATING_FILETYPES = FileTypeSet.of( + CppFileTypes.CPP_HEADER, CppFileTypes.ARCHIVE, CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_LIBRARY, CppFileTypes.ALWAYS_LINK_PIC_LIBRARY, + CppFileTypes.SHARED_LIBRARY); + + private static final Predicate<LibraryToLink> PIC_STATIC_FILTER = new Predicate<LibraryToLink>() { + @Override + public boolean apply(LibraryToLink input) { + String name = input.getArtifact().getExecPath().getBaseName(); + return !name.endsWith(".nopic.a") && !name.endsWith(".nopic.lo"); + } + }; + + private static Runfiles collectRunfiles(RuleContext context, + CcLinkingOutputs ccLinkingOutputs, + boolean neverLink, boolean addDynamicRuntimeInputArtifactsToRunfiles, + boolean linkingStatically) { + Runfiles.Builder builder = new Runfiles.Builder(); + + // neverlink= true creates a library that will never be linked into any binary that depends on + // it, but instead be loaded as an extension. So we need the dynamic library for this in the + // runfiles. + builder.addArtifacts(ccLinkingOutputs.getLibrariesForRunfiles(linkingStatically && !neverLink)); + builder.add(context, CppRunfilesProvider.runfilesFunction(linkingStatically)); + if (context.getRule().isAttrDefined("implements", Type.LABEL_LIST)) { + builder.addTargets(context.getPrerequisites("implements", Mode.TARGET), + RunfilesProvider.DEFAULT_RUNFILES); + builder.addTargets(context.getPrerequisites("implements", Mode.TARGET), + CppRunfilesProvider.runfilesFunction(linkingStatically)); + } + if (context.getRule().isAttrDefined("implementation", Type.LABEL_LIST)) { + builder.addTargets(context.getPrerequisites("implementation", Mode.TARGET), + RunfilesProvider.DEFAULT_RUNFILES); + builder.addTargets(context.getPrerequisites("implementation", Mode.TARGET), + CppRunfilesProvider.runfilesFunction(linkingStatically)); + } + + builder.addDataDeps(context); + + if (addDynamicRuntimeInputArtifactsToRunfiles) { + builder.addTransitiveArtifacts(CppHelper.getToolchain(context).getDynamicRuntimeLinkInputs()); + } + return builder.build(); + } + + @Override + public ConfiguredTarget create(RuleContext context) { + RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(context); + LinkTargetType linkType = getStaticLinkType(context); + boolean linkStatic = context.attributes().get("linkstatic", Type.BOOLEAN); + init(semantics, context, builder, linkType, + /*neverLink =*/ false, + linkStatic, + /*collectLinkstamp =*/ true, + /*addDynamicRuntimeInputArtifactsToRunfiles =*/ false); + return builder.build(); + } + + public static void init(CppSemantics semantics, RuleContext ruleContext, + RuleConfiguredTargetBuilder targetBuilder, LinkTargetType linkType, + boolean neverLink, + boolean linkStatic, + boolean collectLinkstamp, + boolean addDynamicRuntimeInputArtifactsToRunfiles) { + final CcCommon common = new CcCommon(ruleContext); + + CcLibraryHelper helper = new CcLibraryHelper(ruleContext, semantics) + .setLinkType(linkType) + .enableCcNativeLibrariesProvider() + .enableInterfaceSharedObjects() + .enableCompileProviders() + .setNeverLink(neverLink) + .setHeadersCheckingMode(common.determineHeadersCheckingMode()) + .addCopts(common.getCopts()) + .setNoCopts(common.getNoCopts()) + .addLinkopts(common.getLinkopts()) + .addDefines(common.getDefines()) + .addCompilationPrerequisites(common.getSharedLibrariesFromSrcs()) + .addCompilationPrerequisites(common.getStaticLibrariesFromSrcs()) + .addSources(common.getCAndCppSources()) + .addPublicHeaders(common.getHeaders()) + .addObjectFiles(common.getObjectFilesFromSrcs(false)) + .addPicObjectFiles(common.getObjectFilesFromSrcs(true)) + .addPicIndependentObjectFiles(common.getLinkerScripts()) + .addDeps(ruleContext.getPrerequisites("deps", Mode.TARGET)) + .setEnableLayeringCheck(ruleContext.getFeatures().contains(CppRuleClasses.LAYERING_CHECK)) + .setCompileHeaderModules(ruleContext.getFeatures().contains(CppRuleClasses.HEADER_MODULES)) + .addSystemIncludeDirs(common.getSystemIncludeDirs()) + .addIncludeDirs(common.getIncludeDirs()) + .addLooseIncludeDirs(common.getLooseIncludeDirs()) + .setEmitHeaderTargetModuleMaps( + ruleContext.getRule().getRuleClass().equals("cc_public_library")); + + if (collectLinkstamp) { + helper.addLinkstamps(ruleContext.getPrerequisites("linkstamp", Mode.TARGET)); + } + + if (ruleContext.getRule().isAttrDefined("implements", Type.LABEL_LIST)) { + helper.addDeps(ruleContext.getPrerequisites("implements", Mode.TARGET)); + } + + if (ruleContext.getRule().isAttrDefined("implementation", Type.LABEL_LIST)) { + helper.addDeps(ruleContext.getPrerequisites("implementation", Mode.TARGET)); + } + + PathFragment soImplFilename = null; + if (ruleContext.getRule().isAttrDefined("outs", Type.STRING_LIST)) { + List<String> outs = ruleContext.attributes().get("outs", Type.STRING_LIST); + if (outs.size() > 1) { + ruleContext.attributeError("outs", "must be a singleton list"); + } else if (outs.size() == 1) { + soImplFilename = CppHelper.getLinkedFilename(ruleContext, LinkTargetType.DYNAMIC_LIBRARY); + soImplFilename = soImplFilename.replaceName(outs.get(0)); + if (!soImplFilename.getPathString().endsWith(".so")) { // Sanity check. + ruleContext.attributeError("outs", "file name must end in '.so'"); + } + } + } + + if (ruleContext.getRule().isAttrDefined("srcs", Type.LABEL_LIST)) { + helper.addPrivateHeaders(FileType.filter( + ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list(), + CppFileTypes.CPP_HEADER)); + ruleContext.checkSrcsSamePackage(true); + } + + if (common.getLinkopts().contains("-static")) { + ruleContext.attributeWarning("linkopts", "Using '-static' here won't work. " + + "Did you mean to use 'linkstatic=1' instead?"); + } + + boolean createDynamicLibrary = + !linkStatic && !appearsToHaveNoObjectFiles(ruleContext.attributes()); + helper.setCreateDynamicLibrary(createDynamicLibrary); + helper.setDynamicLibraryPath(soImplFilename); + + /* + * Add the libraries from srcs, if any. For static/mostly static + * linking we setup the dynamic libraries if there are no static libraries + * to choose from. Path to the libraries will be mangled to avoid using + * absolute path names on the -rpath, but library filenames will be + * preserved (since some libraries might have SONAME tag) - symlink will + * be created to the parent directory instead. + * + * For compatibility with existing BUILD files, any ".a" or ".lo" files listed in + * srcs are assumed to be position-independent code, or at least suitable for + * inclusion in shared libraries, unless they end with ".nopic.a" or ".nopic.lo". + * + * Note that some target platforms do not require shared library code to be PIC. + */ + Iterable<LibraryToLink> staticLibrariesFromSrcs = + LinkerInputs.opaqueLibrariesToLink(common.getStaticLibrariesFromSrcs()); + helper.addStaticLibraries(staticLibrariesFromSrcs); + helper.addPicStaticLibraries(Iterables.filter(staticLibrariesFromSrcs, PIC_STATIC_FILTER)); + helper.addPicStaticLibraries(common.getPicStaticLibrariesFromSrcs()); + helper.addDynamicLibraries(Iterables.transform(common.getSharedLibrariesFromSrcs(), + new Function<Artifact, LibraryToLink>() { + @Override + public LibraryToLink apply(Artifact library) { + return common.getDynamicLibrarySymlink(library, true); + } + })); + CcLibraryHelper.Info info = helper.build(); + + /* + * We always generate a static library, even if there aren't any source files. + * This keeps things simpler by avoiding special cases when making use of the library. + * For example, this is needed to ensure that building a library with "bazel build" + * will also build all of the library's "deps". + * However, we only generate a dynamic library if there are source files. + */ + // For now, we don't add the precompiled libraries to the files to build. + CcLinkingOutputs linkedLibraries = info.getCcLinkingOutputsExcludingPrecompiledLibraries(); + + NestedSet<Artifact> artifactsToForce = + collectArtifactsToForce(ruleContext, common, info.getCcCompilationOutputs()); + + NestedSetBuilder<Artifact> filesBuilder = NestedSetBuilder.stableOrder(); + filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkedLibraries.getStaticLibraries())); + filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkedLibraries.getPicStaticLibraries())); + filesBuilder.addAll(LinkerInputs.toNonSolibArtifacts(linkedLibraries.getDynamicLibraries())); + filesBuilder.addAll( + LinkerInputs.toNonSolibArtifacts(linkedLibraries.getExecutionDynamicLibraries())); + + CcLinkingOutputs linkingOutputs = info.getCcLinkingOutputs(); + warnAboutEmptyLibraries( + ruleContext, info.getCcCompilationOutputs(), linkType, linkStatic); + NestedSet<Artifact> filesToBuild = filesBuilder.build(); + + Runfiles staticRunfiles = collectRunfiles(ruleContext, + linkingOutputs, neverLink, addDynamicRuntimeInputArtifactsToRunfiles, true); + Runfiles sharedRunfiles = collectRunfiles(ruleContext, + linkingOutputs, neverLink, addDynamicRuntimeInputArtifactsToRunfiles, false); + + List<Artifact> instrumentedObjectFiles = new ArrayList<>(); + instrumentedObjectFiles.addAll(info.getCcCompilationOutputs().getObjectFiles(false)); + instrumentedObjectFiles.addAll(info.getCcCompilationOutputs().getObjectFiles(true)); + InstrumentedFilesProvider instrumentedFilesProvider = + common.getInstrumentedFilesProvider(instrumentedObjectFiles); + targetBuilder + .setFilesToBuild(filesToBuild) + .addProviders(info.getProviders()) + .add(InstrumentedFilesProvider.class, instrumentedFilesProvider) + .add(RunfilesProvider.class, RunfilesProvider.withData(staticRunfiles, sharedRunfiles)) + // Remove this? + .add(CppRunfilesProvider.class, new CppRunfilesProvider(staticRunfiles, sharedRunfiles)) + .setBaselineCoverageArtifacts(BaselineCoverageAction.getBaselineCoverageArtifacts( + ruleContext, instrumentedFilesProvider.getInstrumentedFiles())) + .add(ImplementedCcPublicLibrariesProvider.class, + new ImplementedCcPublicLibrariesProvider(getImplementedCcPublicLibraries(ruleContext))) + .add(AlwaysBuiltArtifactsProvider.class, + new AlwaysBuiltArtifactsProvider(artifactsToForce)); + } + + private static NestedSet<Artifact> collectArtifactsToForce(RuleContext ruleContext, + CcCommon common, CcCompilationOutputs ccCompilationOutputs) { + // Ensure that we build all the dependencies, otherwise users may get confused. + NestedSetBuilder<Artifact> artifactsToForceBuilder = NestedSetBuilder.stableOrder(); + artifactsToForceBuilder.addTransitive( + NestedSetBuilder.wrap(Order.STABLE_ORDER, common.getFilesToCompile(ccCompilationOutputs))); + for (AlwaysBuiltArtifactsProvider dep : + ruleContext.getPrerequisites("deps", Mode.TARGET, AlwaysBuiltArtifactsProvider.class)) { + artifactsToForceBuilder.addTransitive(dep.getArtifactsToAlwaysBuild()); + } + return artifactsToForceBuilder.build(); + } + + /** + * Returns the type of the generated static library. + */ + private static LinkTargetType getStaticLinkType(RuleContext context) { + return context.attributes().get("alwayslink", Type.BOOLEAN) + ? LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY + : LinkTargetType.STATIC_LIBRARY; + } + + private static void warnAboutEmptyLibraries(RuleContext ruleContext, + CcCompilationOutputs ccCompilationOutputs, LinkTargetType linkType, + boolean linkstaticAttribute) { + if (ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector()) { + // Do not signal warnings in the lipo context collector configuration. These will be duly + // signaled in the target configuration, and there can be spurious warnings since targets in + // the LIPO context collector configuration do not compile anything. + return; + } + if (ccCompilationOutputs.getObjectFiles(false).isEmpty() + && ccCompilationOutputs.getObjectFiles(true).isEmpty()) { + if (linkType == LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY + || linkType == LinkTargetType.ALWAYS_LINK_PIC_STATIC_LIBRARY) { + ruleContext.attributeWarning("alwayslink", + "'alwayslink' has no effect if there are no 'srcs'"); + } + if (!linkstaticAttribute && !appearsToHaveNoObjectFiles(ruleContext.attributes())) { + ruleContext.attributeWarning("linkstatic", + "setting 'linkstatic=1' is recommended if there are no object files"); + } + } else { + if (!linkstaticAttribute && appearsToHaveNoObjectFiles(ruleContext.attributes())) { + Artifact element = ccCompilationOutputs.getObjectFiles(false).isEmpty() + ? ccCompilationOutputs.getObjectFiles(true).get(0) + : ccCompilationOutputs.getObjectFiles(false).get(0); + ruleContext.attributeWarning("srcs", + "this library appears at first glance to have no object files, " + + "but on closer inspection it does have something to link, e.g. " + + element.prettyPrint() + ". " + + "(You may have used some very confusing rule names in srcs? " + + "Or the library consists entirely of a linker script?) " + + "Bazel assumed linkstatic=1, but this may be inappropriate. " + + "You may need to add an explicit '.cc' file to 'srcs'. " + + "Alternatively, add 'linkstatic=1' to suppress this warning"); + } + } + } + + private static ImmutableList<Label> getImplementedCcPublicLibraries(RuleContext context) { + if (context.getRule().getRuleClassObject().hasAttr("implements", Type.LABEL_LIST)) { + return ImmutableList.copyOf(context.attributes().get("implements", Type.LABEL_LIST)); + } else { + return ImmutableList.of(); + } + } + + /** + * Returns true if the rule (which must be a cc_library rule) + * appears to have no object files. This only looks at the rule + * itself, not at any other rules (from this package or other + * packages) that it might reference. + * + * <p> + * In some cases, this may return "false" even + * though the rule actually has no object files. + * For example, it will return false for a rule such as + * <code>cc_library(name = 'foo', srcs = [':bar'])</code> + * because we can't tell what ':bar' is; it might + * be a genrule that generates a source file, or it might + * be a genrule that generates a header file. + * + * <p> + * In other cases, this may return "true" even + * though the rule actually does have object files. + * For example, it will return true for a rule such as + * <code>cc_library(name = 'foo', srcs = ['bar.h'])</code> + * but as in the other example above, we can't tell whether + * 'bar.h' is a file name or a rule name, and 'bar.h' could + * in fact be the name of a genrule that generates a source file. + */ + public static boolean appearsToHaveNoObjectFiles(AttributeMap rule) { + // Temporary hack while configurable attributes is under development. This has no effect + // for any rule that doesn't use configurable attributes. + // TODO(bazel-team): remove this hack for a more principled solution. + try { + rule.get("srcs", Type.LABEL_LIST); + } catch (ClassCastException e) { + // "srcs" is actually a configurable selector. Assume object files are possible somewhere. + return false; + } + + List<Label> srcs = rule.get("srcs", Type.LABEL_LIST); + if (srcs != null) { + for (Label srcfile : srcs) { + /* + * We cheat a little bit here by looking at the file extension + * of the Label treated as file name. In general that might + * not necessarily work, because of the possibility that the + * user might give a rule a funky name ending in one of these + * extensions, e.g. + * genrule(name = 'foo.h', outs = ['foo.cc'], ...) // Funky rule name! + * cc_library(name = 'bar', srcs = ['foo.h']) // This DOES have object files. + */ + if (!NO_OBJECT_GENERATING_FILETYPES.matches(srcfile.getName())) { + return false; + } + } + } + return true; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibraryHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibraryHelper.java new file mode 100644 index 0000000..3812bf5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibraryHelper.java
@@ -0,0 +1,905 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.CompilationPrerequisitesProvider; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToCompileProvider; +import com.google.devtools.build.lib.analysis.LanguageDependentFragment; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TempsProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.HeadersCheckingMode; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +/** + * A class to create C/C++ compile and link actions in a way that is consistent with cc_library. + * Rules that generate source files and emulate cc_library on top of that should use this class + * instead of the lower-level APIs in CppHelper and CppModel. + * + * <p>Rules that want to use this class are required to have implicit dependencies on the + * toolchain, the STL, the lipo context, and so on. Optionally, they can also have copts, + * and malloc attributes, but note that these require explicit calls to the corresponding setter + * methods. + */ +public final class CcLibraryHelper { + /** Function for extracting module maps from CppCompilationDependencies. */ + public static final Function<TransitiveInfoCollection, CppModuleMap> CPP_DEPS_TO_MODULES = + new Function<TransitiveInfoCollection, CppModuleMap>() { + @Override + @Nullable + public CppModuleMap apply(TransitiveInfoCollection dep) { + CppCompilationContext context = dep.getProvider(CppCompilationContext.class); + return context == null ? null : context.getCppModuleMap(); + } + }; + + /** + * Contains the providers as well as the compilation and linking outputs, and the compilation + * context. + */ + public static final class Info { + private final Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers; + private final CcCompilationOutputs compilationOutputs; + private final CcLinkingOutputs linkingOutputs; + private final CcLinkingOutputs linkingOutputsExcludingPrecompiledLibraries; + private final CppCompilationContext context; + + private Info(Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers, + CcCompilationOutputs compilationOutputs, CcLinkingOutputs linkingOutputs, + CcLinkingOutputs linkingOutputsExcludingPrecompiledLibraries, + CppCompilationContext context) { + this.providers = Collections.unmodifiableMap(providers); + this.compilationOutputs = compilationOutputs; + this.linkingOutputs = linkingOutputs; + this.linkingOutputsExcludingPrecompiledLibraries = + linkingOutputsExcludingPrecompiledLibraries; + this.context = context; + } + + public Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> getProviders() { + return providers; + } + + public CcCompilationOutputs getCcCompilationOutputs() { + return compilationOutputs; + } + + public CcLinkingOutputs getCcLinkingOutputs() { + return linkingOutputs; + } + + /** + * Returns the linking outputs before adding the pre-compiled libraries. Avoid using this - + * pre-compiled and locally compiled libraries should be treated identically. This method only + * exists for backwards compatibility. + */ + public CcLinkingOutputs getCcLinkingOutputsExcludingPrecompiledLibraries() { + return linkingOutputsExcludingPrecompiledLibraries; + } + + public CppCompilationContext getCppCompilationContext() { + return context; + } + + /** + * Adds the static, pic-static, and dynamic (both compile-time and execution-time) libraries to + * the given builder. + */ + public void addLinkingOutputsTo(NestedSetBuilder<Artifact> filesBuilder) { + filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkingOutputs.getStaticLibraries())); + filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkingOutputs.getPicStaticLibraries())); + filesBuilder.addAll(LinkerInputs.toNonSolibArtifacts(linkingOutputs.getDynamicLibraries())); + filesBuilder.addAll( + LinkerInputs.toNonSolibArtifacts(linkingOutputs.getExecutionDynamicLibraries())); + } + } + + private final RuleContext ruleContext; + private final BuildConfiguration configuration; + private final CppSemantics semantics; + + private final List<Artifact> publicHeaders = new ArrayList<>(); + private final List<Artifact> privateHeaders = new ArrayList<>(); + private final List<PathFragment> additionalExportedHeaders = new ArrayList<>(); + private final List<Pair<Artifact, Label>> sources = new ArrayList<>(); + private final List<Artifact> objectFiles = new ArrayList<>(); + private final List<Artifact> picObjectFiles = new ArrayList<>(); + private final List<String> copts = new ArrayList<>(); + @Nullable private Pattern nocopts; + private final List<String> linkopts = new ArrayList<>(); + private final Set<String> defines = new LinkedHashSet<>(); + private final List<TransitiveInfoCollection> deps = new ArrayList<>(); + private final List<Artifact> linkstamps = new ArrayList<>(); + private final List<Artifact> prerequisites = new ArrayList<>(); + private final List<PathFragment> looseIncludeDirs = new ArrayList<>(); + private final List<PathFragment> systemIncludeDirs = new ArrayList<>(); + private final List<PathFragment> includeDirs = new ArrayList<>(); + @Nullable private PathFragment dynamicLibraryPath; + private LinkTargetType linkType = LinkTargetType.STATIC_LIBRARY; + private HeadersCheckingMode headersCheckingMode = HeadersCheckingMode.LOOSE; + private boolean neverlink; + private boolean fake; + + private final List<LibraryToLink> staticLibraries = new ArrayList<>(); + private final List<LibraryToLink> picStaticLibraries = new ArrayList<>(); + private final List<LibraryToLink> dynamicLibraries = new ArrayList<>(); + + private boolean emitCppModuleMaps = true; + private boolean enableLayeringCheck; + private boolean compileHeaderModules; + private boolean emitCompileActionsIfEmpty = true; + private boolean emitCcNativeLibrariesProvider; + private boolean emitCcSpecificLinkParamsProvider; + private boolean emitInterfaceSharedObjects; + private boolean emitDynamicLibrary = true; + private boolean checkDepsGenerateCpp = true; + private boolean emitCompileProviders; + private boolean emitHeaderTargetModuleMaps = false; + + public CcLibraryHelper(RuleContext ruleContext, CppSemantics semantics) { + this.ruleContext = Preconditions.checkNotNull(ruleContext); + this.configuration = ruleContext.getConfiguration(); + this.semantics = Preconditions.checkNotNull(semantics); + } + + /** + * Add the corresponding files as header files, i.e., these files will not be compiled, but are + * made visible as includes to dependent rules. + */ + public CcLibraryHelper addPublicHeaders(Collection<Artifact> headers) { + this.publicHeaders.addAll(headers); + return this; + } + + /** + * Add the corresponding files as public header files, i.e., these files will not be compiled, but + * are made visible as includes to dependent rules in module maps. + */ + public CcLibraryHelper addPublicHeaders(Artifact... headers) { + return addPublicHeaders(Arrays.asList(headers)); + } + + /** + * Add the corresponding files as private header files, i.e., these files will not be compiled, + * but are not made visible as includes to dependent rules in module maps. + */ + public CcLibraryHelper addPrivateHeaders(Iterable<Artifact> privateHeaders) { + Iterables.addAll(this.privateHeaders, privateHeaders); + return this; + } + + /** + * Add the corresponding files as public header files, i.e., these files will not be compiled, but + * are made visible as includes to dependent rules in module maps. + */ + public CcLibraryHelper addAdditionalExportedHeaders( + Iterable<PathFragment> additionalExportedHeaders) { + Iterables.addAll(this.additionalExportedHeaders, additionalExportedHeaders); + return this; + } + + /** + * Add the corresponding files as source files. These may also be header files, in which case + * they will not be compiled, but also not made visible as includes to dependent rules. + */ + // TODO(bazel-team): This is inconsistent with the documentation on CppModel. + public CcLibraryHelper addSources(Collection<Artifact> sources) { + for (Artifact source : sources) { + this.sources.add(Pair.of(source, ruleContext.getLabel())); + } + return this; + } + + /** + * Add the corresponding files as source files. These may also be header files, in which case + * they will not be compiled, but also not made visible as includes to dependent rules. + */ + // TODO(bazel-team): This is inconsistent with the documentation on CppModel. + public CcLibraryHelper addSources(Iterable<Pair<Artifact, Label>> sources) { + Iterables.addAll(this.sources, sources); + return this; + } + + /** + * Add the corresponding files as source files. These may also be header files, in which case + * they will not be compiled, but also not made visible as includes to dependent rules. + */ + public CcLibraryHelper addSources(Artifact... sources) { + return addSources(Arrays.asList(sources)); + } + + /** + * Add the corresponding files as linker inputs for non-PIC links. If the corresponding files are + * compiled with PIC, the final link may or may not fail. Note that the final link may not happen + * here, if {@code --start_end_lib} is enabled, but instead at any binary that transitively + * depends on the current rule. + */ + public CcLibraryHelper addObjectFiles(Iterable<Artifact> objectFiles) { + Iterables.addAll(this.objectFiles, objectFiles); + return this; + } + + /** + * Add the corresponding files as linker inputs for PIC links. If the corresponding files are not + * compiled with PIC, the final link may or may not fail. Note that the final link may not happen + * here, if {@code --start_end_lib} is enabled, but instead at any binary that transitively + * depends on the current rule. + */ + public CcLibraryHelper addPicObjectFiles(Iterable<Artifact> picObjectFiles) { + Iterables.addAll(this.picObjectFiles, picObjectFiles); + return this; + } + + /** + * Add the corresponding files as linker inputs for both PIC and non-PIC links. + */ + public CcLibraryHelper addPicIndependentObjectFiles(Iterable<Artifact> objectFiles) { + addPicObjectFiles(objectFiles); + return addObjectFiles(objectFiles); + } + + /** + * Add the corresponding files as linker inputs for both PIC and non-PIC links. + */ + public CcLibraryHelper addPicIndependentObjectFiles(Artifact... objectFiles) { + return addPicIndependentObjectFiles(Arrays.asList(objectFiles)); + } + + /** + * Add the corresponding files as static libraries into the linker outputs (i.e., after the linker + * action) - this makes them available for linking to binary rules that depend on this rule. + */ + public CcLibraryHelper addStaticLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(staticLibraries, libraries); + return this; + } + + /** + * Add the corresponding files as static libraries into the linker outputs (i.e., after the linker + * action) - this makes them available for linking to binary rules that depend on this rule. + */ + public CcLibraryHelper addPicStaticLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(picStaticLibraries, libraries); + return this; + } + + /** + * Add the corresponding files as dynamic libraries into the linker outputs (i.e., after the + * linker action) - this makes them available for linking to binary rules that depend on this + * rule. + */ + public CcLibraryHelper addDynamicLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(dynamicLibraries, libraries); + return this; + } + + /** + * Adds the copts to the compile command line. + */ + public CcLibraryHelper addCopts(Iterable<String> copts) { + Iterables.addAll(this.copts, copts); + return this; + } + + /** + * Sets a pattern that is used to filter copts; set to {@code null} for no filtering. + */ + public CcLibraryHelper setNoCopts(@Nullable Pattern nocopts) { + this.nocopts = nocopts; + return this; + } + + /** + * Adds the given options as linker options to the link command. + */ + public CcLibraryHelper addLinkopts(Iterable<String> linkopts) { + Iterables.addAll(this.linkopts, linkopts); + return this; + } + + /** + * Adds the given defines to the compiler command line. + */ + public CcLibraryHelper addDefines(Iterable<String> defines) { + Iterables.addAll(this.defines, defines); + return this; + } + + /** + * Adds the given targets as dependencies - this can include explicit dependencies on other + * rules (like from a "deps" attribute) and also implicit dependencies on runtime libraries. + */ + public CcLibraryHelper addDeps(Iterable<? extends TransitiveInfoCollection> deps) { + for (TransitiveInfoCollection dep : deps) { + Preconditions.checkArgument(dep.getConfiguration() == null + || dep.getConfiguration().equals(configuration)); + this.deps.add(dep); + } + return this; + } + + /** + * Adds the given linkstamps. Note that linkstamps are usually not compiled at the library level, + * but only in the dependent binary rules. + */ + public CcLibraryHelper addLinkstamps(Iterable<? extends TransitiveInfoCollection> linkstamps) { + for (TransitiveInfoCollection linkstamp : linkstamps) { + Iterables.addAll(this.linkstamps, + linkstamp.getProvider(FileProvider.class).getFilesToBuild()); + } + return this; + } + + /** + * Adds the given prerequisites as prerequisites for the generated compile actions. This ensures + * that the corresponding files exist - otherwise the action fails. Note that these dependencies + * add edges to the action graph, and can therefore increase the length of the critical path, + * i.e., make the build slower. + */ + public CcLibraryHelper addCompilationPrerequisites(Iterable<Artifact> prerequisites) { + Iterables.addAll(this.prerequisites, prerequisites); + return this; + } + + /** + * Adds the given directories to the loose include directories that are only allowed to be + * referenced when headers checking is {@link HeadersCheckingMode#LOOSE} or {@link + * HeadersCheckingMode#WARN}. + */ + public CcLibraryHelper addLooseIncludeDirs(Iterable<PathFragment> looseIncludeDirs) { + Iterables.addAll(this.looseIncludeDirs, looseIncludeDirs); + return this; + } + + /** + * Adds the given directories to the system include directories (they are passed with {@code + * "-isystem"} to the compiler); these are also passed to dependent rules. + */ + public CcLibraryHelper addSystemIncludeDirs(Iterable<PathFragment> systemIncludeDirs) { + Iterables.addAll(this.systemIncludeDirs, systemIncludeDirs); + return this; + } + + /** + * Adds the given directories to the quote include directories (they are passed with {@code + * "-iquote"} to the compiler); these are also passed to dependent rules. + */ + public CcLibraryHelper addIncludeDirs(Iterable<PathFragment> includeDirs) { + Iterables.addAll(this.includeDirs, includeDirs); + return this; + } + + /** + * Overrides the path for the generated dynamic library - this should only be called if the + * dynamic library is an implicit or explicit output of the rule, i.e., if it is accessible by + * name from other rules in the same package. Set to {@code null} to use the default computation. + */ + public CcLibraryHelper setDynamicLibraryPath(@Nullable PathFragment dynamicLibraryPath) { + this.dynamicLibraryPath = dynamicLibraryPath; + return this; + } + + /** + * Marks the output of this rule as alwayslink, i.e., the corresponding symbols will be retained + * by the linker even if they are not otherwise used. This is useful for libraries that register + * themselves somewhere during initialization. + * + * <p>This only sets the link type (see {@link #setLinkType}), either to a static library or to + * an alwayslink static library (blaze uses a different file extension to signal alwayslink to + * downstream code). + */ + public CcLibraryHelper setAlwayslink(boolean alwayslink) { + linkType = alwayslink + ? LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY + : LinkTargetType.STATIC_LIBRARY; + return this; + } + + /** + * Directly set the link type. This can be used instead of {@link #setAlwayslink}. Setting + * anything other than a static link causes this class to skip the link action creation. + */ + public CcLibraryHelper setLinkType(LinkTargetType linkType) { + this.linkType = Preconditions.checkNotNull(linkType); + return this; + } + + /** + * Marks the resulting code as neverlink, i.e., the code will not be linked into dependent + * libraries or binaries - the header files are still available. + */ + public CcLibraryHelper setNeverLink(boolean neverlink) { + this.neverlink = neverlink; + return this; + } + + /** + * Sets the given headers checking mode. The default is {@link HeadersCheckingMode#LOOSE}. + */ + public CcLibraryHelper setHeadersCheckingMode(HeadersCheckingMode headersCheckingMode) { + this.headersCheckingMode = Preconditions.checkNotNull(headersCheckingMode); + return this; + } + + /** + * Marks the resulting code as fake, i.e., the code will not actually be compiled or linked, but + * instead, the compile command is written to a file and added to the runfiles. This is currently + * used for non-compilation tests. Unfortunately, the design is problematic, so please don't add + * any further uses. + */ + public CcLibraryHelper setFake(boolean fake) { + this.fake = fake; + return this; + } + + /** + * This adds the {@link CcNativeLibraryProvider} to the providers created by this class. + */ + public CcLibraryHelper enableCcNativeLibrariesProvider() { + this.emitCcNativeLibrariesProvider = true; + return this; + } + + /** + * This adds the {@link CcSpecificLinkParamsProvider} to the providers created by this class. + * Otherwise the result will contain an instance of {@link CcLinkParamsProvider}. + */ + public CcLibraryHelper enableCcSpecificLinkParamsProvider() { + this.emitCcSpecificLinkParamsProvider = true; + return this; + } + + /** + * This disables C++ module map generation for the current rule. Don't call this unless you know + * what you are doing. + */ + public CcLibraryHelper disableCppModuleMapGeneration() { + this.emitCppModuleMaps = false; + return this; + } + + /** + * This enables or disables use of module maps during compilation, i.e., layering checks. + */ + public CcLibraryHelper setEnableLayeringCheck(boolean enableLayeringCheck) { + this.enableLayeringCheck = enableLayeringCheck; + return this; + } + + /** + * This enabled or disables compilation of C++ header modules. + * TODO(bazel-team): Add a cc_toolchain flag that allows fully disabling this feature and document + * this feature. + * See http://clang.llvm.org/docs/Modules.html. + */ + public CcLibraryHelper setCompileHeaderModules(boolean compileHeaderModules) { + this.compileHeaderModules = compileHeaderModules; + return this; + } + + /** + * Enables or disables generation of compile actions if there are no sources. Some rules declare a + * .a or .so implicit output, which requires that these files are created even if there are no + * source files, so be careful when calling this. + */ + public CcLibraryHelper setGenerateCompileActionsIfEmpty(boolean emitCompileActionsIfEmpty) { + this.emitCompileActionsIfEmpty = emitCompileActionsIfEmpty; + return this; + } + + /** + * Enables the optional generation of interface dynamic libraries - this is only used when the + * linker generates a dynamic library, and only if the crosstool supports it. The default is not + * to generate interface dynamic libraries. + */ + public CcLibraryHelper enableInterfaceSharedObjects() { + this.emitInterfaceSharedObjects = true; + return this; + } + + /** + * This enables or disables the generation of a dynamic library link action. The default is to + * generate a dynamic library. Note that the selection between dynamic or static linking is + * performed at the binary rule level. + */ + public CcLibraryHelper setCreateDynamicLibrary(boolean emitDynamicLibrary) { + this.emitDynamicLibrary = emitDynamicLibrary; + return this; + } + + /** + * Disables checking that the deps actually are C++ rules. By default, the {@link #build} method + * uses {@link LanguageDependentFragment.Checker#depSupportsLanguage} to check that all deps + * provide C++ providers. + */ + public CcLibraryHelper setCheckDepsGenerateCpp(boolean checkDepsGenerateCpp) { + this.checkDepsGenerateCpp = checkDepsGenerateCpp; + return this; + } + + /** + * Enables the output of {@link FilesToCompileProvider} and {@link + * CompilationPrerequisitesProvider}. + */ + // TODO(bazel-team): We probably need to adjust this for the multi-language rules. + public CcLibraryHelper enableCompileProviders() { + this.emitCompileProviders = true; + return this; + } + + /** + * Sets whether to emit the transitive module map references of a public library headers target. + */ + public CcLibraryHelper setEmitHeaderTargetModuleMaps(boolean emitHeaderTargetModuleMaps) { + this.emitHeaderTargetModuleMaps = emitHeaderTargetModuleMaps; + return this; + } + + /** + * Create the C++ compile and link actions, and the corresponding C++-related providers. + */ + public Info build() { + // Fail early if there is no lipo context collector on the rule - otherwise we end up failing + // in lipo optimization. + Preconditions.checkState( + // 'cc_inc_library' rules do not compile, and thus are not affected by LIPO. + ruleContext.getRule().getRuleClass().equals("cc_inc_library") + || ruleContext.getRule().isAttrDefined(":lipo_context_collector", Type.LABEL)); + + if (checkDepsGenerateCpp) { + for (LanguageDependentFragment dep : + AnalysisUtils.getProviders(deps, LanguageDependentFragment.class)) { + LanguageDependentFragment.Checker.depSupportsLanguage( + ruleContext, dep, CppRuleClasses.LANGUAGE); + } + } + + CcLinkingOutputs ccLinkingOutputs = CcLinkingOutputs.EMPTY; + CcCompilationOutputs ccOutputs = new CcCompilationOutputs.Builder().build(); + FeatureConfiguration featureConfiguration = CcCommon.configureFeatures(ruleContext); + + CppModel model = new CppModel(ruleContext, semantics) + .addSources(sources) + .addCopts(copts) + .setLinkTargetType(linkType) + .setNeverLink(neverlink) + .setFake(fake) + .setAllowInterfaceSharedObjects(emitInterfaceSharedObjects) + .setCreateDynamicLibrary(emitDynamicLibrary) + // Note: this doesn't actually save the temps, it just makes the CppModel use the + // configurations --save_temps setting to decide whether to actually save the temps. + .setSaveTemps(true) + .setEnableLayeringCheck(enableLayeringCheck) + .setCompileHeaderModules(compileHeaderModules) + .setNoCopts(nocopts) + .setDynamicLibraryPath(dynamicLibraryPath) + .addLinkopts(linkopts) + .setFeatureConfiguration(featureConfiguration); + CppCompilationContext cppCompilationContext = + initializeCppCompilationContext(model, featureConfiguration); + model.setContext(cppCompilationContext); + if (emitCompileActionsIfEmpty || !sources.isEmpty() || compileHeaderModules) { + Preconditions.checkState( + !compileHeaderModules || cppCompilationContext.getCppModuleMap() != null, + "All cc rules must support module maps."); + ccOutputs = model.createCcCompileActions(); + if (!objectFiles.isEmpty() || !picObjectFiles.isEmpty()) { + // Merge the pre-compiled object files into the compiler outputs. + ccOutputs = new CcCompilationOutputs.Builder() + .merge(ccOutputs) + .addObjectFiles(objectFiles) + .addPicObjectFiles(picObjectFiles) + .build(); + } + if (linkType.isStaticLibraryLink()) { + // TODO(bazel-team): This can't create the link action for a cc_binary yet. + ccLinkingOutputs = model.createCcLinkActions(ccOutputs); + } + } + CcLinkingOutputs originalLinkingOutputs = ccLinkingOutputs; + if (!( + staticLibraries.isEmpty() && picStaticLibraries.isEmpty() && dynamicLibraries.isEmpty())) { + // Merge the pre-compiled libraries (static & dynamic) into the linker outputs. + ccLinkingOutputs = new CcLinkingOutputs.Builder() + .merge(ccLinkingOutputs) + .addStaticLibraries(staticLibraries) + .addPicStaticLibraries(picStaticLibraries) + .addDynamicLibraries(dynamicLibraries) + .addExecutionDynamicLibraries(dynamicLibraries) + .build(); + } + + DwoArtifactsCollector dwoArtifacts = DwoArtifactsCollector.transitiveCollector(ccOutputs, deps); + Runfiles cppStaticRunfiles = collectCppRunfiles(ccLinkingOutputs, true); + Runfiles cppSharedRunfiles = collectCppRunfiles(ccLinkingOutputs, false); + + // By very careful when adding new providers here - it can potentially affect a lot of rules. + // We should consider merging most of these providers into a single provider. + Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers = + new LinkedHashMap<>(); + providers.put(CppRunfilesProvider.class, + new CppRunfilesProvider(cppStaticRunfiles, cppSharedRunfiles)); + providers.put(CppCompilationContext.class, cppCompilationContext); + providers.put(CppDebugFileProvider.class, new CppDebugFileProvider( + dwoArtifacts.getDwoArtifacts(), dwoArtifacts.getPicDwoArtifacts())); + providers.put(TransitiveLipoInfoProvider.class, collectTransitiveLipoInfo(ccOutputs)); + providers.put(TempsProvider.class, getTemps(ccOutputs)); + if (emitCompileProviders) { + providers.put(FilesToCompileProvider.class, new FilesToCompileProvider( + getFilesToCompile(ccOutputs))); + providers.put(CompilationPrerequisitesProvider.class, + CcCommon.collectCompilationPrerequisites(ruleContext, cppCompilationContext)); + } + + // TODO(bazel-team): Maybe we can infer these from other data at the places where they are + // used. + if (emitCcNativeLibrariesProvider) { + providers.put(CcNativeLibraryProvider.class, + new CcNativeLibraryProvider(collectNativeCcLibraries(ccLinkingOutputs))); + } + providers.put(CcExecutionDynamicLibrariesProvider.class, + collectExecutionDynamicLibraryArtifacts(ccLinkingOutputs.getExecutionDynamicLibraries())); + + boolean forcePic = ruleContext.getFragment(CppConfiguration.class).forcePic(); + if (emitCcSpecificLinkParamsProvider) { + providers.put(CcSpecificLinkParamsProvider.class, new CcSpecificLinkParamsProvider( + createCcLinkParamsStore(ccLinkingOutputs, cppCompilationContext, forcePic))); + } else { + providers.put(CcLinkParamsProvider.class, new CcLinkParamsProvider( + createCcLinkParamsStore(ccLinkingOutputs, cppCompilationContext, forcePic))); + } + return new Info(providers, ccOutputs, ccLinkingOutputs, originalLinkingOutputs, + cppCompilationContext); + } + + /** + * Create context for cc compile action from generated inputs. + */ + private CppCompilationContext initializeCppCompilationContext(CppModel model, + FeatureConfiguration featureConfiguration) { + CppCompilationContext.Builder contextBuilder = + new CppCompilationContext.Builder(ruleContext); + + // Setup the include path; local include directories come before those inherited from deps or + // from the toolchain; in case of aliasing (same include file found on different entries), + // prefer the local include rather than the inherited one. + + // Add in the roots for well-formed include names for source files and + // generated files. It is important that the execRoot (EMPTY_FRAGMENT) comes + // before the genfilesFragment to preferably pick up source files. Otherwise + // we might pick up stale generated files. + contextBuilder.addQuoteIncludeDir(PathFragment.EMPTY_FRAGMENT); + contextBuilder.addQuoteIncludeDir(ruleContext.getConfiguration().getGenfilesFragment()); + + for (PathFragment systemIncludeDir : systemIncludeDirs) { + contextBuilder.addSystemIncludeDir(systemIncludeDir); + } + for (PathFragment includeDir : includeDirs) { + contextBuilder.addIncludeDir(includeDir); + } + + contextBuilder.mergeDependentContexts( + AnalysisUtils.getProviders(deps, CppCompilationContext.class)); + CppHelper.mergeToolchainDependentContext(ruleContext, contextBuilder); + + // But defines come after those inherited from deps. + contextBuilder.addDefines(defines); + + // There are no ordering constraints for declared include dirs/srcs, or the pregrepped headers. + contextBuilder.addDeclaredIncludeSrcs(publicHeaders); + contextBuilder.addDeclaredIncludeSrcs(privateHeaders); + contextBuilder.addPregreppedHeaderMap( + CppHelper.createExtractInclusions(ruleContext, publicHeaders)); + contextBuilder.addPregreppedHeaderMap( + CppHelper.createExtractInclusions(ruleContext, privateHeaders)); + contextBuilder.addCompilationPrerequisites(prerequisites); + + // Add this package's dir to declaredIncludeDirs, & this rule's headers to declaredIncludeSrcs + // Note: no include dir for STRICT mode. + if (headersCheckingMode == HeadersCheckingMode.WARN) { + contextBuilder.addDeclaredIncludeWarnDir(ruleContext.getLabel().getPackageFragment()); + for (PathFragment looseIncludeDir : looseIncludeDirs) { + contextBuilder.addDeclaredIncludeWarnDir(looseIncludeDir); + } + } else if (headersCheckingMode == HeadersCheckingMode.LOOSE) { + contextBuilder.addDeclaredIncludeDir(ruleContext.getLabel().getPackageFragment()); + for (PathFragment looseIncludeDir : looseIncludeDirs) { + contextBuilder.addDeclaredIncludeDir(looseIncludeDir); + } + } + + if (emitCppModuleMaps) { + CppModuleMap cppModuleMap = CppHelper.addCppModuleMapToContext(ruleContext, contextBuilder); + // TODO(bazel-team): addCppModuleMapToContext second-guesses whether module maps should + // actually be enabled, so we need to double-check here. Who would write code like this? + if (cppModuleMap != null) { + CppModuleMapAction action = new CppModuleMapAction(ruleContext.getActionOwner(), + cppModuleMap, + privateHeaders, + publicHeaders, + collectModuleMaps(), + additionalExportedHeaders, + compileHeaderModules, + featureConfiguration.isEnabled(CppRuleClasses.MODULE_MAP_HOME_CWD)); + ruleContext.registerAction(action); + } + if (model.getGeneratesPicHeaderModule()) { + contextBuilder.setPicHeaderModule(model.getPicHeaderModule(cppModuleMap.getArtifact())); + } + if (model.getGeratesNoPicHeaderModule()) { + contextBuilder.setHeaderModule(model.getHeaderModule(cppModuleMap.getArtifact())); + } + } + + semantics.setupCompilationContext(ruleContext, contextBuilder); + return contextBuilder.build(); + } + + private Iterable<CppModuleMap> collectModuleMaps() { + // Cpp module maps may be null for some rules. We filter the nulls out at the end. + List<CppModuleMap> result = new ArrayList<>(); + Iterables.addAll(result, Iterables.transform(deps, CPP_DEPS_TO_MODULES)); + CppCompilationContext stl = + ruleContext.getPrerequisite(":stl", Mode.TARGET, CppCompilationContext.class); + if (stl != null) { + result.add(stl.getCppModuleMap()); + } + + CcToolchainProvider toolchain = CppHelper.getToolchain(ruleContext); + if (toolchain != null) { + result.add(toolchain.getCppCompilationContext().getCppModuleMap()); + } + + if (emitHeaderTargetModuleMaps) { + for (HeaderTargetModuleMapProvider provider : AnalysisUtils.getProviders( + deps, HeaderTargetModuleMapProvider.class)) { + result.addAll(provider.getCppModuleMaps()); + } + } + + return Iterables.filter(result, Predicates.<CppModuleMap>notNull()); + } + + private TransitiveLipoInfoProvider collectTransitiveLipoInfo(CcCompilationOutputs outputs) { + if (ruleContext.getFragment(CppConfiguration.class).getFdoSupport().getFdoRoot() == null) { + return TransitiveLipoInfoProvider.EMPTY; + } + NestedSetBuilder<IncludeScannable> scannableBuilder = NestedSetBuilder.stableOrder(); + // TODO(bazel-team): Only fetch the STL prerequisite in one place. + TransitiveInfoCollection stl = ruleContext.getPrerequisite(":stl", Mode.TARGET); + if (stl != null) { + TransitiveLipoInfoProvider provider = stl.getProvider(TransitiveLipoInfoProvider.class); + if (provider != null) { + scannableBuilder.addTransitive(provider.getTransitiveIncludeScannables()); + } + } + + for (TransitiveLipoInfoProvider dep : + AnalysisUtils.getProviders(deps, TransitiveLipoInfoProvider.class)) { + scannableBuilder.addTransitive(dep.getTransitiveIncludeScannables()); + } + + for (IncludeScannable scannable : outputs.getLipoScannables()) { + Preconditions.checkState(scannable.getIncludeScannerSources().size() == 1); + scannableBuilder.add(scannable); + } + return new TransitiveLipoInfoProvider(scannableBuilder.build()); + } + + private Runfiles collectCppRunfiles( + CcLinkingOutputs ccLinkingOutputs, boolean linkingStatically) { + Runfiles.Builder builder = new Runfiles.Builder(); + builder.addTargets(deps, RunfilesProvider.DEFAULT_RUNFILES); + builder.addTargets(deps, CppRunfilesProvider.runfilesFunction(linkingStatically)); + // Add the shared libraries to the runfiles. + builder.addArtifacts(ccLinkingOutputs.getLibrariesForRunfiles(linkingStatically)); + return builder.build(); + } + + private CcLinkParamsStore createCcLinkParamsStore( + final CcLinkingOutputs ccLinkingOutputs, final CppCompilationContext cppCompilationContext, + final boolean forcePic) { + return new CcLinkParamsStore() { + @Override + protected void collect(CcLinkParams.Builder builder, boolean linkingStatically, + boolean linkShared) { + builder.addLinkstamps(linkstamps, cppCompilationContext); + builder.addTransitiveTargets(deps, + CcLinkParamsProvider.TO_LINK_PARAMS, CcSpecificLinkParamsProvider.TO_LINK_PARAMS); + if (!neverlink) { + builder.addLibraries(ccLinkingOutputs.getPreferredLibraries(linkingStatically, + /*preferPic=*/linkShared || forcePic)); + builder.addLinkOpts(linkopts); + } + } + }; + } + + private NestedSet<LinkerInput> collectNativeCcLibraries(CcLinkingOutputs ccLinkingOutputs) { + NestedSetBuilder<LinkerInput> result = NestedSetBuilder.linkOrder(); + result.addAll(ccLinkingOutputs.getDynamicLibraries()); + for (CcNativeLibraryProvider dep : AnalysisUtils.getProviders( + deps, CcNativeLibraryProvider.class)) { + result.addTransitive(dep.getTransitiveCcNativeLibraries()); + } + + return result.build(); + } + + private CcExecutionDynamicLibrariesProvider collectExecutionDynamicLibraryArtifacts( + List<LibraryToLink> executionDynamicLibraries) { + Iterable<Artifact> artifacts = LinkerInputs.toLibraryArtifacts(executionDynamicLibraries); + if (!Iterables.isEmpty(artifacts)) { + return new CcExecutionDynamicLibrariesProvider( + NestedSetBuilder.wrap(Order.STABLE_ORDER, artifacts)); + } + + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (CcExecutionDynamicLibrariesProvider dep : + AnalysisUtils.getProviders(deps, CcExecutionDynamicLibrariesProvider.class)) { + builder.addTransitive(dep.getExecutionDynamicLibraryArtifacts()); + } + return builder.isEmpty() + ? CcExecutionDynamicLibrariesProvider.EMPTY + : new CcExecutionDynamicLibrariesProvider(builder.build()); + } + + private TempsProvider getTemps(CcCompilationOutputs compilationOutputs) { + return ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector() + ? new TempsProvider(ImmutableList.<Artifact>of()) + : new TempsProvider(compilationOutputs.getTemps()); + } + + private ImmutableList<Artifact> getFilesToCompile(CcCompilationOutputs compilationOutputs) { + return ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector() + ? ImmutableList.<Artifact>of() + : compilationOutputs.getObjectFiles(CppHelper.usePic(ruleContext, false)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParams.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParams.java new file mode 100644 index 0000000..4e4804b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParams.java
@@ -0,0 +1,357 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; + +import java.util.Collection; +import java.util.Objects; + +/** + * Parameters to be passed to the linker. + * + * <p>The parameters concerned are the link options (strings) passed to the linker, linkstamps and a + * list of libraries to be linked in. + * + * <p>Items in the collections are stored in nested sets. Link options and libraries are stored in + * link order (preorder) and linkstamps are sorted. + */ +public final class CcLinkParams { + private final NestedSet<ImmutableList<String>> linkOpts; + private final NestedSet<Linkstamp> linkstamps; + private final NestedSet<LibraryToLink> libraries; + + private CcLinkParams(NestedSet<ImmutableList<String>> linkOpts, + NestedSet<Linkstamp> linkstamps, + NestedSet<LibraryToLink> libraries) { + this.linkOpts = linkOpts; + this.linkstamps = linkstamps; + this.libraries = libraries; + } + + /** + * @return the linkopts + */ + public NestedSet<ImmutableList<String>> getLinkopts() { + return linkOpts; + } + + public ImmutableList<String> flattenedLinkopts() { + return ImmutableList.copyOf(Iterables.concat(linkOpts)); + } + + /** + * @return the linkstamps + */ + public NestedSet<Linkstamp> getLinkstamps() { + return linkstamps; + } + + /** + * @return the libraries + */ + public NestedSet<LibraryToLink> getLibraries() { + return libraries; + } + + public static final Builder builder(boolean linkingStatically, boolean linkShared) { + return new Builder(linkingStatically, linkShared); + } + + /** + * Builder for {@link CcLinkParams}. + * + * + */ + public static final class Builder { + + /** + * linkingStatically is true when we're linking this target in either FULLY STATIC mode + * (linkopts=["-static"]) or MOSTLY STATIC mode (linkstatic=1). When this is true, we want to + * use static versions of any libraries that this target depends on (except possibly system + * libraries, which are not handled by CcLinkParams). When this is false, we want to use dynamic + * versions of any libraries that this target depends on. + */ + private final boolean linkingStatically; + + /** + * linkShared is true when we're linking with "-shared" (linkshared=1). + */ + private final boolean linkShared; + + private ImmutableList.Builder<String> localLinkoptsBuilder = ImmutableList.builder(); + + private final NestedSetBuilder<ImmutableList<String>> linkOptsBuilder = + NestedSetBuilder.linkOrder(); + private final NestedSetBuilder<Linkstamp> linkstampsBuilder = + NestedSetBuilder.compileOrder(); + private final NestedSetBuilder<LibraryToLink> librariesBuilder = + NestedSetBuilder.linkOrder(); + + private boolean built = false; + + private Builder(boolean linkingStatically, boolean linkShared) { + this.linkingStatically = linkingStatically; + this.linkShared = linkShared; + } + + /** + * Build a {@link CcLinkParams} object. + */ + public CcLinkParams build() { + Preconditions.checkState(!built); + // Not thread-safe, but builders should not be shared across threads. + built = true; + ImmutableList<String> localLinkopts = localLinkoptsBuilder.build(); + if (!localLinkopts.isEmpty()) { + linkOptsBuilder.add(localLinkopts); + } + return new CcLinkParams(linkOptsBuilder.build(), linkstampsBuilder.build(), + librariesBuilder.build()); + } + + private boolean add(CcLinkParamsStore store) { + if (store != null) { + CcLinkParams args = store.get(linkingStatically, linkShared); + addTransitiveArgs(args); + } + return store != null; + } + + /** + * Includes link parameters from a collection of dependency targets. + */ + public Builder addTransitiveTargets(Iterable<? extends TransitiveInfoCollection> targets) { + for (TransitiveInfoCollection target : targets) { + addTransitiveTarget(target); + } + return this; + } + + /** + * Includes link parameters from a dependency target. + * + * <p>The target should implement {@link CcLinkParamsProvider}. If it does not, + * the method does not do anything. + */ + public Builder addTransitiveTarget(TransitiveInfoCollection target) { + return addTransitiveProvider(target.getProvider(CcLinkParamsProvider.class)); + } + + /** + * Includes link parameters from a dependency target. The target is checked for the given + * mappings in the order specified, and the first mapping that returns a non-null result is + * added. + */ + @SafeVarargs + public final Builder addTransitiveTarget(TransitiveInfoCollection target, + Function<TransitiveInfoCollection, CcLinkParamsStore> firstMapping, + @SuppressWarnings("unchecked") // Java arrays don't preserve generic arguments. + Function<TransitiveInfoCollection, CcLinkParamsStore>... remainingMappings) { + if (add(firstMapping.apply(target))) { + return this; + } + for (Function<TransitiveInfoCollection, CcLinkParamsStore> mapping : remainingMappings) { + if (add(mapping.apply(target))) { + return this; + } + } + return this; + } + + /** + * Includes link parameters from a CcLinkParamsProvider provider. + */ + public Builder addTransitiveProvider(CcLinkParamsProvider provider) { + if (provider == null) { + return this; + } + + CcLinkParams args = provider.getCcLinkParams(linkingStatically, linkShared); + addTransitiveArgs(args); + return this; + } + + /** + * Includes link parameters from the given targets. Each target is checked for the given + * mappings in the order specified, and the first mapping that returns a non-null result is + * added. + */ + @SafeVarargs + public final Builder addTransitiveTargets( + Iterable<? extends TransitiveInfoCollection> targets, + Function<TransitiveInfoCollection, CcLinkParamsStore> firstMapping, + @SuppressWarnings("unchecked") // Java arrays don't preserve generic arguments. + Function<TransitiveInfoCollection, CcLinkParamsStore>... remainingMappings) { + for (TransitiveInfoCollection target : targets) { + addTransitiveTarget(target, firstMapping, remainingMappings); + } + return this; + } + + /** + * Includes link parameters from the given targets. Each target is checked for the given + * mappings in the order specified, and the first mapping that returns a non-null result is + * added. + * + * @deprecated don't add any new uses; all existing uses need to be audited and possibly merged + * into a single call - some of them may introduce semantic changes which need to be + * carefully vetted + */ + @Deprecated + @SafeVarargs + public final Builder addTransitiveLangTargets( + Iterable<? extends TransitiveInfoCollection> targets, + Function<TransitiveInfoCollection, CcLinkParamsStore> firstMapping, + @SuppressWarnings("unchecked") // Java arrays don't preserve generic arguments. + Function<TransitiveInfoCollection, CcLinkParamsStore>... remainingMappings) { + return addTransitiveTargets(targets, firstMapping, remainingMappings); + } + + /** + * Merges the other {@link CcLinkParams} object into this one. + */ + public Builder addTransitiveArgs(CcLinkParams args) { + linkOptsBuilder.addTransitive(args.getLinkopts()); + linkstampsBuilder.addTransitive(args.getLinkstamps()); + librariesBuilder.addTransitive(args.getLibraries()); + return this; + } + + /** + * Adds a collection of link options. + */ + public Builder addLinkOpts(Collection<String> linkOpts) { + localLinkoptsBuilder.addAll(linkOpts); + return this; + } + + /** + * Adds a collection of linkstamps. + */ + public Builder addLinkstamps(Iterable<Artifact> linkstamps, CppCompilationContext context) { + ImmutableList<Artifact> declaredIncludeSrcs = + ImmutableList.copyOf(context.getDeclaredIncludeSrcs()); + for (Artifact linkstamp : linkstamps) { + linkstampsBuilder.add(new Linkstamp(linkstamp, declaredIncludeSrcs)); + } + return this; + } + + /** + * Adds a library artifact. + */ + public Builder addLibrary(LibraryToLink library) { + librariesBuilder.add(library); + return this; + } + + /** + * Adds a collection of library artifacts. + */ + public Builder addLibraries(Iterable<LibraryToLink> libraries) { + librariesBuilder.addAll(libraries); + return this; + } + + /** + * Processes typical dependencies a C/C++ library. + * + * <p>A helper method that processes getValues() and merges contents of + * getPreferredLibraries() and getLinkOpts() into the current link params + * object. + */ + public Builder addCcLibrary(RuleContext context, CcCommon common, boolean neverlink, + CcLinkingOutputs linkingOutputs) { + addTransitiveTargets( + context.getPrerequisites("deps", Mode.TARGET), + CcLinkParamsProvider.TO_LINK_PARAMS, CcSpecificLinkParamsProvider.TO_LINK_PARAMS); + + if (!neverlink) { + addLibraries(linkingOutputs.getPreferredLibraries(linkingStatically, + linkShared || context.getFragment(CppConfiguration.class).forcePic())); + addLinkOpts(common.getLinkopts()); + } + return this; + } + } + + /** + * A linkstamp that also knows about its declared includes. + * + * <p>This object is required because linkstamp files may include other headers which + * will have to be provided during compilation. + */ + public static final class Linkstamp { + private final Artifact artifact; + private final ImmutableList<Artifact> declaredIncludeSrcs; + + private Linkstamp(Artifact artifact, ImmutableList<Artifact> declaredIncludeSrcs) { + this.artifact = Preconditions.checkNotNull(artifact); + this.declaredIncludeSrcs = Preconditions.checkNotNull(declaredIncludeSrcs); + } + + /** + * Returns the linkstamp artifact. + */ + public Artifact getArtifact() { + return artifact; + } + + /** + * Returns the declared includes. + */ + public ImmutableList<Artifact> getDeclaredIncludeSrcs() { + return declaredIncludeSrcs; + } + + @Override + public int hashCode() { + return Objects.hash(artifact, declaredIncludeSrcs); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof Linkstamp)) { + return false; + } + Linkstamp other = (Linkstamp) obj; + return artifact.equals(other.artifact) + && declaredIncludeSrcs.equals(other.declaredIncludeSrcs); + } + } + + /** + * Empty CcLinkParams. + */ + public static final CcLinkParams EMPTY = new CcLinkParams( + NestedSetBuilder.<ImmutableList<String>>emptySet(Order.LINK_ORDER), + NestedSetBuilder.<Linkstamp>emptySet(Order.COMPILE_ORDER), + NestedSetBuilder.<LibraryToLink>emptySet(Order.LINK_ORDER)); +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsProvider.java new file mode 100644 index 0000000..11f6011 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsProvider.java
@@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore.CcLinkParamsStoreImpl; + +/** + * A target that provides C linker parameters. + */ +@Immutable +public final class CcLinkParamsProvider implements TransitiveInfoProvider { + public static final Function<TransitiveInfoCollection, CcLinkParamsStore> TO_LINK_PARAMS = + new Function<TransitiveInfoCollection, CcLinkParamsStore>() { + @Override + public CcLinkParamsStore apply(TransitiveInfoCollection input) { + CcLinkParamsProvider provider = input.getProvider( + CcLinkParamsProvider.class); + return provider == null ? null : provider.store; + } + }; + + private final CcLinkParamsStoreImpl store; + + public CcLinkParamsProvider(CcLinkParamsStore store) { + this.store = new CcLinkParamsStoreImpl(store); + } + + /** + * Returns link parameters given static / shared linking settings. + */ + public CcLinkParams getCcLinkParams(boolean linkingStatically, boolean linkShared) { + return store.get(linkingStatically, linkShared); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsStore.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsStore.java new file mode 100644 index 0000000..a150488 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsStore.java
@@ -0,0 +1,136 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.rules.cpp.CcLinkParams.Builder; + +/** + * A cache of C link parameters. + * + * <p>The cache holds instances of {@link CcLinkParams} for combinations of + * linkingStatically and linkShared. If a requested value is not available in + * the cache, it is computed and then stored. + * + * <p>Typically this class is used on targets that may be linked in as C + * libraries as in the following example: + * + * <pre> + * class SomeTarget implements CcLinkParamsProvider { + * private final CcLinkParamsStore ccLinkParamsStore = new CcLinkParamsStore() { + * @Override + * protected void collect(CcLinkParams.Builder builder, boolean linkingStatically, + * boolean linkShared) { + * builder.add[...] + * } + * }; + * + * @Override + * public CcLinkParams getCcLinkParams(boolean linkingStatically, boolean linkShared) { + * return ccLinkParamsStore.get(linkingStatically, linkShared); + * } + * } + * </pre> + */ +public abstract class CcLinkParamsStore { + + private CcLinkParams staticSharedParams; + private CcLinkParams staticNoSharedParams; + private CcLinkParams noStaticSharedParams; + private CcLinkParams noStaticNoSharedParams; + + private CcLinkParams compute(boolean linkingStatically, boolean linkShared) { + CcLinkParams.Builder builder = CcLinkParams.builder(linkingStatically, linkShared); + collect(builder, linkingStatically, linkShared); + return builder.build(); + } + + /** + * Returns {@link CcLinkParams} for a combination of parameters. + * + * <p>The {@link CcLinkParams} instance is computed lazily and cached. + */ + public synchronized CcLinkParams get(boolean linkingStatically, boolean linkShared) { + CcLinkParams result = lookup(linkingStatically, linkShared); + if (result == null) { + result = compute(linkingStatically, linkShared); + put(linkingStatically, linkShared, result); + } + return result; + } + + private CcLinkParams lookup(boolean linkingStatically, boolean linkShared) { + if (linkingStatically) { + return linkShared ? staticSharedParams : staticNoSharedParams; + } else { + return linkShared ? noStaticSharedParams : noStaticNoSharedParams; + } + } + + private void put(boolean linkingStatically, boolean linkShared, CcLinkParams params) { + Preconditions.checkNotNull(params); + if (linkingStatically) { + if (linkShared) { + staticSharedParams = params; + } else { + staticNoSharedParams = params; + } + } else { + if (linkShared) { + noStaticSharedParams = params; + } else { + noStaticNoSharedParams = params; + } + } + } + + /** + * Hook for building the actual link params. + * + * <p>Users should override this method and call methods of the builder to + * set up the actual CcLinkParams objects. + * + * <p>Implementations of this method must not fail or try to report errors on the + * configured target. + */ + protected abstract void collect(CcLinkParams.Builder builder, boolean linkingStatically, + boolean linkShared); + + /** + * An empty CcLinkParamStore. + */ + public static final CcLinkParamsStore EMPTY = new CcLinkParamsStore() { + + @Override + protected void collect(Builder builder, boolean linkingStatically, boolean linkShared) {} + }; + + /** + * An implementation class for the CcLinkParamsStore. + */ + public static final class CcLinkParamsStoreImpl extends CcLinkParamsStore { + + public CcLinkParamsStoreImpl(CcLinkParamsStore store) { + super.staticSharedParams = store.get(true, true); + super.staticNoSharedParams = store.get(true, false); + super.noStaticSharedParams = store.get(false, true); + super.noStaticNoSharedParams = store.get(false, false); + } + + @Override + protected void collect(Builder builder, boolean linkingStatically, boolean linkShared) {} + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkingOutputs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkingOutputs.java new file mode 100644 index 0000000..6b45c79 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkingOutputs.java
@@ -0,0 +1,243 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.vfs.FileSystemUtils; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * A structured representation of the link outputs of a C++ rule. + */ +public class CcLinkingOutputs { + + public static final CcLinkingOutputs EMPTY = new Builder().build(); + + private final ImmutableList<LibraryToLink> staticLibraries; + + private final ImmutableList<LibraryToLink> picStaticLibraries; + + private final ImmutableList<LibraryToLink> dynamicLibraries; + + private final ImmutableList<LibraryToLink> executionDynamicLibraries; + + private CcLinkingOutputs(ImmutableList<LibraryToLink> staticLibraries, + ImmutableList<LibraryToLink> picStaticLibraries, + ImmutableList<LibraryToLink> dynamicLibraries, + ImmutableList<LibraryToLink> executionDynamicLibraries) { + this.staticLibraries = staticLibraries; + this.picStaticLibraries = picStaticLibraries; + this.dynamicLibraries = dynamicLibraries; + this.executionDynamicLibraries = executionDynamicLibraries; + } + + public ImmutableList<LibraryToLink> getStaticLibraries() { + return staticLibraries; + } + + public ImmutableList<LibraryToLink> getPicStaticLibraries() { + return picStaticLibraries; + } + + public ImmutableList<LibraryToLink> getDynamicLibraries() { + return dynamicLibraries; + } + + public ImmutableList<LibraryToLink> getExecutionDynamicLibraries() { + return executionDynamicLibraries; + } + + /** + * Add the ".a", ".pic.a" and/or ".so" files in appropriate order of preference depending on the + * link preferences. + * + * <p>This method tries to simulate a search path for adding static and dynamic libraries, + * allowing either to be preferred over the other depending on the link {@link LinkStaticness}. + * + * TODO(bazel-team): (2009) we should preserve the relative ordering of first and second + * choice libraries. E.g. if srcs=['foo.a','bar.so','baz.a'] then we should link them in the + * same order. Currently we link entries from the first choice list before those from the + * second choice list, i.e. in the order {@code ['bar.so', 'foo.a', 'baz.a']}. + * + * @param linkingStatically whether to prefer static over dynamic libraries. Should be + * <code>true</code> for binaries that are linked in fully static or mostly static mode. + * @param preferPic whether to prefer pic over non pic libraries (usually used when linking + * shared) + */ + public List<LibraryToLink> getPreferredLibraries( + boolean linkingStatically, boolean preferPic) { + return getPreferredLibraries(linkingStatically, preferPic, false); + } + + /** + * Returns the shared libraries that are linked against and therefore also need to be in the + * runfiles. + */ + public Iterable<Artifact> getLibrariesForRunfiles(boolean linkingStatically) { + List<LibraryToLink> libraries = + getPreferredLibraries(linkingStatically, /*preferPic*/false, true); + return CcCommon.getSharedLibrariesFrom(LinkerInputs.toLibraryArtifacts(libraries)); + } + + /** + * Add the ".a", ".pic.a" and/or ".so" files in appropriate order of + * preference depending on the link preferences. + */ + private List<LibraryToLink> getPreferredLibraries(boolean linkingStatically, boolean preferPic, + boolean forRunfiles) { + List<LibraryToLink> candidates = new ArrayList<>(); + // It's important that this code keeps the invariant that preferPic has no effect on the output + // of .so libraries. That is, the resulting list should contain the same .so files in the same + // order. + if (linkingStatically) { // Prefer the static libraries. + if (preferPic) { + // First choice is the PIC static libraries. + // Second choice is the other static libraries (may cause link error if they're not PIC, + // but I think this is preferable to linking dynamically when you asked for statically). + candidates.addAll(picStaticLibraries); + candidates.addAll(staticLibraries); + } else { + // First choice is the non-pic static libraries (best performance); + // second choice is the staticPicLibraries (at least they're static; + // we can live with the extra overhead of PIC). + candidates.addAll(staticLibraries); + candidates.addAll(picStaticLibraries); + } + candidates.addAll(forRunfiles ? executionDynamicLibraries : dynamicLibraries); + } else { + // First choice is the dynamicLibraries. + candidates.addAll(forRunfiles ? executionDynamicLibraries : dynamicLibraries); + if (preferPic) { + // Second choice is the staticPicLibraries (at least they're PIC, so we won't get a + // link error). + candidates.addAll(picStaticLibraries); + candidates.addAll(staticLibraries); + } else { + candidates.addAll(staticLibraries); + candidates.addAll(picStaticLibraries); + } + } + return filterCandidates(candidates); + } + + /** + * Helper method to filter the candidates by removing equivalent library + * entries from the list of candidates. + * + * @param candidates the library candidates to filter + * @return the list of libraries with equivalent duplicate libraries removed. + */ + private List<LibraryToLink> filterCandidates(List<LibraryToLink> candidates) { + List<LibraryToLink> libraries = new ArrayList<>(); + Set<String> identifiers = new HashSet<>(); + for (LibraryToLink library : candidates) { + if (identifiers.add(libraryIdentifierOf(library.getOriginalLibraryArtifact()))) { + libraries.add(library); + } + } + return libraries; + } + + /** + * Returns the library identifier of an artifact: a string that is different for different + * libraries, but is the same for the shared, static and pic versions of the same library. + */ + private static String libraryIdentifierOf(Artifact libraryArtifact) { + String name = libraryArtifact.getRootRelativePath().getPathString(); + String basename = FileSystemUtils.removeExtension(name); + // Need to special-case file types with double extension. + return name.endsWith(".pic.a") + ? FileSystemUtils.removeExtension(basename) + : name.endsWith(".nopic.a") + ? FileSystemUtils.removeExtension(basename) + : name.endsWith(".pic.lo") + ? FileSystemUtils.removeExtension(basename) + : basename; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private final Set<LibraryToLink> staticLibraries = new LinkedHashSet<>(); + private final Set<LibraryToLink> picStaticLibraries = new LinkedHashSet<>(); + private final Set<LibraryToLink> dynamicLibraries = new LinkedHashSet<>(); + private final Set<LibraryToLink> executionDynamicLibraries = new LinkedHashSet<>(); + + public CcLinkingOutputs build() { + return new CcLinkingOutputs(ImmutableList.copyOf(staticLibraries), + ImmutableList.copyOf(picStaticLibraries), ImmutableList.copyOf(dynamicLibraries), + ImmutableList.copyOf(executionDynamicLibraries)); + } + + public Builder merge(CcLinkingOutputs outputs) { + staticLibraries.addAll(outputs.getStaticLibraries()); + picStaticLibraries.addAll(outputs.getPicStaticLibraries()); + dynamicLibraries.addAll(outputs.getDynamicLibraries()); + executionDynamicLibraries.addAll(outputs.getExecutionDynamicLibraries()); + return this; + } + + public Builder addStaticLibrary(LibraryToLink library) { + staticLibraries.add(library); + return this; + } + + public Builder addStaticLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(staticLibraries, libraries); + return this; + } + + public Builder addPicStaticLibrary(LibraryToLink library) { + picStaticLibraries.add(library); + return this; + } + + public Builder addPicStaticLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(picStaticLibraries, libraries); + return this; + } + + public Builder addDynamicLibrary(LibraryToLink library) { + dynamicLibraries.add(library); + return this; + } + + public Builder addDynamicLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(dynamicLibraries, libraries); + return this; + } + + public Builder addExecutionDynamicLibrary(LibraryToLink library) { + executionDynamicLibraries.add(library); + return this; + } + + public Builder addExecutionDynamicLibraries(Iterable<LibraryToLink> libraries) { + Iterables.addAll(executionDynamicLibraries, libraries); + return this; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcNativeLibraryProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcNativeLibraryProvider.java new file mode 100644 index 0000000..5e96291 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcNativeLibraryProvider.java
@@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A target that provides native libraries in the transitive closure of its deps that are needed for + * executing C++ code. + */ +@Immutable +public final class CcNativeLibraryProvider implements TransitiveInfoProvider { + + private final NestedSet<LinkerInput> transitiveCcNativeLibraries; + + public CcNativeLibraryProvider(NestedSet<LinkerInput> transitiveCcNativeLibraries) { + this.transitiveCcNativeLibraries = transitiveCcNativeLibraries; + } + + /** + * Collects native libraries in the transitive closure of its deps that are needed for executing + * C/C++ code. + * + * <p>In effect, returns all dynamic library (.so) artifacts provided by the transitive closure. + */ + public NestedSet<LinkerInput> getTransitiveCcNativeLibraries() { + return transitiveCcNativeLibraries; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcSpecificLinkParamsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcSpecificLinkParamsProvider.java new file mode 100644 index 0000000..dfcecc2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcSpecificLinkParamsProvider.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore.CcLinkParamsStoreImpl; + +/** + * A target that provides libraries to be only linked into other C++ targets (and not targets + * for other languages) + */ +@Immutable +public final class CcSpecificLinkParamsProvider implements TransitiveInfoProvider { + private final CcLinkParamsStoreImpl store; + + public CcSpecificLinkParamsProvider(CcLinkParamsStore store) { + this.store = new CcLinkParamsStoreImpl(store); + } + + public CcLinkParamsStore getLinkParams() { + return store; + } + + public static final Function<TransitiveInfoCollection, CcLinkParamsStore> TO_LINK_PARAMS = + new Function<TransitiveInfoCollection, CcLinkParamsStore>() { + @Override + public CcLinkParamsStore apply(TransitiveInfoCollection input) { + CcSpecificLinkParamsProvider provider = input.getProvider( + CcSpecificLinkParamsProvider.class); + return provider == null ? null : provider.getLinkParams(); + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcTest.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcTest.java new file mode 100644 index 0000000..7827183 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcTest.java
@@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * A configured target class for cc_test rules. + */ +public abstract class CcTest implements RuleConfiguredTargetFactory { + + private final CppSemantics semantics; + + protected CcTest(CppSemantics semantics) { + this.semantics = semantics; + } + + @Override + public ConfiguredTarget create(RuleContext context) throws InterruptedException { + return CcBinary.init(semantics, context, /*fake =*/ false, /*useTestOnlyFlags =*/ true); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java new file mode 100644 index 0000000..bd39d0f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java
@@ -0,0 +1,249 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Actions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.CompilationHelper; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.LicensesProvider; +import com.google.devtools.build.lib.analysis.LicensesProvider.TargetLicense; +import com.google.devtools.build.lib.analysis.MiddlemanProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.License; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.List; + +/** + * Implementation for the cc_toolchain rule. + */ +public class CcToolchain implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + final Label label = ruleContext.getLabel(); + final NestedSet<Artifact> crosstool = ruleContext.getPrerequisite("all_files", Mode.HOST) + .getProvider(FileProvider.class).getFilesToBuild(); + final NestedSet<Artifact> crosstoolMiddleman = getFiles(ruleContext, "all_files"); + final NestedSet<Artifact> compile = getFiles(ruleContext, "compiler_files"); + final NestedSet<Artifact> strip = getFiles(ruleContext, "strip_files"); + final NestedSet<Artifact> objcopy = getFiles(ruleContext, "objcopy_files"); + final NestedSet<Artifact> link = getFiles(ruleContext, "linker_files"); + final NestedSet<Artifact> dwp = getFiles(ruleContext, "dwp_files"); + final NestedSet<Artifact> libcLink = inputsForLibcLink(ruleContext); + String purposePrefix = Actions.escapeLabel(label) + "_"; + String runtimeSolibDirBase = "_solib_" + "_" + Actions.escapeLabel(label); + final PathFragment runtimeSolibDir = ruleContext.getConfiguration() + .getBinFragment().getRelative(runtimeSolibDirBase); + + CppConfiguration cppConfiguration = ruleContext.getFragment(CppConfiguration.class); + // Static runtime inputs. + TransitiveInfoCollection staticRuntimeLibDep = selectDep(ruleContext, "static_runtime_libs", + cppConfiguration.getStaticRuntimeLibsLabel()); + final NestedSet<Artifact> staticRuntimeLinkInputs; + final Artifact staticRuntimeLinkMiddleman; + if (cppConfiguration.supportsEmbeddedRuntimes()) { + staticRuntimeLinkInputs = staticRuntimeLibDep + .getProvider(FileProvider.class) + .getFilesToBuild(); + } else { + staticRuntimeLinkInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (!staticRuntimeLinkInputs.isEmpty()) { + NestedSet<Artifact> staticRuntimeLinkMiddlemanSet = CompilationHelper.getAggregatingMiddleman( + ruleContext, + purposePrefix + "static_runtime_link", + staticRuntimeLibDep); + staticRuntimeLinkMiddleman = staticRuntimeLinkMiddlemanSet.isEmpty() + ? null : Iterables.getOnlyElement(staticRuntimeLinkMiddlemanSet); + } else { + staticRuntimeLinkMiddleman = null; + } + + Preconditions.checkState( + (staticRuntimeLinkMiddleman == null) == staticRuntimeLinkInputs.isEmpty()); + + // Dynamic runtime inputs. + TransitiveInfoCollection dynamicRuntimeLibDep = selectDep(ruleContext, "dynamic_runtime_libs", + cppConfiguration.getDynamicRuntimeLibsLabel()); + final NestedSet<Artifact> dynamicRuntimeLinkInputs; + final Artifact dynamicRuntimeLinkMiddleman; + if (cppConfiguration.supportsEmbeddedRuntimes()) { + NestedSetBuilder<Artifact> dynamicRuntimeLinkInputsBuilder = NestedSetBuilder.stableOrder(); + for (Artifact artifact : dynamicRuntimeLibDep + .getProvider(FileProvider.class).getFilesToBuild()) { + if (CppHelper.SHARED_LIBRARY_FILETYPES.matches(artifact.getFilename())) { + dynamicRuntimeLinkInputsBuilder.add(SolibSymlinkAction.getCppRuntimeSymlink( + ruleContext, artifact, runtimeSolibDirBase, + ruleContext.getConfiguration()).getArtifact()); + } else { + dynamicRuntimeLinkInputsBuilder.add(artifact); + } + } + dynamicRuntimeLinkInputs = dynamicRuntimeLinkInputsBuilder.build(); + } else { + dynamicRuntimeLinkInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (!dynamicRuntimeLinkInputs.isEmpty()) { + List<Artifact> dynamicRuntimeLinkMiddlemanSet = + CppHelper.getAggregatingMiddlemanForCppRuntimes( + ruleContext, + purposePrefix + "dynamic_runtime_link", + dynamicRuntimeLibDep, + runtimeSolibDirBase, + ruleContext.getConfiguration()); + dynamicRuntimeLinkMiddleman = dynamicRuntimeLinkMiddlemanSet.isEmpty() + ? null : Iterables.getOnlyElement(dynamicRuntimeLinkMiddlemanSet); + } else { + dynamicRuntimeLinkMiddleman = null; + } + + Preconditions.checkState( + (dynamicRuntimeLinkMiddleman == null) == dynamicRuntimeLinkInputs.isEmpty()); + + CppCompilationContext.Builder contextBuilder = + new CppCompilationContext.Builder(ruleContext); + CppModuleMap moduleMap = createCrosstoolModuleMap(ruleContext); + if (moduleMap != null) { + contextBuilder.setCppModuleMap(moduleMap); + } + final CppCompilationContext context = contextBuilder.build(); + boolean supportsParamFiles = ruleContext.attributes().get("supports_param_files", BOOLEAN); + boolean supportsHeaderParsing = + ruleContext.attributes().get("supports_header_parsing", BOOLEAN); + + CcToolchainProvider provider = new CcToolchainProvider( + Preconditions.checkNotNull(ruleContext.getFragment(CppConfiguration.class)), + crosstool, + fullInputsForCrosstool(ruleContext, crosstoolMiddleman), + compile, + strip, + objcopy, + fullInputsForLink(ruleContext, link), + dwp, + libcLink, + staticRuntimeLinkInputs, + staticRuntimeLinkMiddleman, + dynamicRuntimeLinkInputs, + dynamicRuntimeLinkMiddleman, + runtimeSolibDir, + context, + supportsParamFiles, + supportsHeaderParsing); + RuleConfiguredTargetBuilder builder = + new RuleConfiguredTargetBuilder(ruleContext) + .add(CcToolchainProvider.class, provider) + .setFilesToBuild(new NestedSetBuilder<Artifact>(Order.STABLE_ORDER).build()) + .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY)); + + // If output_license is specified on the cc_toolchain rule, override the transitive licenses + // with that one. This is necessary because cc_toolchain is used in the target configuration, + // but it is sort-of-kind-of a tool, but various parts of it are linked into the output... + // ...so we trust the judgment of the author of the cc_toolchain rule to figure out what + // licenses should be propagated to C++ targets. + License outputLicense = ruleContext.getRule().getToolOutputLicense(ruleContext.attributes()); + if (outputLicense != null && outputLicense != License.NO_LICENSE) { + final NestedSet<TargetLicense> license = NestedSetBuilder.create(Order.STABLE_ORDER, + new TargetLicense(ruleContext.getLabel(), outputLicense)); + LicensesProvider licensesProvider = new LicensesProvider() { + @Override + public NestedSet<TargetLicense> getTransitiveLicenses() { + return license; + } + }; + + builder.add(LicensesProvider.class, licensesProvider); + } + + return builder.build(); + } + + private NestedSet<Artifact> inputsForLibcLink(RuleContext ruleContext) { + TransitiveInfoCollection libcLink = ruleContext.getPrerequisite(":libc_link", Mode.HOST); + return libcLink != null + ? libcLink.getProvider(FileProvider.class).getFilesToBuild() + : NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER); + } + + private NestedSet<Artifact> fullInputsForCrosstool(RuleContext ruleContext, + NestedSet<Artifact> crosstoolMiddleman) { + return NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(crosstoolMiddleman) + // Use "libc_link" here, because it is functionally identical to the case + // below. If we introduce separate filegroups for compiling and linking, we + // need to fix that here. + .addTransitive(AnalysisUtils.getMiddlemanFor(ruleContext, ":libc_link")) + .build(); + } + + private NestedSet<Artifact> fullInputsForLink(RuleContext ruleContext, NestedSet<Artifact> link) { + return NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(link) + .addTransitive(AnalysisUtils.getMiddlemanFor(ruleContext, ":libc_link")) + .add(ruleContext.getAnalysisEnvironment().getEmbeddedToolArtifact( + CppRuleClasses.BUILD_INTERFACE_SO)) + .build(); + } + + private CppModuleMap createCrosstoolModuleMap(RuleContext ruleContext) { + if (ruleContext.getPrerequisite("module_map", Mode.HOST) == null) { + return null; + } + Artifact moduleMapArtifact = ruleContext.getPrerequisiteArtifact("module_map", Mode.HOST); + if (moduleMapArtifact == null) { + return null; + } + return new CppModuleMap(moduleMapArtifact, "crosstool"); + } + + private TransitiveInfoCollection selectDep( + RuleContext ruleContext, String attribute, Label label) { + for (TransitiveInfoCollection dep : ruleContext.getPrerequisites(attribute, Mode.TARGET)) { + if (dep.getLabel().equals(label)) { + return dep; + } + } + + return ruleContext.getPrerequisites(attribute, Mode.TARGET).get(0); + } + + private NestedSet<Artifact> getFiles(RuleContext context, String attribute) { + TransitiveInfoCollection dep = context.getPrerequisite(attribute, Mode.HOST); + MiddlemanProvider middlemanProvider = dep.getProvider(MiddlemanProvider.class); + // We use the middleman if we can (if the dep is a filegroup), otherwise, just the regular + // filesToBuild (e.g. if it is a simple input file) + return middlemanProvider != null + ? middlemanProvider.getMiddlemanArtifact() + : dep.getProvider(FileProvider.class).getFilesToBuild(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java new file mode 100644 index 0000000..29ab45c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java
@@ -0,0 +1,802 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CToolchain; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.Serializable; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; + +/** + * Provides access to features supported by a specific toolchain. + * + * <p>This class can be generated from the CToolchain protocol buffer. + * + * <p>TODO(bazel-team): Implement support for specifying the toolchain configuration directly from + * the BUILD file. + * + * <p>TODO(bazel-team): Find a place to put the public-facing documentation and link to it from + * here. + * + * <p>TODO(bazel-team): Split out Feature as CcToolchainFeature, which will modularize the + * crosstool configuration into one part that is about handling a set of features (including feature + * selection) and one part that is about how to apply a single feature (parsing flags and expanding + * them from build variables). + */ +@Immutable +public class CcToolchainFeatures implements Serializable { + + /** + * Thrown when a flag value cannot be expanded under a set of build variables. + * + * <p>This happens for example when a flag references a variable that is not provided by the + * action, or when a flag group references multiple variables of sequence type. + */ + public static class ExpansionException extends RuntimeException { + ExpansionException(String message) { + super(message); + } + } + + /** + * A piece of a single flag. + * + * <p>A single flag can contain a combination of text and variables (for example + * "-f %{var1}/%{var2}"). We split the flag into chunks, where each chunk represents either a + * text snippet, or a variable that is to be replaced. + */ + interface FlagChunk { + + /** + * Expands this chunk. + * + * @param variables variable names mapped to their values for a single flag expansion. + * @param flag the flag content to append to. + */ + void expand(Map<String, String> variables, StringBuilder flag); + } + + /** + * A plain text chunk of a flag. + */ + @Immutable + private static class StringChunk implements FlagChunk, Serializable { + private final String text; + + private StringChunk(String text) { + this.text = text; + } + + @Override + public void expand(Map<String, String> variables, StringBuilder flag) { + flag.append(text); + } + } + + /** + * A chunk of a flag into which a variable should be expanded. + */ + @Immutable + private static class VariableChunk implements FlagChunk, Serializable { + private final String variableName; + + private VariableChunk(String variableName) { + this.variableName = variableName; + } + + @Override + public void expand(Map<String, String> variables, StringBuilder flag) { + String value = variables.get(variableName); + if (value == null) { + // We check all variables in FlagGroup.expandCommandLine, so if we arrive here with a + // null value, the variable map originally handed to the feature selection must have + // contained an explicit null value. + throw new ExpansionException("Internal blaze error: build variable was set to 'null'."); + } + flag.append(variables.get(variableName)); + } + } + + /** + * Parser for toolchain flags. + * + * <p>A flag contains a snippet of text supporting variable expansion. For example, a flag value + * "-f %{var1}/%{var2}" will expand the values of the variables "var1" and "var2" in the + * corresponding places in the string. + * + * <p>The {@code FlagParser} takes a flag string and parses it into a list of {@code FlagChunk} + * objects, where each chunk represents either a snippet of text or a variable to be expanded. In + * the above example, the resulting chunks would be ["-f ", var1, "/", var2]. + * + * <p>In addition to the list of chunks, the {@code FlagParser} also provides the set of variables + * necessary for the expansion of this flag via {@code getUsedVariables}. + * + * <p>To get a literal percent character, "%%" can be used in the flag text. + */ + private static class FlagParser { + + /** + * The given flag value. + */ + private final String value; + + /** + * The current position in {@value} during parsing. + */ + private int current = 0; + + private final ImmutableList.Builder<FlagChunk> chunks = ImmutableList.builder(); + private final ImmutableSet.Builder<String> usedVariables = ImmutableSet.builder(); + + private FlagParser(String value) throws InvalidConfigurationException { + this.value = value; + parse(); + } + + /** + * @return the parsed chunks for this flag. + */ + private ImmutableList<FlagChunk> getChunks() { + return chunks.build(); + } + + /** + * @return all variable names needed to expand this flag. + */ + private ImmutableSet<String> getUsedVariables() { + return usedVariables.build(); + } + + /** + * Parses the flag. + * + * @throws InvalidConfigurationException if there is a parsing error. + */ + private void parse() throws InvalidConfigurationException { + while (current < value.length()) { + if (atVariableStart()) { + parseVariableChunk(); + } else { + parseStringChunk(); + } + } + } + + /** + * @return whether the current position is the start of a variable. + */ + private boolean atVariableStart() { + // We parse a variable when value starts with '%', but not '%%'. + return value.charAt(current) == '%' + && (current + 1 >= value.length() || value.charAt(current + 1) != '%'); + } + + /** + * Parses a chunk of text until the next '%', which indicates either an escaped literal '%' + * or a variable. + */ + private void parseStringChunk() { + int start = current; + // We only parse string chunks starting with '%' if they also start with '%%'. + // In that case, we want to have a single '%' in the string, so we start at the second + // character. + // Note that for flags like "abc%%def" this will lead to two string chunks, the first + // referencing the subtring "abc", and a second referencing the substring "%def". + if (value.charAt(current) == '%') { + current = current + 1; + start = current; + } + current = value.indexOf('%', current + 1); + if (current == -1) { + current = value.length(); + } + final String text = value.substring(start, current); + chunks.add(new StringChunk(text)); + } + + /** + * Parses a variable to be expanded. + * + * @throws InvalidConfigurationException if there is a parsing error. + */ + private void parseVariableChunk() throws InvalidConfigurationException { + current = current + 1; + if (current >= value.length() || value.charAt(current) != '{') { + abort("expected '{'"); + } + current = current + 1; + if (current >= value.length() || value.charAt(current) == '}') { + abort("expected variable name"); + } + int end = value.indexOf('}', current); + final String name = value.substring(current, end); + usedVariables.add(name); + chunks.add(new VariableChunk(name)); + current = end + 1; + } + + /** + * @throws InvalidConfigurationException with the given error text, adding information about + * the current position in the flag. + */ + private void abort(String error) throws InvalidConfigurationException { + throw new InvalidConfigurationException("Invalid toolchain configuration: " + error + + " at position " + current + " while parsing a flag containing '" + value + "'"); + } + } + + /** + * A single flag to be expanded under a set of variables. + * + * <p>TODO(bazel-team): Consider specializing Flag for the simple case that a flag is just a bit + * of text. + */ + @Immutable + private static class Flag implements Serializable { + private final ImmutableList<FlagChunk> chunks; + + private Flag(ImmutableList<FlagChunk> chunks) { + this.chunks = chunks; + } + + /** + * Expand this flag into a single new entry in {@code commandLine}. + */ + private void expandCommandLine(Map<String, String> variables, List<String> commandLine) { + StringBuilder flag = new StringBuilder(); + for (FlagChunk chunk : chunks) { + chunk.expand(variables, flag); + } + commandLine.add(flag.toString()); + } + } + + /** + * A group of flags. + */ + @Immutable + private static class FlagGroup implements Serializable { + private final ImmutableList<Flag> flags; + private final ImmutableSet<String> usedVariables; + + private FlagGroup(CToolchain.FlagGroup flagGroup) throws InvalidConfigurationException { + ImmutableList.Builder<Flag> flags = ImmutableList.builder(); + ImmutableSet.Builder<String> usedVariables = ImmutableSet.builder(); + for (String flag : flagGroup.getFlagList()) { + FlagParser parser = new FlagParser(flag); + flags.add(new Flag(parser.getChunks())); + usedVariables.addAll(parser.getUsedVariables()); + } + this.flags = flags.build(); + this.usedVariables = usedVariables.build(); + } + + /** + * Expands all flags in this group and adds them to {@code commandLine}. + * + * <p>The flags of the group will be expanded either: + * <ul> + * <li>once, if there is no variable of sequence type in any of the group's flags, or</li> + * <li>for each element in the sequence, if there is one variable of sequence type within + * the flags.</li> + * </ul> + * + * <p>Having more than a single variable of sequence type in a single flag group is not + * supported. + */ + private void expandCommandLine(Multimap<String, String> variables, List<String> commandLine) { + Map<String, String> variableView = new HashMap<>(); + String sequenceName = null; + for (String name : usedVariables) { + Collection<String> value = variables.get(name); + if (value.isEmpty()) { + throw new ExpansionException("Invalid toolchain configuration: unknown variable '" + name + + "' can not be expanded."); + } else if (value.size() > 1) { + if (sequenceName != null) { + throw new ExpansionException( + "Invalid toolchain configuration: trying to expand two variable list in one " + + "flag group: '" + sequenceName + "' and '" + name + "'"); + } + sequenceName = name; + } else { + variableView.put(name, value.iterator().next()); + } + } + if (sequenceName != null) { + for (String value : variables.get(sequenceName)) { + variableView.put(sequenceName, value); + expandOnce(variableView, commandLine); + } + } else { + expandOnce(variableView, commandLine); + } + } + + /** + * Expanding all flags of this group into {@code commandLine}. + */ + private void expandOnce(Map<String, String> variables, List<String> commandLine) { + for (Flag flag : flags) { + flag.expandCommandLine(variables, commandLine); + } + } + } + + /** + * Groups a set of flags to apply for certain actions. + */ + @Immutable + private static class FlagSet implements Serializable { + private final ImmutableSet<String> actions; + private final ImmutableList<FlagGroup> flagGroups; + + private FlagSet(CToolchain.FlagSet flagSet) throws InvalidConfigurationException { + this.actions = ImmutableSet.copyOf(flagSet.getActionList()); + ImmutableList.Builder<FlagGroup> builder = ImmutableList.builder(); + for (CToolchain.FlagGroup flagGroup : flagSet.getFlagGroupList()) { + builder.add(new FlagGroup(flagGroup)); + } + this.flagGroups = builder.build(); + } + + /** + * Adds the flags that apply to the given {@code action} to {@code commandLine}. + */ + private void expandCommandLine(String action, Multimap<String, String> variables, + List<String> commandLine) { + if (!actions.contains(action)) { + return; + } + for (FlagGroup flagGroup : flagGroups) { + flagGroup.expandCommandLine(variables, commandLine); + } + } + } + + /** + * Contains flags for a specific feature. + */ + @Immutable + private static class Feature implements Serializable { + private final String name; + private final ImmutableList<FlagSet> flagSets; + + private Feature(CToolchain.Feature feature) throws InvalidConfigurationException { + this.name = feature.getName(); + ImmutableList.Builder<FlagSet> builder = ImmutableList.builder(); + for (CToolchain.FlagSet flagSet : feature.getFlagSetList()) { + builder.add(new FlagSet(flagSet)); + } + this.flagSets = builder.build(); + } + + /** + * @return the features's name. + */ + private String getName() { + return name; + } + + /** + * Adds the flags that apply to the given {@code action} to {@code commandLine}. + */ + private void expandCommandLine(String action, Multimap<String, String> variables, + List<String> commandLine) { + for (FlagSet flagSet : flagSets) { + flagSet.expandCommandLine(action, variables, commandLine); + } + } + } + + /** + * Captures the set of enabled features for a rule. + */ + @Immutable + public static class FeatureConfiguration { + private final ImmutableSet<String> enabledFeatureNames; + private final ImmutableList<Feature> enabledFeatures; + + public FeatureConfiguration() { + enabledFeatureNames = ImmutableSet.of(); + enabledFeatures = ImmutableList.of(); + } + + private FeatureConfiguration(ImmutableList<Feature> enabledFeatures) { + this.enabledFeatures = enabledFeatures; + ImmutableSet.Builder<String> builder = ImmutableSet.builder(); + for (Feature feature : enabledFeatures) { + builder.add(feature.getName()); + } + this.enabledFeatureNames = builder.build(); + } + + /** + * @return whether the given {@code feature} is enabled. + */ + boolean isEnabled(String feature) { + return enabledFeatureNames.contains(feature); + } + + /** + * @return the command line for the given {@code action}. + */ + List<String> getCommandLine(String action, Multimap<String, String> variables) { + List<String> commandLine = new ArrayList<>(); + for (Feature feature : enabledFeatures) { + feature.expandCommandLine(action, variables, commandLine); + } + return commandLine; + } + } + + /** + * All features in the order in which they were specified in the configuration. + * + * <p>We guarantee the command line to be in the order in which the flags were specified in the + * configuration. + */ + private final ImmutableList<Feature> features; + + /** + * Maps from the feature's name to the feature. + */ + private final ImmutableMap<String, Feature> featuresByName; + + /** + * Maps from a feature to a set of all the features it has a direct 'implies' edge to. + */ + private final ImmutableMultimap<Feature, Feature> implies; + + /** + * Maps from a feature to all features that have an direct 'implies' edge to this feature. + */ + private final ImmutableMultimap<Feature, Feature> impliedBy; + + /** + * Maps from a feature to a set of feature sets, where: + * <ul> + * <li>a feature set satisfies the 'requires' condition, if all features in the feature set are + * enabled</li> + * <li>the 'requires' condition is satisfied, if at least one of the feature sets satisfies the + * 'requires' condition.</li> + * </ul> + */ + private final ImmutableMultimap<Feature, ImmutableSet<Feature>> requires; + + /** + * Maps from a feature to all features that have a requirement referencing it. + * + * <p>This will be used to determine which features need to be re-checked after a feature was + * disabled. + */ + private final ImmutableMultimap<Feature, Feature> requiredBy; + + /** + * A cache of feature selection results, so we do not recalculate the feature selection for + * all actions. + */ + private transient LoadingCache<Collection<String>, FeatureConfiguration> + configurationCache = buildConfigurationCache(); + + /** + * Constructs the feature configuration from a {@code CToolchain} protocol buffer. + * + * @param toolchain the toolchain configuration as specified by the user. + * @throws InvalidConfigurationException if the configuration has logical errors. + */ + CcToolchainFeatures(CToolchain toolchain) throws InvalidConfigurationException { + // Build up the feature graph. + // First, we build up the map of name -> features in one pass, so that earlier features can + // reference later features in their configuration. + ImmutableList.Builder<Feature> features = ImmutableList.builder(); + HashMap<String, Feature> featuresByName = new HashMap<>(); + for (CToolchain.Feature toolchainFeature : toolchain.getFeatureList()) { + Feature feature = new Feature(toolchainFeature); + features.add(feature); + if (featuresByName.put(feature.getName(), feature) != null) { + throw new InvalidConfigurationException("Invalid toolchain configuration: feature '" + + feature.getName() + "' was specified multiple times."); + } + } + this.features = features.build(); + this.featuresByName = ImmutableMap.copyOf(featuresByName); + + // Next, we build up all forward references for 'implies' and 'requires' edges. + ImmutableMultimap.Builder<Feature, Feature> implies = ImmutableMultimap.builder(); + ImmutableMultimap.Builder<Feature, ImmutableSet<Feature>> requires = + ImmutableMultimap.builder(); + // We also store the reverse 'implied by' and 'required by' edges during this pass. + ImmutableMultimap.Builder<Feature, Feature> impliedBy = ImmutableMultimap.builder(); + ImmutableMultimap.Builder<Feature, Feature> requiredBy = ImmutableMultimap.builder(); + for (CToolchain.Feature toolchainFeature : toolchain.getFeatureList()) { + String name = toolchainFeature.getName(); + Feature feature = featuresByName.get(name); + for (CToolchain.FeatureSet requiredFeatures : toolchainFeature.getRequiresList()) { + ImmutableSet.Builder<Feature> allOf = ImmutableSet.builder(); + for (String requiredName : requiredFeatures.getFeatureList()) { + Feature required = getFeatureOrFail(requiredName, name); + allOf.add(required); + requiredBy.put(required, feature); + } + requires.put(feature, allOf.build()); + } + for (String impliedName : toolchainFeature.getImpliesList()) { + Feature implied = getFeatureOrFail(impliedName, name); + impliedBy.put(implied, feature); + implies.put(feature, implied); + } + } + this.implies = implies.build(); + this.requires = requires.build(); + this.impliedBy = impliedBy.build(); + this.requiredBy = requiredBy.build(); + } + + /** + * Assign an empty cache after default-deserializing all non-transient members. + */ + private void readObject(ObjectInputStream in) throws ClassNotFoundException, IOException { + in.defaultReadObject(); + this.configurationCache = buildConfigurationCache(); + } + + /** + * @return an empty {@code FeatureConfiguration} cache. + */ + private LoadingCache<Collection<String>, FeatureConfiguration> buildConfigurationCache() { + return CacheBuilder.newBuilder() + // TODO(klimek): Benchmark and tweak once we support a larger configuration. + .maximumSize(10000) + .build(new CacheLoader<Collection<String>, FeatureConfiguration>() { + @Override + public FeatureConfiguration load(Collection<String> requestedFeatures) { + return computeFeatureConfiguration(requestedFeatures); + } + }); + } + + /** + * Given a list of {@code requestedFeatures}, returns all features that are enabled by the + * toolchain configuration. + * + * <p>A requested feature will not be enabled if the toolchain does not support it (which may + * depend on other requested features). + * + * <p>Additional features will be enabled if the toolchain supports them and they are implied by + * requested features. + */ + FeatureConfiguration getFeatureConfiguration(Collection<String> requestedFeatures) { + return configurationCache.getUnchecked(requestedFeatures); + } + + private FeatureConfiguration computeFeatureConfiguration(Collection<String> requestedFeatures) { + // Command line flags will be output in the order in which they are specified in the toolchain + // configuration. + return new FeatureSelection(requestedFeatures).run(); + } + + /** + * Convenience method taking a variadic string argument list for testing. + */ + FeatureConfiguration getFeatureConfiguration(String... requestedFeatures) { + return getFeatureConfiguration(Arrays.asList(requestedFeatures)); + } + + /** + * @return the feature with the given {@code name}. + * + * @throws InvalidConfigurationException if no feature with the given name was configured. + */ + private Feature getFeatureOrFail(String name, String reference) + throws InvalidConfigurationException { + if (!featuresByName.containsKey(name)) { + throw new InvalidConfigurationException("Invalid toolchain configuration: feature '" + name + + "', which is referenced from feature '" + reference + "', is not defined."); + } + return featuresByName.get(name); + } + + @VisibleForTesting + Collection<String> getFeatureNames() { + Collection<String> featureNames = new HashSet<>(); + for (Feature feature : features) { + featureNames.add(feature.getName()); + } + return featureNames; + } + + /** + * Implements the feature selection algorithm. + * + * <p>Feature selection is done by first enabling all features reachable by an 'implies' edge, + * and then iteratively pruning features that have unmet requirements. + */ + private class FeatureSelection { + + /** + * The features Bazel would like to enable; either because they are supported and generally + * useful, or because the user required them (for example through the command line). + */ + private final ImmutableSet<Feature> requestedFeatures; + + /** + * The currently enabled feature; during feature selection, we first put all features reachable + * via an 'implies' edge into the enabled feature set, and than prune that set from features + * that have unmet requirements. + */ + private Set<Feature> enabled = new HashSet<>(); + + private FeatureSelection(Collection<String> requestedFeatures) { + ImmutableSet.Builder<Feature> builder = ImmutableSet.builder(); + for (String name : requestedFeatures) { + if (featuresByName.containsKey(name)) { + builder.add(featuresByName.get(name)); + } + } + this.requestedFeatures = builder.build(); + } + + /** + * @return all enabled features in the order in which they were specified in the configuration. + */ + private FeatureConfiguration run() { + for (Feature feature : requestedFeatures) { + enableAllImpliedBy(feature); + } + disableUnsupportedFeatures(); + ImmutableList.Builder<Feature> enabledFeaturesInOrder = ImmutableList.builder(); + for (Feature feature : features) { + if (enabled.contains(feature)) { + enabledFeaturesInOrder.add(feature); + } + } + return new FeatureConfiguration(enabledFeaturesInOrder.build()); + } + + /** + * Transitively and unconditionally enable all features implied by the given feature and the + * feature itself to the enabled feature set. + */ + private void enableAllImpliedBy(Feature feature) { + if (enabled.contains(feature)) { + return; + } + enabled.add(feature); + for (Feature implied : implies.get(feature)) { + enableAllImpliedBy(implied); + } + } + + /** + * Remove all unsupported features from the enabled feature set. + */ + private void disableUnsupportedFeatures() { + Queue<Feature> check = new ArrayDeque<>(enabled); + while (!check.isEmpty()) { + checkFeature(check.poll()); + } + } + + /** + * Check if the given feature is still satisfied within the set of currently enabled features. + * + * <p>If it is not, remove the feature from the set of enabled features, and re-check all + * features that may now also become disabled. + */ + private void checkFeature(Feature feature) { + if (!enabled.contains(feature) || isSatisfied(feature)) { + return; + } + enabled.remove(feature); + + // Once we disable a feature, we have to re-check all features that can be affected by + // that removal. + // 1. A feature that implied the current feature is now going to be disabled. + for (Feature impliesCurrent : impliedBy.get(feature)) { + checkFeature(impliesCurrent); + } + // 2. A feature that required the current feature may now be disabled, depending on whether + // the requirement was optional. + for (Feature requiresCurrent : requiredBy.get(feature)) { + checkFeature(requiresCurrent); + } + // 3. A feature that this feature implied may now be disabled if no other feature also implies + // it. + for (Feature implied : implies.get(feature)) { + checkFeature(implied); + } + } + + /** + * @return whether all requirements of the feature are met in the set of currently enabled + * features. + */ + private boolean isSatisfied(Feature feature) { + return (requestedFeatures.contains(feature) || isImpliedByEnabledFeature(feature)) + && allImplicationsEnabled(feature) && allRequirementsMet(feature); + } + + /** + * @return whether a currently enabled feature implies the given feature. + */ + private boolean isImpliedByEnabledFeature(Feature feature) { + for (Feature implies : impliedBy.get(feature)) { + if (enabled.contains(implies)) { + return true; + } + } + return false; + } + + /** + * @return whether all implications of the given feature are enabled. + */ + private boolean allImplicationsEnabled(Feature feature) { + for (Feature implied : implies.get(feature)) { + if (!enabled.contains(implied)) { + return false; + } + } + return true; + } + + /** + * @return whether all requirements are enabled. + * + * <p>This implies that for any of the feature sets all of the specified features are enabled. + */ + private boolean allRequirementsMet(Feature feature) { + if (!requires.containsKey(feature)) { + return true; + } + for (ImmutableSet<Feature> requiresAllOf : requires.get(feature)) { + boolean requirementMet = true; + for (Feature required : requiresAllOf) { + if (!enabled.contains(required)) { + requirementMet = false; + break; + } + } + if (requirementMet) { + return true; + } + } + return false; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java new file mode 100644 index 0000000..e1940a5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java
@@ -0,0 +1,226 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.vfs.PathFragment; + +import javax.annotation.Nullable; + +/** + * Information about a C++ compiler used by the <code>cc_*</code> rules. + */ +@Immutable +public final class CcToolchainProvider implements TransitiveInfoProvider { + /** + * An empty toolchain to be returned in the error case (instead of null). + */ + public static final CcToolchainProvider EMPTY_TOOLCHAIN_IS_ERROR = new CcToolchainProvider( + null, + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + null, + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + null, + PathFragment.EMPTY_FRAGMENT, + CppCompilationContext.EMPTY, + false, + false); + + @Nullable private final CppConfiguration cppConfiguration; + private final NestedSet<Artifact> crosstool; + private final NestedSet<Artifact> crosstoolMiddleman; + private final NestedSet<Artifact> compile; + private final NestedSet<Artifact> strip; + private final NestedSet<Artifact> objCopy; + private final NestedSet<Artifact> link; + private final NestedSet<Artifact> dwp; + private final NestedSet<Artifact> libcLink; + private final NestedSet<Artifact> staticRuntimeLinkInputs; + @Nullable private final Artifact staticRuntimeLinkMiddleman; + private final NestedSet<Artifact> dynamicRuntimeLinkInputs; + @Nullable private final Artifact dynamicRuntimeLinkMiddleman; + private final PathFragment dynamicRuntimeSolibDir; + private final CppCompilationContext cppCompilationContext; + private final boolean supportsParamFiles; + private final boolean supportsHeaderParsing; + + public CcToolchainProvider( + @Nullable CppConfiguration cppConfiguration, + NestedSet<Artifact> crosstool, + NestedSet<Artifact> crosstoolMiddleman, + NestedSet<Artifact> compile, + NestedSet<Artifact> strip, + NestedSet<Artifact> objCopy, + NestedSet<Artifact> link, + NestedSet<Artifact> dwp, + NestedSet<Artifact> libcLink, + NestedSet<Artifact> staticRuntimeLinkInputs, + @Nullable Artifact staticRuntimeLinkMiddleman, + NestedSet<Artifact> dynamicRuntimeLinkInputs, + @Nullable Artifact dynamicRuntimeLinkMiddleman, + PathFragment dynamicRuntimeSolibDir, + CppCompilationContext cppCompilationContext, + boolean supportsParamFiles, + boolean supportsHeaderParsing) { + this.cppConfiguration = cppConfiguration; + this.crosstool = Preconditions.checkNotNull(crosstool); + this.crosstoolMiddleman = Preconditions.checkNotNull(crosstoolMiddleman); + this.compile = Preconditions.checkNotNull(compile); + this.strip = Preconditions.checkNotNull(strip); + this.objCopy = Preconditions.checkNotNull(objCopy); + this.link = Preconditions.checkNotNull(link); + this.dwp = Preconditions.checkNotNull(dwp); + this.libcLink = Preconditions.checkNotNull(libcLink); + this.staticRuntimeLinkInputs = Preconditions.checkNotNull(staticRuntimeLinkInputs); + this.staticRuntimeLinkMiddleman = staticRuntimeLinkMiddleman; + this.dynamicRuntimeLinkInputs = Preconditions.checkNotNull(dynamicRuntimeLinkInputs); + this.dynamicRuntimeLinkMiddleman = dynamicRuntimeLinkMiddleman; + this.dynamicRuntimeSolibDir = Preconditions.checkNotNull(dynamicRuntimeSolibDir); + this.cppCompilationContext = Preconditions.checkNotNull(cppCompilationContext); + this.supportsParamFiles = supportsParamFiles; + this.supportsHeaderParsing = supportsHeaderParsing; + } + + /** + * Returns all the files in Crosstool. Is not a middleman. + */ + public NestedSet<Artifact> getCrosstool() { + return crosstool; + } + + /** + * Returns a middleman for all the files in Crosstool. + */ + public NestedSet<Artifact> getCrosstoolMiddleman() { + return crosstoolMiddleman; + } + + /** + * Returns the files necessary for compilation. + */ + public NestedSet<Artifact> getCompile() { + // If include scanning is disabled, we need the entire crosstool filegroup, including header + // files. If it is enabled, we use the filegroup without header files - they are found by + // include scanning. For go, we also don't need the header files. + return cppConfiguration != null && cppConfiguration.shouldScanIncludes() ? compile : crosstool; + } + + /** + * Returns the files necessary for a 'strip' invocation. + */ + public NestedSet<Artifact> getStrip() { + return strip; + } + + /** + * Returns the files necessary for an 'objcopy' invocation. + */ + public NestedSet<Artifact> getObjcopy() { + return objCopy; + } + + /** + * Returns the files necessary for linking, including the files needed for libc. + */ + public NestedSet<Artifact> getLink() { + return link; + } + + public NestedSet<Artifact> getDwp() { + return dwp; + } + + public NestedSet<Artifact> getLibcLink() { + return libcLink; + } + + /** + * Returns the static runtime libraries. + */ + public NestedSet<Artifact> getStaticRuntimeLinkInputs() { + return staticRuntimeLinkInputs; + } + + /** + * Returns an aggregating middleman that represents the static runtime libraries. + */ + @Nullable public Artifact getStaticRuntimeLinkMiddleman() { + return staticRuntimeLinkMiddleman; + } + + /** + * Returns the dynamic runtime libraries. + */ + public NestedSet<Artifact> getDynamicRuntimeLinkInputs() { + return dynamicRuntimeLinkInputs; + } + + /** + * Returns an aggregating middleman that represents the dynamic runtime libraries. + */ + @Nullable public Artifact getDynamicRuntimeLinkMiddleman() { + return dynamicRuntimeLinkMiddleman; + } + + /** + * Returns the name of the directory where the solib symlinks for the dynamic runtime libraries + * live. The directory itself will be under the root of the host configuration in the 'bin' + * directory. + */ + public PathFragment getDynamicRuntimeSolibDir() { + return dynamicRuntimeSolibDir; + } + + /** + * Returns the C++ compilation context for the toolchain. + */ + public CppCompilationContext getCppCompilationContext() { + return cppCompilationContext; + } + + /** + * Whether the toolchains supports parameter files. + */ + public boolean supportsParamFiles() { + return supportsParamFiles; + } + + /** + * Whether the toolchains supports header parsing. + */ + public boolean supportsHeaderParsing() { + return supportsHeaderParsing; + } + + /** + * Returns the configured features of the toolchain. + */ + public CcToolchainFeatures getFeatures() { + return cppConfiguration.getFeatures(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java new file mode 100644 index 0000000..6c68f00 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java
@@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.LICENSE; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.syntax.Label; + +/** + * Rule definition for compiler definition. + */ +@BlazeRule(name = "cc_toolchain", + ancestors = { BaseRuleClasses.BaseRule.class }, + factoryClass = CcToolchain.class) +public final class CcToolchainRule implements RuleDefinition { + private static final LateBoundLabel<BuildConfiguration> LIBC_LINK = + new LateBoundLabel<BuildConfiguration>() { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.getFragment(CppConfiguration.class).getLibcLabel(); + } + }; + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .setUndocumented() + .add(attr("output_licenses", LICENSE)) + .add(attr("cpu", STRING).mandatory()) + .add(attr("all_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("compiler_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("strip_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("objcopy_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("linker_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("dwp_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("static_runtime_libs", LABEL_LIST).legacyAllowAnyFileType().mandatory()) + .add(attr("dynamic_runtime_libs", LABEL_LIST).legacyAllowAnyFileType().mandatory()) + .add(attr("module_map", LABEL).legacyAllowAnyFileType().cfg(HOST)) + .add(attr("supports_param_files", BOOLEAN).value(true)) + .add(attr("supports_header_parsing", BOOLEAN).value(false)) + // TODO(bazel-team): Should be using the TARGET configuration. + .add(attr(":libc_link", LABEL).cfg(HOST).value(LIBC_LINK)) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppBuildInfo.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppBuildInfo.java new file mode 100644 index 0000000..78a5f89 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppBuildInfo.java
@@ -0,0 +1,89 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoCollection; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * C++ build info creation - generates header files that contain the corresponding build-info data. + */ +public final class CppBuildInfo implements BuildInfoFactory { + public static final BuildInfoKey KEY = new BuildInfoKey("C++"); + + private static final PathFragment BUILD_INFO_NONVOLATILE_HEADER_NAME = + new PathFragment("build-info-nonvolatile.h"); + private static final PathFragment BUILD_INFO_VOLATILE_HEADER_NAME = + new PathFragment("build-info-volatile.h"); + // TODO(bazel-team): (2011) Get rid of the redacted build info. We should try to make + // the linkstamping process handle the case where those values are undefined. + private static final PathFragment BUILD_INFO_REDACTED_HEADER_NAME = + new PathFragment("build-info-redacted.h"); + + @Override + public BuildInfoCollection create(BuildInfoContext buildInfoContext, BuildConfiguration config, + Artifact buildInfo, Artifact buildChangelist) { + List<Action> actions = new ArrayList<>(); + WriteBuildInfoHeaderAction redactedInfo = getHeader(buildInfoContext, config, + BUILD_INFO_REDACTED_HEADER_NAME, + Artifact.NO_ARTIFACTS, true, true); + WriteBuildInfoHeaderAction nonvolatileInfo = getHeader(buildInfoContext, config, + BUILD_INFO_NONVOLATILE_HEADER_NAME, + ImmutableList.of(buildInfo), + false, true); + WriteBuildInfoHeaderAction volatileInfo = getHeader(buildInfoContext, config, + BUILD_INFO_VOLATILE_HEADER_NAME, + ImmutableList.of(buildChangelist), + true, false); + actions.add(redactedInfo); + actions.add(nonvolatileInfo); + actions.add(volatileInfo); + return new BuildInfoCollection(actions, + ImmutableList.of(nonvolatileInfo.getPrimaryOutput(), volatileInfo.getPrimaryOutput()), + ImmutableList.of(redactedInfo.getPrimaryOutput())); + } + + private WriteBuildInfoHeaderAction getHeader(BuildInfoContext buildInfoContext, + BuildConfiguration config, PathFragment headerName, + Collection<Artifact> inputs, + boolean writeVolatileInfo, boolean writeNonVolatileInfo) { + Root outputPath = config.getIncludeDirectory(); + final Artifact header = + buildInfoContext.getBuildInfoArtifact(headerName, outputPath, + writeVolatileInfo && !inputs.isEmpty() + ? BuildInfoType.NO_REBUILD : BuildInfoType.FORCE_REBUILD_IF_CHANGED); + return new WriteBuildInfoHeaderAction( + inputs, header, writeVolatileInfo, writeNonVolatileInfo); + } + + @Override + public BuildInfoKey getKey() { + return KEY; + } + + @Override + public boolean isEnabled(BuildConfiguration config) { + return config.hasFragment(CppConfiguration.class); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompilationContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompilationContext.java new file mode 100644 index 0000000..cf39ef5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompilationContext.java
@@ -0,0 +1,918 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.MiddlemanFactory; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Immutable store of information needed for C++ compilation that is aggregated + * across dependencies. + */ +@Immutable +public final class CppCompilationContext implements TransitiveInfoProvider { + /** An empty compilation context. */ + public static final CppCompilationContext EMPTY = new Builder(null).build(); + + private final CommandLineContext commandLineContext; + private final ImmutableList<DepsContext> depsContexts; + private final CppModuleMap cppModuleMap; + private final Artifact headerModule; + private final Artifact picHeaderModule; + private final ImmutableSet<Artifact> compilationPrerequisites; + + private CppCompilationContext(CommandLineContext commandLineContext, + List<DepsContext> depsContexts, CppModuleMap cppModuleMap, Artifact headerModule, + Artifact picHeaderModule) { + Preconditions.checkNotNull(commandLineContext); + Preconditions.checkArgument(!depsContexts.isEmpty()); + this.commandLineContext = commandLineContext; + this.depsContexts = ImmutableList.copyOf(depsContexts); + this.cppModuleMap = cppModuleMap; + this.headerModule = headerModule; + this.picHeaderModule = picHeaderModule; + + if (depsContexts.size() == 1) { + // Only LIPO targets have more than one DepsContexts. This codepath avoids creating + // an ImmutableSet.Builder for the vast majority of the cases. + compilationPrerequisites = (depsContexts.get(0).compilationPrerequisiteStampFile != null) + ? ImmutableSet.<Artifact>of(depsContexts.get(0).compilationPrerequisiteStampFile) + : ImmutableSet.<Artifact>of(); + } else { + ImmutableSet.Builder<Artifact> prerequisites = ImmutableSet.builder(); + for (DepsContext depsContext : depsContexts) { + if (depsContext.compilationPrerequisiteStampFile != null) { + prerequisites.add(depsContext.compilationPrerequisiteStampFile); + } + } + compilationPrerequisites = prerequisites.build(); + } + } + + /** + * Returns the compilation prerequisites consolidated into middlemen + * prerequisites, or an empty set if there are no prerequisites. + * + * <p>For correct dependency tracking, and to reduce the overhead to establish + * dependencies on generated headers, we express the dependency on compilation + * prerequisites as a transitive dependency via a middleman. After they have + * been accumulated (using + * {@link Builder#addCompilationPrerequisites(Iterable)}, + * {@link Builder#mergeDependentContext(CppCompilationContext)}, and + * {@link Builder#mergeDependentContexts(Iterable)}, they are consolidated + * into a single middleman Artifact when {@link Builder#build()} is called. + * + * <p>The returned set can be empty if there are no prerequisites. Usually it + * contains a single middleman, but if LIPO is used there can be two. + */ + public ImmutableSet<Artifact> getCompilationPrerequisites() { + return compilationPrerequisites; + } + + /** + * Returns the immutable list of include directories to be added with "-I" + * (possibly empty but never null). This includes the include dirs from the + * transitive deps closure of the target. This list does not contain + * duplicates. All fragments are either absolute or relative to the exec root + * (see {@link BuildConfiguration#getExecRoot}). + */ + public ImmutableList<PathFragment> getIncludeDirs() { + return commandLineContext.includeDirs; + } + + /** + * Returns the immutable list of include directories to be added with + * "-iquote" (possibly empty but never null). This includes the include dirs + * from the transitive deps closure of the target. This list does not contain + * duplicates. All fragments are either absolute or relative to the exec root + * (see {@link BuildConfiguration#getExecRoot}). + */ + public ImmutableList<PathFragment> getQuoteIncludeDirs() { + return commandLineContext.quoteIncludeDirs; + } + + /** + * Returns the immutable list of include directories to be added with + * "-isystem" (possibly empty but never null). This includes the include dirs + * from the transitive deps closure of the target. This list does not contain + * duplicates. All fragments are either absolute or relative to the exec root + * (see {@link BuildConfiguration#getExecRoot}). + */ + public ImmutableList<PathFragment> getSystemIncludeDirs() { + return commandLineContext.systemIncludeDirs; + } + + /** + * Returns the immutable set of declared include directories, relative to a + * "-I" or "-iquote" directory" (possibly empty but never null). The returned + * collection may contain duplicate elements. + * + * <p>Note: The iteration order of this list is preserved as ide_build_info + * writes these directories and sources out and the ordering will help when + * used by consumers. + */ + public NestedSet<PathFragment> getDeclaredIncludeDirs() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).declaredIncludeDirs; + } + + NestedSetBuilder<PathFragment> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.declaredIncludeDirs); + } + + return builder.build(); + } + + /** + * Returns the immutable set of include directories, relative to a "-I" or + * "-iquote" directory", from which inclusion will produce a warning (possibly + * empty but never null). The returned collection may contain duplicate + * elements. + * + * <p>Note: The iteration order of this list is preserved as ide_build_info + * writes these directories and sources out and the ordering will help when + * used by consumers. + */ + public NestedSet<PathFragment> getDeclaredIncludeWarnDirs() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).declaredIncludeWarnDirs; + } + + NestedSetBuilder<PathFragment> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.declaredIncludeWarnDirs); + } + + return builder.build(); + } + + /** + * Returns the immutable set of headers that have been declared in the + * {@code src} or {@code headers attribute} (possibly empty but never null). + * The returned collection may contain duplicate elements. + * + * <p>Note: The iteration order of this list is preserved as ide_build_info + * writes these directories and sources out and the ordering will help when + * used by consumers. + */ + public NestedSet<Artifact> getDeclaredIncludeSrcs() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).declaredIncludeSrcs; + } + + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.declaredIncludeSrcs); + } + + return builder.build(); + } + + /** + * Returns the immutable pairs of (header file, pregrepped header file). + */ + public NestedSet<Pair<Artifact, Artifact>> getPregreppedHeaders() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).pregreppedHdrs; + } + + NestedSetBuilder<Pair<Artifact, Artifact>> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.pregreppedHdrs); + } + + return builder.build(); + } + + /** + * Returns the immutable set of additional transitive inputs needed for + * compilation, like C++ module map artifacts. + */ + public NestedSet<Artifact> getAdditionalInputs() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).auxiliaryInputs; + } + + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.auxiliaryInputs); + } + + return builder.build(); + } + + /** + * Returns optional inputs that are needed by any C++ compilations that use header modules. + * + * <p>For every target that the current target depends on transitively and that is built as header + * module, contains: + * <ul> + * <li>the pic/non-pic header module (pcm file)</li> + * <li>the transitive list of module maps.</li> + * </ul> + */ + private NestedSet<Artifact> getTransitiveAuxiliaryInputs() { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.transitiveAuxiliaryInputs); + } + return builder.build(); + } + + /** + * @return all modules maps in the transitive closure. + */ + private NestedSet<Artifact> getTransitiveModuleMaps() { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.transitiveModuleMaps); + } + return builder.build(); + } + + /** + * @return all headers whose transitive closure of includes needs to be + * available when compiling anything in the current target. + */ + protected NestedSet<Artifact> getTransitiveHeaderModuleSrcs() { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.transitiveHeaderModuleSrcs); + } + return builder.build(); + } + + /** + * @return all declared headers of the current module if the current target + * is compiled as a module. + */ + protected NestedSet<Artifact> getHeaderModuleSrcs() { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.headerModuleSrcs); + } + return builder.build(); + } + + /** + * Returns the set of defines needed to compile this target (possibly empty + * but never null). This includes definitions from the transitive deps closure + * for the target. The order of the returned collection is deterministic. + */ + public ImmutableList<String> getDefines() { + return commandLineContext.defines; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof CppCompilationContext)) { + return false; + } + CppCompilationContext other = (CppCompilationContext) obj; + return Objects.equals(headerModule, other.headerModule) + && Objects.equals(picHeaderModule, other.picHeaderModule) + && commandLineContext.equals(other.commandLineContext) + && depsContexts.equals(other.depsContexts); + } + + @Override + public int hashCode() { + return Objects.hash(headerModule, picHeaderModule, commandLineContext, depsContexts); + } + + /** + * Returns a context that is based on a given context but returns empty sets + * for {@link #getDeclaredIncludeDirs()} and {@link #getDeclaredIncludeWarnDirs()}. + */ + public static CppCompilationContext disallowUndeclaredHeaders(CppCompilationContext context) { + ImmutableList.Builder<DepsContext> builder = ImmutableList.builder(); + for (DepsContext depsContext : context.depsContexts) { + builder.add(new DepsContext( + depsContext.compilationPrerequisiteStampFile, + NestedSetBuilder.<PathFragment>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<PathFragment>emptySet(Order.STABLE_ORDER), + depsContext.declaredIncludeSrcs, + depsContext.pregreppedHdrs, + depsContext.auxiliaryInputs, + depsContext.headerModuleSrcs, + depsContext.transitiveAuxiliaryInputs, + depsContext.transitiveHeaderModuleSrcs, + depsContext.transitiveModuleMaps)); + } + return new CppCompilationContext(context.commandLineContext, builder.build(), + context.cppModuleMap, context.headerModule, context.picHeaderModule); + } + + /** + * Returns the context for a LIPO compile action. This uses the include dirs + * and defines of the library, but the declared inclusion dirs/srcs from both + * the library and the owner binary. + + * TODO(bazel-team): this might make every LIPO target have an unnecessary large set of + * inclusion dirs/srcs. The correct behavior would be to merge only the contexts + * of actual referred targets (as listed in .imports file). + * + * <p>Undeclared inclusion checking ({@link #getDeclaredIncludeDirs()}, + * {@link #getDeclaredIncludeWarnDirs()}, and + * {@link #getDeclaredIncludeSrcs()}) needs to use the union of the contexts + * of the involved source files. + * + * <p>For include and define command line flags ({@link #getIncludeDirs()} + * {@link #getQuoteIncludeDirs()}, {@link #getSystemIncludeDirs()}, and + * {@link #getDefines()}) LIPO compilations use the same values as non-LIPO + * compilation. + * + * <p>Include scanning is not handled by this method. See + * {@code IncludeScannable#getAuxiliaryScannables()} instead. + * + * @param ownerContext the compilation context of the owner binary + * @param libContext the compilation context of the library + */ + public static CppCompilationContext mergeForLipo(CppCompilationContext ownerContext, + CppCompilationContext libContext) { + return new CppCompilationContext(libContext.commandLineContext, + ImmutableList.copyOf(Iterables.concat(ownerContext.depsContexts, libContext.depsContexts)), + libContext.cppModuleMap, libContext.headerModule, libContext.picHeaderModule); + } + + /** + * @return the C++ module map of the owner. + */ + public CppModuleMap getCppModuleMap() { + return cppModuleMap; + } + + /** + * @return the non-pic C++ header module of the owner. + */ + private Artifact getHeaderModule() { + return headerModule; + } + + /** + * @return the pic C++ header module of the owner. + */ + private Artifact getPicHeaderModule() { + return picHeaderModule; + } + + /** + * The parts of the compilation context that influence the command line of + * compilation actions. + */ + @Immutable + private static class CommandLineContext { + private final ImmutableList<PathFragment> includeDirs; + private final ImmutableList<PathFragment> quoteIncludeDirs; + private final ImmutableList<PathFragment> systemIncludeDirs; + private final ImmutableList<String> defines; + + CommandLineContext(ImmutableList<PathFragment> includeDirs, + ImmutableList<PathFragment> quoteIncludeDirs, + ImmutableList<PathFragment> systemIncludeDirs, + ImmutableList<String> defines) { + this.includeDirs = includeDirs; + this.quoteIncludeDirs = quoteIncludeDirs; + this.systemIncludeDirs = systemIncludeDirs; + this.defines = defines; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof CommandLineContext)) { + return false; + } + CommandLineContext other = (CommandLineContext) obj; + return Objects.equals(includeDirs, other.includeDirs) + && Objects.equals(quoteIncludeDirs, other.quoteIncludeDirs) + && Objects.equals(systemIncludeDirs, other.systemIncludeDirs) + && Objects.equals(defines, other.defines); + } + + @Override + public int hashCode() { + return Objects.hash(includeDirs, quoteIncludeDirs, systemIncludeDirs, defines); + } + } + + /** + * The parts of the compilation context that defined the dependencies of + * actions of scheduling and inclusion validity checking. + */ + @Immutable + private static class DepsContext { + private final Artifact compilationPrerequisiteStampFile; + private final NestedSet<PathFragment> declaredIncludeDirs; + private final NestedSet<PathFragment> declaredIncludeWarnDirs; + private final NestedSet<Artifact> declaredIncludeSrcs; + private final NestedSet<Pair<Artifact, Artifact>> pregreppedHdrs; + + /** + * Optional inputs that are used by some forms of compilation, containing: + * <ul> + * <li>module map of the current target</li> + * <li>module maps of all direct dependencies that are not compiled as header modules</li> + * <li>all transitiveAuxiliaryInputs.</li> + * </ul> + */ + private final NestedSet<Artifact> auxiliaryInputs; + + /** + * All declared headers of the current module, if compiled as a header module. + */ + private final NestedSet<Artifact> headerModuleSrcs; + + private final NestedSet<Artifact> transitiveAuxiliaryInputs; + + /** + * Headers whose transitive closure of includes needs to be available when compiling the current + * target. For every target that the current target depends on transitively and that is built as + * header module, contains all headers that are part of its header module. + */ + private final NestedSet<Artifact> transitiveHeaderModuleSrcs; + + /** + * The module maps from all targets the current target depends on transitively. + */ + private final NestedSet<Artifact> transitiveModuleMaps; + + DepsContext(Artifact compilationPrerequisiteStampFile, + NestedSet<PathFragment> declaredIncludeDirs, + NestedSet<PathFragment> declaredIncludeWarnDirs, + NestedSet<Artifact> declaredIncludeSrcs, + NestedSet<Pair<Artifact, Artifact>> pregreppedHdrs, + NestedSet<Artifact> auxiliaryInputs, + NestedSet<Artifact> headerModuleSrcs, + NestedSet<Artifact> transitiveAuxiliaryInputs, + NestedSet<Artifact> transitiveHeaderModuleSrcs, + NestedSet<Artifact> transitiveModuleMaps) { + this.compilationPrerequisiteStampFile = compilationPrerequisiteStampFile; + this.declaredIncludeDirs = declaredIncludeDirs; + this.declaredIncludeWarnDirs = declaredIncludeWarnDirs; + this.declaredIncludeSrcs = declaredIncludeSrcs; + this.pregreppedHdrs = pregreppedHdrs; + this.auxiliaryInputs = auxiliaryInputs; + this.headerModuleSrcs = headerModuleSrcs; + this.transitiveAuxiliaryInputs = transitiveAuxiliaryInputs; + this.transitiveHeaderModuleSrcs = transitiveHeaderModuleSrcs; + this.transitiveModuleMaps = transitiveModuleMaps; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof DepsContext)) { + return false; + } + DepsContext other = (DepsContext) obj; + return Objects.equals( + compilationPrerequisiteStampFile, other.compilationPrerequisiteStampFile) + && Objects.equals(declaredIncludeDirs, other.declaredIncludeDirs) + && Objects.equals(declaredIncludeWarnDirs, other.declaredIncludeWarnDirs) + && Objects.equals(declaredIncludeSrcs, other.declaredIncludeSrcs) + && Objects.equals(auxiliaryInputs, other.auxiliaryInputs) + && Objects.equals(headerModuleSrcs, other.headerModuleSrcs) + // Due to the NestedSet equals being ==, and the code flow only setting them if at least + // auxiliaryInputs is set, these checks cannot be executed. We leave them in so the equals + // is still correct if that connection ever changes.R + && Objects.equals(transitiveAuxiliaryInputs, other.transitiveAuxiliaryInputs) + && Objects.equals(transitiveHeaderModuleSrcs, other.transitiveHeaderModuleSrcs) + && Objects.equals(transitiveModuleMaps, other.transitiveModuleMaps) + ; + } + + @Override + public int hashCode() { + return Objects.hash(compilationPrerequisiteStampFile, + declaredIncludeDirs, + declaredIncludeWarnDirs, + declaredIncludeSrcs, + auxiliaryInputs, + headerModuleSrcs, + transitiveAuxiliaryInputs, + transitiveHeaderModuleSrcs, + transitiveModuleMaps); + } + } + + /** + * Builder class for {@link CppCompilationContext}. + */ + public static class Builder { + private String purpose = "cpp_compilation_prerequisites"; + private final Set<Artifact> compilationPrerequisites = new LinkedHashSet<>(); + private final Set<PathFragment> includeDirs = new LinkedHashSet<>(); + private final Set<PathFragment> quoteIncludeDirs = new LinkedHashSet<>(); + private final Set<PathFragment> systemIncludeDirs = new LinkedHashSet<>(); + private final NestedSetBuilder<PathFragment> declaredIncludeDirs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<PathFragment> declaredIncludeWarnDirs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Artifact> declaredIncludeSrcs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Pair<Artifact, Artifact>> pregreppedHdrs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Artifact> auxiliaryInputs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Artifact> headerModuleSrcs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Artifact> transitiveAuxiliaryInputs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Artifact> transitiveHeaderModuleSrcs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<Artifact> transitiveModuleMaps = + NestedSetBuilder.stableOrder(); + private final Set<String> defines = new LinkedHashSet<>(); + private CppModuleMap cppModuleMap; + private Artifact headerModule; + private Artifact picHeaderModule; + + /** The rule that owns the context */ + private final RuleContext ruleContext; + + /** + * Creates a new builder for a {@link CppCompilationContext} instance. + */ + public Builder(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + /** + * Overrides the purpose of this context. This is useful if a Target + * needs more than one CppCompilationContext. (The purpose is used to + * construct the name of the prerequisites middleman for the context, and + * all artifacts for a given Target must have distinct names.) + * + * @param purpose must be a string which is suitable for use as a filename. + * A single rule may have many middlemen with distinct purposes. + * + * @see MiddlemanFactory#createErrorPropagatingMiddleman + */ + public Builder setPurpose(String purpose) { + this.purpose = purpose; + return this; + } + + public String getPurpose() { + return purpose; + } + + /** + * Merges the context of a dependency into this one by adding the contents + * of all of its attributes. + */ + public Builder mergeDependentContext(CppCompilationContext otherContext) { + Preconditions.checkNotNull(otherContext); + compilationPrerequisites.addAll(otherContext.getCompilationPrerequisites()); + includeDirs.addAll(otherContext.getIncludeDirs()); + quoteIncludeDirs.addAll(otherContext.getQuoteIncludeDirs()); + systemIncludeDirs.addAll(otherContext.getSystemIncludeDirs()); + declaredIncludeDirs.addTransitive(otherContext.getDeclaredIncludeDirs()); + declaredIncludeWarnDirs.addTransitive(otherContext.getDeclaredIncludeWarnDirs()); + declaredIncludeSrcs.addTransitive(otherContext.getDeclaredIncludeSrcs()); + pregreppedHdrs.addTransitive(otherContext.getPregreppedHeaders()); + + // Forward transitive information. + transitiveAuxiliaryInputs.addTransitive(otherContext.getTransitiveAuxiliaryInputs()); + transitiveModuleMaps.addTransitive(otherContext.getTransitiveModuleMaps()); + transitiveHeaderModuleSrcs.addTransitive(otherContext.getTransitiveHeaderModuleSrcs()); + + // All module maps of direct dependencies are inputs to the current compile independently of + // the build type. + if (otherContext.getCppModuleMap() != null) { + auxiliaryInputs.add(otherContext.getCppModuleMap().getArtifact()); + } + if (otherContext.getHeaderModule() != null || otherContext.getPicHeaderModule() != null) { + // If we depend directly on a target that has a compiled header module, all targets + // transitively depending on us will need that header module, and all transitive module + // maps. + if (otherContext.getHeaderModule() != null) { + transitiveAuxiliaryInputs.add(otherContext.getHeaderModule()); + } + if (otherContext.getPicHeaderModule() != null) { + transitiveAuxiliaryInputs.add(otherContext.getPicHeaderModule()); + } + transitiveAuxiliaryInputs.addAll(otherContext.getTransitiveModuleMaps()); + + // All targets transitively depending on us will need to have the full transitive #include + // closure of the headers in that module available. + transitiveHeaderModuleSrcs.addAll(otherContext.getHeaderModuleSrcs()); + } + // All compile actions in the current target will need the transitive inputs. + auxiliaryInputs.addAll(transitiveAuxiliaryInputs.build().toCollection()); + + defines.addAll(otherContext.getDefines()); + return this; + } + + /** + * Merges the context of some targets into this one by adding the contents + * of all of their attributes. Targets that do not implement + * {@link CppCompilationContext} are ignored. + */ + public Builder mergeDependentContexts(Iterable<CppCompilationContext> targets) { + for (CppCompilationContext target : targets) { + mergeDependentContext(target); + } + return this; + } + + /** + * Adds multiple compilation prerequisites. + */ + public Builder addCompilationPrerequisites(Iterable<Artifact> prerequisites) { + // LIPO collector must not add compilation prerequisites in order to avoid + // the creation of a middleman action. + Iterables.addAll(compilationPrerequisites, prerequisites); + return this; + } + + /** + * Add a single include directory to be added with "-I". It can be either + * relative to the exec root (see {@link BuildConfiguration#getExecRoot}) or + * absolute. Before it is stored, the include directory is normalized. + */ + public Builder addIncludeDir(PathFragment includeDir) { + includeDirs.add(includeDir.normalize()); + return this; + } + + /** + * Add multiple include directories to be added with "-I". These can be + * either relative to the exec root (see {@link + * BuildConfiguration#getExecRoot}) or absolute. The entries are normalized + * before they are stored. + */ + public Builder addIncludeDirs(Iterable<PathFragment> includeDirs) { + for (PathFragment includeDir : includeDirs) { + addIncludeDir(includeDir); + } + return this; + } + + /** + * Add a single include directory to be added with "-iquote". It can be + * either relative to the exec root (see {@link + * BuildConfiguration#getExecRoot}) or absolute. Before it is stored, the + * include directory is normalized. + */ + public Builder addQuoteIncludeDir(PathFragment quoteIncludeDir) { + quoteIncludeDirs.add(quoteIncludeDir.normalize()); + return this; + } + + /** + * Add a single include directory to be added with "-isystem". It can be + * either relative to the exec root (see {@link + * BuildConfiguration#getExecRoot}) or absolute. Before it is stored, the + * include directory is normalized. + */ + public Builder addSystemIncludeDir(PathFragment systemIncludeDir) { + systemIncludeDirs.add(systemIncludeDir.normalize()); + return this; + } + + /** + * Add a single declared include dir, relative to a "-I" or "-iquote" + * directory". + */ + public Builder addDeclaredIncludeDir(PathFragment dir) { + declaredIncludeDirs.add(dir); + return this; + } + + /** + * Add a single declared include directory, relative to a "-I" or "-iquote" + * directory", from which inclusion will produce a warning. + */ + public Builder addDeclaredIncludeWarnDir(PathFragment dir) { + declaredIncludeWarnDirs.add(dir); + return this; + } + + /** + * Adds a header that has been declared in the {@code src} or {@code headers attribute}. The + * header will also be added to the compilation prerequisites. + */ + public Builder addDeclaredIncludeSrc(Artifact header) { + declaredIncludeSrcs.add(header); + compilationPrerequisites.add(header); + headerModuleSrcs.add(header); + return this; + } + + /** + * Adds multiple headers that have been declared in the {@code src} or {@code headers + * attribute}. The headers will also be added to the compilation prerequisites. + */ + public Builder addDeclaredIncludeSrcs(Iterable<Artifact> declaredIncludeSrcs) { + this.declaredIncludeSrcs.addAll(declaredIncludeSrcs); + this.headerModuleSrcs.addAll(declaredIncludeSrcs); + return addCompilationPrerequisites(declaredIncludeSrcs); + } + + /** + * Add a map of generated source or header Artifact to an output Artifact after grepping + * the file for include statements. + */ + public Builder addPregreppedHeaderMap(Map<Artifact, Artifact> pregrepped) { + addCompilationPrerequisites(pregrepped.values()); + for (Map.Entry<Artifact, Artifact> entry : pregrepped.entrySet()) { + this.pregreppedHdrs.add(Pair.of(entry.getKey(), entry.getValue())); + } + return this; + } + + /** + * Adds a single define. + */ + public Builder addDefine(String define) { + defines.add(define); + return this; + } + + /** + * Adds multiple defines. + */ + public Builder addDefines(Iterable<String> defines) { + Iterables.addAll(this.defines, defines); + return this; + } + + /** + * Sets the C++ module map. + */ + public Builder setCppModuleMap(CppModuleMap cppModuleMap) { + this.cppModuleMap = cppModuleMap; + return this; + } + + /** + * Sets the C++ header module in non-pic mode. + */ + public Builder setHeaderModule(Artifact headerModule) { + this.headerModule = headerModule; + return this; + } + + /** + * Sets the C++ header module in pic mode. + */ + public Builder setPicHeaderModule(Artifact picHeaderModule) { + this.picHeaderModule = picHeaderModule; + return this; + } + + /** + * Builds the {@link CppCompilationContext}. + */ + public CppCompilationContext build() { + return build( + ruleContext == null ? null : ruleContext.getActionOwner(), + ruleContext == null ? null : ruleContext.getAnalysisEnvironment().getMiddlemanFactory()); + } + + @VisibleForTesting // productionVisibility = Visibility.PRIVATE + public CppCompilationContext build(ActionOwner owner, MiddlemanFactory middlemanFactory) { + if (cppModuleMap != null) { + // .cppmap files should also be mandatory inputs for compile actions + auxiliaryInputs.add(cppModuleMap.getArtifact()); + transitiveModuleMaps.add(cppModuleMap.getArtifact()); + } + + // We don't create middlemen in LIPO collector subtree, because some target CT + // will do that instead. + Artifact prerequisiteStampFile = (ruleContext != null + && ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector()) + ? getMiddlemanArtifact(middlemanFactory) + : createMiddleman(owner, middlemanFactory); + + return new CppCompilationContext( + new CommandLineContext(ImmutableList.copyOf(includeDirs), + ImmutableList.copyOf(quoteIncludeDirs), ImmutableList.copyOf(systemIncludeDirs), + ImmutableList.copyOf(defines)), + ImmutableList.of(new DepsContext(prerequisiteStampFile, + declaredIncludeDirs.build(), + declaredIncludeWarnDirs.build(), + declaredIncludeSrcs.build(), + pregreppedHdrs.build(), + auxiliaryInputs.build(), + headerModuleSrcs.build(), + transitiveAuxiliaryInputs.build(), + transitiveHeaderModuleSrcs.build(), + transitiveModuleMaps.build())), + cppModuleMap, + headerModule, + picHeaderModule); + } + + /** + * Creates a middleman for the compilation prerequisites. + * + * @return the middleman or null if there are no prerequisites + */ + private Artifact createMiddleman(ActionOwner owner, + MiddlemanFactory middlemanFactory) { + if (compilationPrerequisites.isEmpty()) { + return null; + } + + // Compilation prerequisites gathered in the compilationPrerequisites + // must be generated prior to executing C++ compilation step that depends + // on them (since these prerequisites include all potential header files, etc + // that could be referenced during compilation). So there is a definite need + // to ensure scheduling edge dependency. However, those prerequisites should + // have no effect on the decision whether C++ compilation should happen in + // the first place - only CppCompileAction outputs (*.o and *.d files) and + // all files referenced by the *.d file should be used to make that decision. + // If this action was never executed, then *.d file would be missing, forcing + // compilation to occur. If *.d file is present and has not changed then the + // only reason that would force us to re-compile would be change in one of + // the files referenced by the *.d file, since no other files participated + // in the compilation. We also need to propagate errors through this + // dependency link. So we use an error propagating middleman. + // Such middleman will be ignored by the dependency checker yet will still + // represent an edge in the action dependency graph - forcing proper execution + // order and error propagation. + return middlemanFactory.createErrorPropagatingMiddleman( + owner, ruleContext.getLabel().toString(), purpose, + ImmutableList.copyOf(compilationPrerequisites), + ruleContext.getConfiguration().getMiddlemanDirectory()); + } + + /** + * Returns the same set of artifacts as createMiddleman() would, but without + * actually creating middlemen. + */ + private Artifact getMiddlemanArtifact(MiddlemanFactory middlemanFactory) { + if (compilationPrerequisites.isEmpty()) { + return null; + } + + return middlemanFactory.getErrorPropagatingMiddlemanArtifact(ruleContext.getLabel() + .toString(), purpose, ruleContext.getConfiguration().getMiddlemanDirectory()); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java new file mode 100644 index 0000000..e90f9f7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java
@@ -0,0 +1,1356 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Artifact.MiddlemanExpander; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.extra.CppCompileInfo; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.PerLabelOptions; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.Tool; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.DependencySet; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import javax.annotation.Nullable; + +/** + * Action that represents some kind of C++ compilation step. + */ +@ThreadCompatible +public class CppCompileAction extends AbstractAction implements IncludeScannable { + /** + * Represents logic that determines which artifacts, if any, should be added to the actual inputs + * for each included file (in addition to the included file itself) + */ + public interface IncludeResolver { + /** + * Returns the set of files to be added for an included file (as returned in the .d file) + */ + Iterable<Artifact> getInputsForIncludedFile( + Artifact includedFile, ArtifactResolver artifactResolver); + } + + public static final IncludeResolver VOID_INCLUDE_RESOLVER = new IncludeResolver() { + @Override + public Iterable<Artifact> getInputsForIncludedFile(Artifact includedFile, + ArtifactResolver artifactResolver) { + return ImmutableList.of(); + } + }; + + private static final int VALIDATION_DEBUG = 0; // 0==none, 1==warns/errors, 2==all + private static final boolean VALIDATION_DEBUG_WARN = VALIDATION_DEBUG >= 1; + + /** + * A string constant for the c compilation action. + */ + public static final String C_COMPILE = "c-compile"; + + /** + * A string constant for the c++ compilation action. + */ + public static final String CPP_COMPILE = "c++-compile"; + + /** + * A string constant for the c++ header parsing. + */ + public static final String CPP_HEADER_PARSING = "c++-header-parsing"; + + /** + * A string constant for the c++ header preprocessing. + */ + public static final String CPP_HEADER_PREPROCESSING = "c++-header-preprocessing"; + + /** + * A string constant for the c++ module compilation action. + * Note: currently we don't support C module compilation. + */ + public static final String CPP_MODULE_COMPILE = "c++-module-compile"; + + /** + * A string constant for the preprocessing assembler action. + */ + public static final String PREPROCESS_ASSEMBLE = "preprocess-assemble"; + + + private final BuildConfiguration configuration; + protected final Artifact outputFile; + private final Label sourceLabel; + private final Artifact dwoFile; + private final Artifact optionalSourceFile; + private final NestedSet<Artifact> mandatoryInputs; + private final CppCompilationContext context; + private final Collection<PathFragment> extraSystemIncludePrefixes; + private final Iterable<IncludeScannable> lipoScannables; + private final CppCompileCommandLine cppCompileCommandLine; + private final boolean enableLayeringCheck; + private final boolean compileHeaderModules; + + @VisibleForTesting + final CppConfiguration cppConfiguration; + private final Class<? extends CppCompileActionContext> actionContext; + private final IncludeResolver includeResolver; + + /** + * Identifier for the actual execution time behavior of the action. + * + * <p>Required because the behavior of this class can be modified by injecting code in the + * constructor or by inheritance, and we want to have different cache keys for those. + */ + private final UUID actionClassId; + + private boolean inputsKnown = false; + + /** + * Set when the action prepares for execution. Used to preserve state between preparation and + * execution. + */ + private Collection<? extends ActionInput> additionalInputs = null; + + /** + * Creates a new action to compile C/C++ source files. + * + * @param owner the owner of the action, usually the configured target that + * emitted it + * @param sourceFile the source file that should be compiled. {@code mandatoryInputs} must + * contain this file + * @param sourceLabel the label of the rule the source file is generated by + * @param mandatoryInputs any additional files that need to be present for the + * compilation to succeed, can be empty but not null, for example, extra sources for FDO. + * @param outputFile the object file that is written as result of the + * compilation, or the fake object for {@link FakeCppCompileAction}s + * @param dotdFile the .d file that is generated as a side-effect of + * compilation + * @param gcnoFile the coverage notes that are written in coverage mode, can + * be null + * @param dwoFile the .dwo output file where debug information is stored for Fission + * builds (null if Fission mode is disabled) + * @param optionalSourceFile an additional optional source file (null if unneeded) + * @param configuration the build configurations + * @param context the compilation context + * @param copts options for the compiler + * @param coptsFilter regular expression to remove options from {@code copts} + * @param compileHeaderModules whether to compile C++ header modules + */ + protected CppCompileAction(ActionOwner owner, + // TODO(bazel-team): Eventually we will remove 'features'; all functionality in 'features' + // will be provided by 'featureConfiguration'. + ImmutableList<String> features, + FeatureConfiguration featureConfiguration, + Artifact sourceFile, + Label sourceLabel, + NestedSet<Artifact> mandatoryInputs, + Artifact outputFile, + DotdFile dotdFile, + @Nullable Artifact gcnoFile, + @Nullable Artifact dwoFile, + Artifact optionalSourceFile, + BuildConfiguration configuration, + CppConfiguration cppConfiguration, + CppCompilationContext context, + Class<? extends CppCompileActionContext> actionContext, + ImmutableList<String> copts, + ImmutableList<String> pluginOpts, + Predicate<String> coptsFilter, + ImmutableList<PathFragment> extraSystemIncludePrefixes, + boolean enableLayeringCheck, + @Nullable String fdoBuildStamp, + IncludeResolver includeResolver, + Iterable<IncludeScannable> lipoScannables, + UUID actionClassId, + boolean compileHeaderModules) { + // getInputs() method is overridden in this class so we pass a dummy empty + // list to the AbstractAction constructor in place of a real input collection. + super(owner, + Artifact.NO_ARTIFACTS, + CollectionUtils.asListWithoutNulls(outputFile, dotdFile.artifact(), + gcnoFile, dwoFile)); + this.configuration = configuration; + this.sourceLabel = sourceLabel; + this.outputFile = Preconditions.checkNotNull(outputFile); + this.dwoFile = dwoFile; + this.optionalSourceFile = optionalSourceFile; + this.context = context; + this.extraSystemIncludePrefixes = extraSystemIncludePrefixes; + this.enableLayeringCheck = enableLayeringCheck; + this.includeResolver = includeResolver; + this.cppConfiguration = cppConfiguration; + if (cppConfiguration != null && !cppConfiguration.shouldScanIncludes()) { + inputsKnown = true; + } + this.cppCompileCommandLine = new CppCompileCommandLine(sourceFile, dotdFile, + context.getCppModuleMap(), copts, coptsFilter, pluginOpts, + (gcnoFile != null), features, featureConfiguration, fdoBuildStamp); + this.actionContext = actionContext; + this.lipoScannables = lipoScannables; + this.actionClassId = actionClassId; + this.compileHeaderModules = compileHeaderModules; + + // We do not need to include the middleman artifact since it is a generated + // artifact and will definitely exist prior to this action execution. + this.mandatoryInputs = mandatoryInputs; + setInputs(createInputs(mandatoryInputs, context.getCompilationPrerequisites(), + optionalSourceFile)); + } + + private static NestedSet<Artifact> createInputs( + NestedSet<Artifact> mandatoryInputs, + Set<Artifact> prerequisites, Artifact optionalSourceFile) { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + if (optionalSourceFile != null) { + builder.add(optionalSourceFile); + } + builder.addAll(prerequisites); + builder.addTransitive(mandatoryInputs); + return builder.build(); + } + + public boolean shouldScanIncludes() { + return cppConfiguration.shouldScanIncludes(); + } + + @Override + public List<PathFragment> getBuiltInIncludeDirectories() { + return cppConfiguration.getBuiltInIncludeDirectories(); + } + + public String getHostSystemName() { + return cppConfiguration.getHostSystemName(); + } + + @Override + public NestedSet<Artifact> getMandatoryInputs() { + return mandatoryInputs; + } + + @Override + public boolean inputsKnown() { + return inputsKnown; + } + + /** + * Returns the list of additional inputs found by dependency discovery, during action preparation, + * and clears the stored list. {@link #prepare} must be called before this method is called, on + * each action execution. + */ + public Collection<? extends ActionInput> getAdditionalInputs() { + Collection<? extends ActionInput> result = Preconditions.checkNotNull(additionalInputs); + additionalInputs = null; + return result; + } + + @Override + public boolean discoversInputs() { + return true; + } + + @Override + public void discoverInputs(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + try { + this.additionalInputs = executor.getContext(CppCompileActionContext.class) + .findAdditionalInputs(this, actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException("Include scanning of rule '" + getOwner().getLabel() + "'", + executor.getVerboseFailures(), this); + } + } + + @Override + public Artifact getPrimaryInput() { + return getSourceFile(); + } + + @Override + public Artifact getPrimaryOutput() { + return getOutputFile(); + } + + /** + * Returns the path of the c/cc source for gcc. + */ + public final Artifact getSourceFile() { + return cppCompileCommandLine.sourceFile; + } + + /** + * Returns the path where gcc should put its result. + */ + public Artifact getOutputFile() { + return outputFile; + } + + /** + * Returns the path of the debug info output file (when debug info is + * spliced out of the .o file via fission). + */ + @Nullable + Artifact getDwoFile() { + return dwoFile; + } + + protected PathFragment getInternalOutputFile() { + return outputFile.getExecPath(); + } + + @VisibleForTesting + public List<String> getPluginOpts() { + return cppCompileCommandLine.pluginOpts; + } + + Collection<PathFragment> getExtraSystemIncludePrefixes() { + return extraSystemIncludePrefixes; + } + + @Override + public Map<Artifact, Path> getLegalGeneratedScannerFileMap() { + Map<Artifact, Path> legalOuts = new HashMap<>(); + + for (Artifact a : context.getDeclaredIncludeSrcs()) { + if (!a.isSourceArtifact()) { + legalOuts.put(a, null); + } + } + for (Pair<Artifact, Artifact> pregreppedSrcs : context.getPregreppedHeaders()) { + Artifact hdr = pregreppedSrcs.getFirst(); + Preconditions.checkState(!hdr.isSourceArtifact(), hdr); + legalOuts.put(hdr, pregreppedSrcs.getSecond().getPath()); + } + return Collections.unmodifiableMap(legalOuts); + } + + /** + * Returns the path where gcc should put the discovered dependency + * information. + */ + public DotdFile getDotdFile() { + return cppCompileCommandLine.dotdFile; + } + + protected boolean needsIncludeScanning(Executor executor) { + return executor.getContext(actionContext).needsIncludeScanning(); + } + + @Override + public String describeStrategy(Executor executor) { + return executor.getContext(actionContext).strategyLocality(); + } + + @VisibleForTesting + public CppCompilationContext getContext() { + return context; + } + + @Override + public List<PathFragment> getQuoteIncludeDirs() { + return context.getQuoteIncludeDirs(); + } + + @Override + public List<PathFragment> getIncludeDirs() { + ImmutableList.Builder<PathFragment> result = ImmutableList.builder(); + result.addAll(context.getIncludeDirs()); + for (String opt : cppCompileCommandLine.copts) { + if (opt.startsWith("-I") && opt.length() > 2) { + // We insist on the combined form "-Idir". + result.add(new PathFragment(opt.substring(2))); + } + } + return result.build(); + } + + @Override + public List<PathFragment> getSystemIncludeDirs() { + ImmutableList.Builder<PathFragment> result = ImmutableList.builder(); + result.addAll(context.getSystemIncludeDirs()); + for (String opt : cppCompileCommandLine.copts) { + if (opt.startsWith("-isystem") && opt.length() > 8) { + // We insist on the combined form "-isystemdir". + result.add(new PathFragment(opt.substring(8))); + } + } + return result.build(); + } + + @Override + public List<String> getCmdlineIncludes() { + ImmutableList.Builder<String> cmdlineIncludes = ImmutableList.builder(); + List<String> args = getArgv(); + for (Iterator<String> argi = args.iterator(); argi.hasNext();) { + String arg = argi.next(); + if (arg.equals("-include") && argi.hasNext()) { + cmdlineIncludes.add(argi.next()); + } + } + return cmdlineIncludes.build(); + } + + @Override + public Collection<Artifact> getIncludeScannerSources() { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + // For every header module we use for the build we need the set of sources that it can + // reference. + builder.addAll(context.getTransitiveHeaderModuleSrcs()); + if (CppFileTypes.CPP_MODULE_MAP.matches(getSourceFile().getPath())) { + // If this is an action that compiles the header module itself, the source we build is the + // module map, and we need to include-scan all headers that are referenced in the module map. + // We need to do include scanning as long as we want to support building code bases that are + // not fully strict layering clean. + builder.addAll(context.getHeaderModuleSrcs()); + } else { + builder.add(getSourceFile()); + } + return builder.build().toCollection(); + } + + @Override + public Iterable<IncludeScannable> getAuxiliaryScannables() { + return lipoScannables; + } + + /** + * Returns the list of "-D" arguments that should be used by this gcc + * invocation. Only used for testing. + */ + @VisibleForTesting + public ImmutableCollection<String> getDefines() { + return context.getDefines(); + } + + /** + * Returns an (immutable) map of environment key, value pairs to be + * provided to the C++ compiler. + */ + public ImmutableMap<String, String> getEnvironment() { + Map<String, String> environment = + new LinkedHashMap<>(configuration.getDefaultShellEnvironment()); + if (configuration.isCodeCoverageEnabled()) { + environment.put("PWD", "/proc/self/cwd"); + } + if (OS.getCurrent() == OS.WINDOWS) { + // TODO(bazel-team): Both GCC and clang rely on their execution directories being on + // PATH, otherwise they fail to find dependent DLLs (and they fail silently...). On + // the other hand, Windows documentation says that the directory of the executable + // is always searched for DLLs first. Not sure what to make of it. + // Other options are to forward the system path (brittle), or to add a PATH field to + // the crosstool file. + environment.put("PATH", cppConfiguration.getToolPathFragment(Tool.GCC).getParentDirectory() + .getPathString()); + } + return ImmutableMap.copyOf(environment); + } + + /** + * Returns a new, mutable list of command and arguments (argv) to be passed + * to the gcc subprocess. + */ + public final List<String> getArgv() { + return getArgv(getInternalOutputFile()); + } + + protected final List<String> getArgv(PathFragment outputFile) { + return cppCompileCommandLine.getArgv(outputFile); + } + + @Override + public ExtraActionInfo.Builder getExtraActionInfo() { + CppCompileInfo.Builder info = CppCompileInfo.newBuilder(); + info.setTool(cppConfiguration.getToolPathFragment(Tool.GCC).getPathString()); + for (String option : getCompilerOptions()) { + info.addCompilerOption(option); + } + info.setOutputFile(outputFile.getExecPathString()); + info.setSourceFile(getSourceFile().getExecPathString()); + if (inputsKnown()) { + info.addAllSourcesAndHeaders(Artifact.toExecPaths(getInputs())); + } else { + info.addSourcesAndHeaders(getSourceFile().getExecPathString()); + info.addAllSourcesAndHeaders( + Artifact.toExecPaths(context.getDeclaredIncludeSrcs())); + } + + return super.getExtraActionInfo() + .setExtension(CppCompileInfo.cppCompileInfo, info.build()); + } + + /** + * Returns the compiler options. + */ + @VisibleForTesting + public List<String> getCompilerOptions() { + return cppCompileCommandLine.getCompilerOptions(); + } + + /** + * Enforce that the includes actually visited during the compile were properly + * declared in the rules. + * + * <p>The technique is to walk through all of the reported includes that gcc + * emits into the .d file, and verify that they came from acceptable + * relative include directories. This is done in two steps: + * + * <p>First, each included file is stripped of any include path prefix from + * {@code quoteIncludeDirs} to produce an effective relative include dir+name. + * + * <p>Second, the remaining directory is looked up in {@code declaredIncludeDirs}, + * a list of acceptable dirs. This list contains a set of dir fragments that + * have been calculated by the configured target to be allowable for inclusion + * by this source. If no match is found, an error is reported and an exception + * is thrown. + * + * @throws ActionExecutionException iff there was an undeclared dependency + */ + @VisibleForTesting + public void validateInclusions( + MiddlemanExpander middlemanExpander, EventHandler eventHandler) + throws ActionExecutionException { + if (!cppConfiguration.shouldScanIncludes() || !inputsKnown()) { + return; + } + + IncludeProblems errors = new IncludeProblems(); + IncludeProblems warnings = new IncludeProblems(); + Set<Artifact> allowedIncludes = new HashSet<>(); + for (Artifact input : mandatoryInputs) { + if (input.isMiddlemanArtifact()) { + middlemanExpander.expand(input, allowedIncludes); + } + allowedIncludes.add(input); + } + + if (optionalSourceFile != null) { + allowedIncludes.add(optionalSourceFile); + } + List<PathFragment> cxxSystemIncludeDirs = + cppConfiguration.getBuiltInIncludeDirectories(); + Iterable<PathFragment> ignoreDirs = Iterables.concat(cxxSystemIncludeDirs, + extraSystemIncludePrefixes, context.getSystemIncludeDirs()); + + // Copy the sets to hash sets for fast contains checking. + // Avoid immutable sets here to limit memory churn. + Set<PathFragment> declaredIncludeDirs = Sets.newHashSet(context.getDeclaredIncludeDirs()); + Set<PathFragment> warnIncludeDirs = Sets.newHashSet(context.getDeclaredIncludeWarnDirs()); + Set<Artifact> declaredIncludeSrcs = Sets.newHashSet(context.getDeclaredIncludeSrcs()); + for (Artifact input : getInputs()) { + if (context.getCompilationPrerequisites().contains(input) + || allowedIncludes.contains(input)) { + continue; // ignore our fixed source in mandatoryInput: we just want includes + } + // Ignore headers from built-in include directories. + if (FileSystemUtils.startsWithAny(input.getExecPath(), ignoreDirs)) { + continue; + } + if (!isDeclaredIn(input, declaredIncludeDirs, declaredIncludeSrcs)) { + // This call can never match the declared include sources (they would be matched above). + // There are no declared include sources we need to warn about, so use an empty set here. + if (isDeclaredIn(input, warnIncludeDirs, ImmutableSet.<Artifact>of())) { + warnings.add(input.getPath().toString()); + } else { + errors.add(input.getPath().toString()); + } + } + } + if (VALIDATION_DEBUG_WARN) { + synchronized (System.err) { + if (VALIDATION_DEBUG >= 2 || errors.hasProblems() || warnings.hasProblems()) { + if (errors.hasProblems()) { + System.err.println("ERROR: Include(s) were not in declared srcs:"); + } else if (warnings.hasProblems()) { + System.err.println("WARN: Include(s) were not in declared srcs:"); + } else { + System.err.println("INFO: Include(s) were OK for '" + getSourceFile() + + "', declared srcs:"); + } + for (Artifact a : context.getDeclaredIncludeSrcs()) { + System.err.println(" '" + a.toDetailString() + "'"); + } + System.err.println(" or under declared dirs:"); + for (PathFragment f : Sets.newTreeSet(context.getDeclaredIncludeDirs())) { + System.err.println(" '" + f + "'"); + } + System.err.println(" or under declared warn dirs:"); + for (PathFragment f : Sets.newTreeSet(context.getDeclaredIncludeWarnDirs())) { + System.err.println(" '" + f + "'"); + } + System.err.println(" with prefixes:"); + for (PathFragment dirpath : context.getQuoteIncludeDirs()) { + System.err.println(" '" + dirpath + "'"); + } + } + } + } + + if (warnings.hasProblems()) { + eventHandler.handle( + new Event(EventKind.WARNING, + getOwner().getLocation(), warnings.getMessage(this, getSourceFile()), + Label.print(getOwner().getLabel()))); + } + errors.assertProblemFree(this, getSourceFile()); + } + + /** + * Returns true if an included artifact is declared in a set of allowed + * include directories. The simple case is that the artifact's parent + * directory is contained in the set, or is empty. + * + * <p>This check also supports a wildcard suffix of '**' for the cases where the + * calculations are inexact. + * + * <p>It also handles unseen non-nested-package subdirs by walking up the path looking + * for matches. + */ + private static boolean isDeclaredIn(Artifact input, Set<PathFragment> declaredIncludeDirs, + Set<Artifact> declaredIncludeSrcs) { + // First check if it's listed in "srcs". If so, then its declared & OK. + if (declaredIncludeSrcs.contains(input)) { + return true; + } + // If it's a derived artifact, then it MUST be listed in "srcs" as checked above. + // We define derived here as being not source and not under the include link tree. + if (!input.isSourceArtifact() + && !input.getRoot().getExecPath().getBaseName().equals("include")) { + return false; + } + // Need to do dir/package matching: first try a quick exact lookup. + PathFragment includeDir = input.getRootRelativePath().getParentDirectory(); + if (includeDir.segmentCount() == 0 || declaredIncludeDirs.contains(includeDir)) { + return true; // OK: quick exact match. + } + // Not found in the quick lookup: try the wildcards. + for (PathFragment declared : declaredIncludeDirs) { + if (declared.getBaseName().equals("**")) { + if (includeDir.startsWith(declared.getParentDirectory())) { + return true; // OK: under a wildcard dir. + } + } + } + // Still not found: see if it is in a subdir of a declared package. + Path root = input.getRoot().getPath(); + for (Path dir = input.getPath().getParentDirectory();;) { + if (dir.getRelative("BUILD").exists()) { + return false; // Bad: this is a sub-package, not a subdir of a declared package. + } + dir = dir.getParentDirectory(); + if (dir.equals(root)) { + return false; // Bad: at the top, give up. + } + if (declaredIncludeDirs.contains(dir.relativeTo(root))) { + return true; // OK: found under a declared dir. + } + } + } + + /** + * Recalculates this action's live input collection, including sources, middlemen. + * + * @throws ActionExecutionException iff any errors happen during update. + */ + @VisibleForTesting + @ThreadCompatible + public final void updateActionInputs(Path execRoot, + ArtifactResolver artifactResolver, CppCompileActionContext.Reply reply) + throws ActionExecutionException { + if (!cppConfiguration.shouldScanIncludes()) { + return; + } + inputsKnown = false; + NestedSetBuilder<Artifact> inputs = NestedSetBuilder.stableOrder(); + Profiler.instance().startTask(ProfilerTask.ACTION_UPDATE, this); + try { + inputs.addTransitive(mandatoryInputs); + if (optionalSourceFile != null) { + inputs.add(optionalSourceFile); + } + inputs.addAll(context.getCompilationPrerequisites()); + populateActionInputs(execRoot, artifactResolver, reply, inputs); + inputsKnown = true; + } finally { + Profiler.instance().completeTask(ProfilerTask.ACTION_UPDATE); + synchronized (this) { + setInputs(inputs.build()); + } + } + } + + private DependencySet processDepset(Path execRoot, CppCompileActionContext.Reply reply) + throws IOException { + DependencySet depSet = new DependencySet(execRoot); + + // artifact() is null if we are not using in-memory .d files. We also want to prepare for the + // case where we expected an in-memory .d file, but we did not get an appropriate response. + // Perhaps we produced the file locally. + if (getDotdFile().artifact() != null || reply == null) { + return depSet.read(getDotdFile().getPath()); + } else { + // This is an in-memory .d file. + return depSet.process(reply.getContents()); + } + } + + /** + * Populates the given ordered collection with additional input artifacts + * relevant to the specific action implementation. + * + * <p>The default implementation updates this Action's input set by reading + * dynamically-discovered dependency information out of the .d file. + * + * <p>Artifacts are considered inputs but not "mandatory" inputs. + * + * + * @param reply the reply from the compilation. + * @param inputs the ordered collection of inputs to append to + * @throws ActionExecutionException iff the .d is missing, malformed or has + * unresolvable included artifacts. + */ + @ThreadCompatible + private void populateActionInputs(Path execRoot, + ArtifactResolver artifactResolver, CppCompileActionContext.Reply reply, + NestedSetBuilder<Artifact> inputs) + throws ActionExecutionException { + try { + // Read .d file. + DependencySet depSet = processDepset(execRoot, reply); + + // Determine prefixes of allowed absolute inclusions. + CppConfiguration toolchain = cppConfiguration; + List<PathFragment> systemIncludePrefixes = new ArrayList<>(); + for (PathFragment includePath : toolchain.getBuiltInIncludeDirectories()) { + if (includePath.isAbsolute()) { + systemIncludePrefixes.add(includePath); + } + } + systemIncludePrefixes.addAll(extraSystemIncludePrefixes); + + // Check inclusions. + IncludeProblems problems = new IncludeProblems(); + Map<PathFragment, Artifact> allowedDerivedInputsMap = getAllowedDerivedInputsMap(); + for (PathFragment execPath : depSet.getDependencies()) { + if (execPath.isAbsolute()) { + // Absolute includes from system paths are ignored. + if (FileSystemUtils.startsWithAny(execPath, systemIncludePrefixes)) { + continue; + } + // Since gcc is given only relative paths on the command line, + // non-system include paths here should never be absolute. If they + // are, it's probably due to a non-hermetic #include, & we should stop + // the build with an error. + if (execPath.startsWith(execRoot.asFragment())) { + execPath = execPath.relativeTo(execRoot.asFragment()); // funky but tolerable path + } else { + problems.add(execPath.getPathString()); + continue; + } + } + Artifact artifact = allowedDerivedInputsMap.get(execPath); + if (artifact == null) { + artifact = artifactResolver.resolveSourceArtifact(execPath); + } + if (artifact != null) { + inputs.add(artifact); + // In some cases, execution backends need extra files for each included file. Add them + // to the set of actual inputs. + inputs.addAll(includeResolver.getInputsForIncludedFile(artifact, artifactResolver)); + } else { + // Abort if we see files that we can't resolve, likely caused by + // undeclared includes or illegal include constructs. + problems.add(execPath.getPathString()); + } + } + problems.assertProblemFree(this, getSourceFile()); + } catch (IOException e) { + // Some kind of IO or parse exception--wrap & rethrow it to stop the build. + throw new ActionExecutionException("error while parsing .d file", e, this, false); + } + } + + @Override + public void updateInputsFromCache( + ArtifactResolver artifactResolver, Collection<PathFragment> inputPaths) { + // Note that this method may trigger a violation of the desirable invariant that getInputs() + // is a superset of getMandatoryInputs(). See bug about an "action not in canonical form" + // error message and the integration test test_crosstool_change_and_failure(). + + Map<PathFragment, Artifact> allowedDerivedInputsMap = getAllowedDerivedInputsMap(); + List<Artifact> inputs = new ArrayList<>(); + for (PathFragment execPath : inputPaths) { + // The artifact may be a derived artifact, and if it has been created already, then we still + // want to keep it to preserve incrementality. + Artifact artifact = allowedDerivedInputsMap.get(execPath); + if (artifact == null) { + artifact = artifactResolver.resolveSourceArtifact(execPath); + } + // If PathFragment cannot be resolved into the artifact - ignore it. This could happen if + // rule definition has changed and action no longer depends on, e.g., additional source file + // in the separate package and that package is no longer referenced anywhere else. + // It is safe to ignore such paths because dependency checker would identify change in inputs + // (ignored path was used before) and will force action execution. + if (artifact != null) { + inputs.add(artifact); + } + } + inputsKnown = true; + synchronized (this) { + setInputs(inputs); + } + } + + private Map<PathFragment, Artifact> getAllowedDerivedInputsMap() { + Map<PathFragment, Artifact> allowedDerivedInputMap = new HashMap<>(); + addToMap(allowedDerivedInputMap, mandatoryInputs); + addToMap(allowedDerivedInputMap, context.getDeclaredIncludeSrcs()); + addToMap(allowedDerivedInputMap, context.getCompilationPrerequisites()); + Artifact artifact = getSourceFile(); + if (!artifact.isSourceArtifact()) { + allowedDerivedInputMap.put(artifact.getExecPath(), artifact); + } + return allowedDerivedInputMap; + } + + private void addToMap(Map<PathFragment, Artifact> map, Iterable<Artifact> artifacts) { + for (Artifact artifact : artifacts) { + if (!artifact.isSourceArtifact()) { + map.put(artifact.getExecPath(), artifact); + } + } + } + + @Override + protected String getRawProgressMessage() { + return "Compiling " + getSourceFile().prettyPrint(); + } + + /** + * Return the directories in which to look for headers (pertains to headers + * not specifically listed in {@code declaredIncludeSrcs}). The return value + * may contain duplicate elements. + */ + public NestedSet<PathFragment> getDeclaredIncludeDirs() { + return context.getDeclaredIncludeDirs(); + } + + /** + * Return the directories in which to look for headers and issue a warning. + * (pertains to headers not specifically listed in {@code + * declaredIncludeSrcs}). The return value may contain duplicate elements. + */ + public NestedSet<PathFragment> getDeclaredIncludeWarnDirs() { + return context.getDeclaredIncludeWarnDirs(); + } + + /** + * Return explicit header files (i.e., header files explicitly listed). The + * return value may contain duplicate elements. + */ + public NestedSet<Artifact> getDeclaredIncludeSrcs() { + return context.getDeclaredIncludeSrcs(); + } + + /** + * Return explicit header files (i.e., header files explicitly listed) in an order + * that is stable between builds. + */ + protected final List<PathFragment> getDeclaredIncludeSrcsInStableOrder() { + List<PathFragment> paths = new ArrayList<>(); + for (Artifact declaredIncludeSrc : context.getDeclaredIncludeSrcs()) { + paths.add(declaredIncludeSrc.getExecPath()); + } + Collections.sort(paths); // Order is not important, but stability is. + return paths; + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return executor.getContext(actionContext).estimateResourceConsumption(this); + } + + @VisibleForTesting + public Class<? extends CppCompileActionContext> getActionContext() { + return actionContext; + } + + /** + * Estimate resource consumption when this action is executed locally. + */ + public ResourceSet estimateResourceConsumptionLocal() { + // We use a local compile, so much of the time is spent waiting for IO, + // but there is still significant CPU; hence we estimate 50% cpu usage. + return new ResourceSet(/*memoryMb=*/200, /*cpuUsage=*/0.5, /*ioUsage=*/0.0); + } + + @Override + public String computeKey() { + Fingerprint f = new Fingerprint(); + f.addUUID(actionClassId); + f.addStrings(getArgv()); + + /* + * getArgv() above captures all changes which affect the compilation + * command and hence the contents of the object file. But we need to + * also make sure that we reexecute the action if any of the fields + * that affect whether validateIncludes() will report an error or warning + * have changed, otherwise we might miss some errors. + */ + f.addPaths(context.getDeclaredIncludeDirs()); + f.addPaths(context.getDeclaredIncludeWarnDirs()); + f.addPaths(getDeclaredIncludeSrcsInStableOrder()); + f.addPaths(getExtraSystemIncludePrefixes()); + return f.hexDigestAndReset(); + } + + @Override + @ThreadCompatible + public void execute( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + CppCompileActionContext.Reply reply; + try { + reply = executor.getContext(actionContext).execWithReply(this, actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException("C++ compilation of rule '" + getOwner().getLabel() + "'", + executor.getVerboseFailures(), this); + } + ensureCoverageNotesFilesExist(); + IncludeScanningContext scanningContext = executor.getContext(IncludeScanningContext.class); + updateActionInputs(executor.getExecRoot(), scanningContext.getArtifactResolver(), reply); + reply = null; // Clear in-memory .d files early. + validateInclusions(actionExecutionContext.getMiddlemanExpander(), executor.getEventHandler()); + } + + /** + * Gcc only creates ".gcno" files if the compilation unit is non-empty. + * To ensure that the set of outputs for a CppCompileAction remains consistent + * and doesn't vary dynamically depending on the _contents_ of the input files, + * we create empty ".gcno" files if gcc didn't create them. + */ + private void ensureCoverageNotesFilesExist() throws ActionExecutionException { + for (Artifact output : getOutputs()) { + if (CppFileTypes.COVERAGE_NOTES.matches(output.getFilename()) // ".gcno" + && !output.getPath().exists()) { + try { + FileSystemUtils.createEmptyFile(output.getPath()); + } catch (IOException e) { + throw new ActionExecutionException( + "Error creating file '" + output.getPath() + "': " + e.getMessage(), e, this, false); + } + } + } + } + + /** + * Provides list of include files needed for performing extra actions on this action when run + * remotely. The list of include files is created by performing a header scan on the known input + * files. + */ + @Override + public Iterable<Artifact> getInputFilesForExtraAction( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Collection<Artifact> scannedIncludes = + actionExecutionContext.getExecutor().getContext(actionContext) + .getScannedIncludeFiles(this, actionExecutionContext); + // Use a set to eliminate duplicates. + ImmutableSet.Builder<Artifact> result = ImmutableSet.builder(); + return result.addAll(getInputs()).addAll(scannedIncludes).build(); + } + + @Override + public String getMnemonic() { return "CppCompile"; } + + @Override + public String describeKey() { + StringBuilder message = new StringBuilder(); + message.append(getProgressMessage()); + message.append('\n'); + message.append(" Command: "); + message.append( + ShellEscaper.escapeString(cppConfiguration.getLdExecutable().getPathString())); + message.append('\n'); + // Outputting one argument per line makes it easier to diff the results. + for (String argument : ShellEscaper.escapeAll(getArgv())) { + message.append(" Argument: "); + message.append(argument); + message.append('\n'); + } + + for (PathFragment path : context.getDeclaredIncludeDirs()) { + message.append(" Declared include directory: "); + message.append(ShellEscaper.escapeString(path.getPathString())); + message.append('\n'); + } + + for (PathFragment path : getDeclaredIncludeSrcsInStableOrder()) { + message.append(" Declared include source: "); + message.append(ShellEscaper.escapeString(path.getPathString())); + message.append('\n'); + } + + for (PathFragment path : getExtraSystemIncludePrefixes()) { + message.append(" Extra system include prefix: "); + message.append(ShellEscaper.escapeString(path.getPathString())); + message.append('\n'); + } + return message.toString(); + } + + /** + * The compile command line for the enclosing C++ compile action. + */ + public final class CppCompileCommandLine { + private final Artifact sourceFile; + private final DotdFile dotdFile; + private final CppModuleMap cppModuleMap; + private final List<String> copts; + private final Predicate<String> coptsFilter; + private final List<String> pluginOpts; + private final boolean isInstrumented; + private final Collection<String> features; + private final FeatureConfiguration featureConfiguration; + + // The value of the BUILD_FDO_TYPE macro to be defined on command line + @Nullable private final String fdoBuildStamp; + + public CppCompileCommandLine(Artifact sourceFile, DotdFile dotdFile, CppModuleMap cppModuleMap, + ImmutableList<String> copts, Predicate<String> coptsFilter, + ImmutableList<String> pluginOpts, boolean isInstrumented, + Collection<String> features, FeatureConfiguration featureConfiguration, + @Nullable String fdoBuildStamp) { + this.sourceFile = Preconditions.checkNotNull(sourceFile); + this.dotdFile = Preconditions.checkNotNull(dotdFile); + this.cppModuleMap = cppModuleMap; + this.copts = Preconditions.checkNotNull(copts); + this.coptsFilter = coptsFilter; + this.pluginOpts = Preconditions.checkNotNull(pluginOpts); + this.isInstrumented = isInstrumented; + this.features = Preconditions.checkNotNull(features); + this.featureConfiguration = featureConfiguration; + this.fdoBuildStamp = fdoBuildStamp; + } + + protected List<String> getArgv(PathFragment outputFile) { + List<String> commandLine = new ArrayList<>(); + + // first: The command name. + commandLine.add(cppConfiguration.getToolPathFragment(Tool.GCC).getPathString()); + + // second: The compiler options. + commandLine.addAll(getCompilerOptions()); + + // third: The file to compile! + commandLine.add("-c"); + commandLine.add(sourceFile.getExecPathString()); + + // finally: The output file. (Prefixed with -o). + commandLine.add("-o"); + commandLine.add(outputFile.getPathString()); + + return commandLine; + } + + private String getActionName() { + PathFragment sourcePath = sourceFile.getExecPath(); + if (CppFileTypes.CPP_MODULE_MAP.matches(sourcePath)) { + return CPP_MODULE_COMPILE; + } else if (CppFileTypes.CPP_HEADER.matches(sourcePath)) { + // TODO(bazel-team): Handle C headers that probably don't work in C++ mode. + // TODO(bazel-team): Replace use of features.contains with featureConfiguration.isEnabled + // here (the other instances will need to stay with the current feature selection process + // until all crosstool configurations have been converted). + if (features.contains(CppRuleClasses.PARSE_HEADERS)) { + return CPP_HEADER_PARSING; + } else if (features.contains(CppRuleClasses.PREPROCESS_HEADERS)) { + return CPP_HEADER_PREPROCESSING; + } else { + // CcCommon.collectCAndCppSources() ensures we do not add headers to + // the compilation artifacts unless either 'parse_headers' or + // 'preprocess_headers' is set. + throw new IllegalStateException(); + } + } else if (CppFileTypes.C_SOURCE.matches(sourcePath)) { + return C_COMPILE; + } else if (CppFileTypes.CPP_SOURCE.matches(sourcePath)) { + return CPP_COMPILE; + } else if (CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR.matches(sourcePath)) { + return PREPROCESS_ASSEMBLE; + } + // CcLibraryHelper ensures CppCompileAction only gets instantiated for supported file types. + throw new IllegalStateException(); + } + + public List<String> getCompilerOptions() { + List<String> options = new ArrayList<>(); + + // TODO(bazel-team): Extract combinations of options into sections in the CROSSTOOL file. + if (CppFileTypes.CPP_MODULE_MAP.matches(sourceFile.getExecPath())) { + options.add("-x"); + options.add("c++"); + } else if (CppFileTypes.CPP_HEADER.matches(sourceFile.getExecPath())) { + // TODO(bazel-team): Read the compiler flag settings out of the CROSSTOOL file. + // TODO(bazel-team): Handle C headers that probably don't work in C++ mode. + if (features.contains(CppRuleClasses.PARSE_HEADERS)) { + options.add("-x"); + options.add("c++-header"); + // Specifying -x c++-header will make clang/gcc create precompiled + // headers, which we suppress with -fsyntax-only. + options.add("-fsyntax-only"); + } else if (features.contains(CppRuleClasses.PREPROCESS_HEADERS)) { + options.add("-E"); + options.add("-x"); + options.add("c++"); + } else { + // CcCommon.collectCAndCppSources() ensures we do not add headers to + // the compilation artifacts unless either 'parse_headers' or + // 'preprocess_headers' is set. + throw new IllegalStateException(); + } + } + + for (PathFragment quoteIncludePath : context.getQuoteIncludeDirs()) { + // "-iquote" is a gcc-specific option. For C compilers that don't support "-iquote", + // we should instead use "-I". + options.add("-iquote"); + options.add(quoteIncludePath.getSafePathString()); + } + for (PathFragment includePath : context.getIncludeDirs()) { + options.add("-I" + includePath.getSafePathString()); + } + for (PathFragment systemIncludePath : context.getSystemIncludeDirs()) { + options.add("-isystem"); + options.add(systemIncludePath.getSafePathString()); + } + + CppConfiguration toolchain = cppConfiguration; + + // pluginOpts has to be added before defaultCopts because -fplugin must precede -plugin-arg. + options.addAll(pluginOpts); + addFilteredOptions(options, toolchain.getCompilerOptions(features)); + + // Enable instrumentation if requested. + if (isInstrumented) { + addFilteredOptions(options, ImmutableList.of("-fprofile-arcs", "-ftest-coverage")); + } + + String sourceFilename = sourceFile.getExecPathString(); + if (CppFileTypes.C_SOURCE.matches(sourceFilename)) { + addFilteredOptions(options, toolchain.getCOptions()); + } + if (CppFileTypes.CPP_SOURCE.matches(sourceFilename) + || CppFileTypes.CPP_HEADER.matches(sourceFilename) + || CppFileTypes.CPP_MODULE_MAP.matches(sourceFilename)) { + addFilteredOptions(options, toolchain.getCxxOptions(features)); + } + + // Users don't expect the explicit copts to be filtered by coptsFilter, add them verbatim. + options.addAll(copts); + + for (String warn : cppConfiguration.getCWarns()) { + options.add("-W" + warn); + } + for (String define : context.getDefines()) { + options.add("-D" + define); + } + + // Stamp FDO builds with FDO subtype string + if (fdoBuildStamp != null) { + options.add("-D" + CppConfiguration.FDO_STAMP_MACRO + "=\"" + fdoBuildStamp + "\""); + } + + options.addAll(toolchain.getUnfilteredCompilerOptions(features)); + + // GCC gives randomized names to symbols which are defined in + // an anonymous namespace but have external linkage. To make + // computation of these deterministic, we want to override the + // default seed for the random number generator. It's safe to use + // any value which differs for all translation units; we use the + // path to the object file. + options.add("-frandom-seed=" + outputFile.getExecPathString()); + + // Add the options of --per_file_copt, if the label or the base name of the source file + // matches the specified regular expression filter. + for (PerLabelOptions perLabelOptions : cppConfiguration.getPerFileCopts()) { + if ((sourceLabel != null && perLabelOptions.isIncluded(sourceLabel)) + || perLabelOptions.isIncluded(sourceFile)) { + options.addAll(perLabelOptions.getOptions()); + } + } + + // Enable <object>.d file generation. + if (dotdFile != null) { + // Gcc options: + // -MD turns on .d file output as a side-effect (doesn't imply -E) + // -MM[D] enables user includes only, not system includes + // -MF <name> specifies the dotd file name + // Issues: + // -M[M] alone subverts actual .o output (implies -E) + // -M[M]D alone breaks some of the .d naming assumptions + // This combination gets user and system includes with specified name: + // -MD -MF <name> + options.add("-MD"); + options.add("-MF"); + options.add(dotdFile.getSafeExecPath().getPathString()); + } + + if (cppModuleMap != null && (compileHeaderModules || enableLayeringCheck)) { + options.add("-Xclang-only=-fmodule-maps"); + options.add("-Xclang-only=-fmodule-name=" + cppModuleMap.getName()); + options.add("-Xclang-only=-fmodule-map-file=" + + cppModuleMap.getArtifact().getExecPathString()); + options.add("-Xclang=-fno-modules-implicit-maps"); + + if (compileHeaderModules) { + options.add("-Xclang-only=-fmodules"); + if (CppFileTypes.CPP_MODULE_MAP.matches(sourceFilename)) { + options.add("-Xclang=-emit-module"); + options.add("-Xcrosstool-module-compilation"); + } + // Select .pcm inputs to pass on the command line depending on whether + // we are in pic or non-pic mode. + // TODO(bazel-team): We want to add these to the compile even if the + // current target is not built as a module; currently that implies + // passing -fmodules to the compiler, which is experimental; thus, we + // do not use the header modules files for now if the current + // compilation is not modules enabled on its own. + boolean pic = copts.contains("-fPIC"); + for (Artifact source : context.getAdditionalInputs()) { + if ((pic && source.getFilename().endsWith(".pic.pcm")) || (!pic + && !source.getFilename().endsWith(".pic.pcm") + && source.getFilename().endsWith(".pcm"))) { + options.add("-Xclang=-fmodule-file=" + source.getExecPathString()); + } + } + } + if (enableLayeringCheck) { + options.add("-Xclang-only=-fmodules-strict-decluse"); + } + } + + if (FileType.contains(outputFile, CppFileTypes.ASSEMBLER, CppFileTypes.PIC_ASSEMBLER)) { + options.add("-S"); + } else if (FileType.contains(outputFile, CppFileTypes.PREPROCESSED_C, + CppFileTypes.PREPROCESSED_CPP, CppFileTypes.PIC_PREPROCESSED_C, + CppFileTypes.PIC_PREPROCESSED_CPP)) { + options.add("-E"); + } + + if (cppConfiguration.useFission()) { + options.add("-gsplit-dwarf"); + } + + options.addAll(featureConfiguration.getCommandLine(getActionName(), + ImmutableMultimap.<String, String>of())); + return options; + } + + // For each option in 'in', add it to 'out' unless it is matched by the 'coptsFilter' regexp. + private void addFilteredOptions(List<String> out, List<String> in) { + Iterables.addAll(out, Iterables.filter(in, coptsFilter)); + } + } + + /** + * A reference to a .d file. There are two modes: + * <ol> + * <li>an Artifact that represents a real on-disk file + * <li>just an execPath that refers to a virtual .d file that is not written to disk + * </ol> + */ + public static class DotdFile { + private final Artifact artifact; + private final PathFragment execPath; + + public DotdFile(Artifact artifact) { + this.artifact = artifact; + this.execPath = null; + } + + public DotdFile(PathFragment execPath) { + this.artifact = null; + this.execPath = execPath; + } + + /** + * @return the Artifact or null + */ + public Artifact artifact() { + return artifact; + } + + /** + * @return Gets the execPath regardless of whether this is a real Artifact + */ + public PathFragment getSafeExecPath() { + return execPath == null ? artifact.getExecPath() : execPath; + } + + /** + * @return the on-disk location of the .d file or null + */ + public Path getPath() { + return artifact.getPath(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java new file mode 100644 index 0000000..0d8da3e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java
@@ -0,0 +1,439 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Functions; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppCompileAction.DotdFile; +import com.google.devtools.build.lib.rules.cpp.CppCompileAction.IncludeResolver; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.UUID; +import java.util.regex.Pattern; + +/** + * Builder class to construct C++ compile actions. + */ +public class CppCompileActionBuilder { + public static final UUID GUID = UUID.fromString("cee5db0a-d2ad-4c69-9b81-97c936a29075"); + + private final ActionOwner owner; + private final List<String> features = new ArrayList<>(); + private CcToolchainFeatures.FeatureConfiguration featureConfiguration; + private final Artifact sourceFile; + private final Label sourceLabel; + private final NestedSetBuilder<Artifact> mandatoryInputsBuilder; + private NestedSetBuilder<Artifact> pluginInputsBuilder; + private Artifact optionalSourceFile; + private Artifact outputFile; + private PathFragment tempOutputFile; + private DotdFile dotdFile; + private Artifact gcnoFile; + private final BuildConfiguration configuration; + private CppCompilationContext context = CppCompilationContext.EMPTY; + private final List<String> copts = new ArrayList<>(); + private final List<String> pluginOpts = new ArrayList<>(); + private final List<Pattern> nocopts = new ArrayList<>(); + private AnalysisEnvironment analysisEnvironment; + private ImmutableList<PathFragment> extraSystemIncludePrefixes = ImmutableList.of(); + private boolean enableLayeringCheck; + private boolean compileHeaderModules; + private String fdoBuildStamp; + private IncludeResolver includeResolver = CppCompileAction.VOID_INCLUDE_RESOLVER; + private UUID actionClassId = GUID; + private Class<? extends CppCompileActionContext> actionContext; + private CppConfiguration cppConfiguration; + private ImmutableMap<Artifact, IncludeScannable> lipoScannableMap; + + /** + * Creates a builder from a rule. This also uses the configuration and + * artifact factory from the rule. + */ + public CppCompileActionBuilder(RuleContext ruleContext, Artifact sourceFile, Label sourceLabel) { + this.owner = ruleContext.getActionOwner(); + this.actionContext = CppCompileActionContext.class; + this.cppConfiguration = ruleContext.getFragment(CppConfiguration.class); + this.analysisEnvironment = ruleContext.getAnalysisEnvironment(); + this.sourceFile = sourceFile; + this.sourceLabel = sourceLabel; + this.configuration = ruleContext.getConfiguration(); + this.mandatoryInputsBuilder = NestedSetBuilder.stableOrder(); + this.pluginInputsBuilder = NestedSetBuilder.stableOrder(); + this.lipoScannableMap = getLipoScannableMap(ruleContext); + + features.addAll(ruleContext.getFeatures()); + } + + private static ImmutableMap<Artifact, IncludeScannable> getLipoScannableMap( + RuleContext ruleContext) { + if (!ruleContext.getFragment(CppConfiguration.class).isLipoOptimization()) { + return null; + } + + LipoContextProvider provider = ruleContext.getPrerequisite( + ":lipo_context_collector", Mode.DONT_CHECK, LipoContextProvider.class); + return provider.getIncludeScannables(); + } + + /** + * Creates a builder for an owner that is not required to be rule. + */ + public CppCompileActionBuilder( + ActionOwner owner, AnalysisEnvironment analysisEnvironment, Artifact sourceFile, + Label sourceLabel, BuildConfiguration configuration) { + this.owner = owner; + this.actionContext = CppCompileActionContext.class; + this.cppConfiguration = configuration.getFragment(CppConfiguration.class); + this.analysisEnvironment = analysisEnvironment; + this.sourceFile = sourceFile; + this.sourceLabel = sourceLabel; + this.configuration = configuration; + this.mandatoryInputsBuilder = NestedSetBuilder.stableOrder(); + this.pluginInputsBuilder = NestedSetBuilder.stableOrder(); + this.lipoScannableMap = ImmutableMap.of(); + } + + /** + * Creates a builder that is a copy of another builder. + */ + public CppCompileActionBuilder(CppCompileActionBuilder other) { + this.owner = other.owner; + this.features.addAll(other.features); + this.featureConfiguration = other.featureConfiguration; + this.sourceFile = other.sourceFile; + this.sourceLabel = other.sourceLabel; + this.mandatoryInputsBuilder = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(other.mandatoryInputsBuilder.build()); + this.pluginInputsBuilder = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(other.pluginInputsBuilder.build()); + this.optionalSourceFile = other.optionalSourceFile; + this.outputFile = other.outputFile; + this.tempOutputFile = other.tempOutputFile; + this.dotdFile = other.dotdFile; + this.gcnoFile = other.gcnoFile; + this.configuration = other.configuration; + this.context = other.context; + this.copts.addAll(other.copts); + this.pluginOpts.addAll(other.pluginOpts); + this.nocopts.addAll(other.nocopts); + this.analysisEnvironment = other.analysisEnvironment; + this.extraSystemIncludePrefixes = ImmutableList.copyOf(other.extraSystemIncludePrefixes); + this.enableLayeringCheck = other.enableLayeringCheck; + this.compileHeaderModules = other.compileHeaderModules; + this.includeResolver = other.includeResolver; + this.actionClassId = other.actionClassId; + this.actionContext = other.actionContext; + this.cppConfiguration = other.cppConfiguration; + this.lipoScannableMap = other.lipoScannableMap; + } + + public PathFragment getTempOutputFile() { + return tempOutputFile; + } + + public Artifact getSourceFile() { + return sourceFile; + } + + public CppCompilationContext getContext() { + return context; + } + + public NestedSet<Artifact> getMandatoryInputs() { + return mandatoryInputsBuilder.build(); + } + + /** + * Returns the .dwo output file that matches the specified .o output file. If Fission mode + * isn't enabled for this build, this is null (we don't produce .dwo files in that case). + */ + private static Artifact getDwoFile(Artifact outputFile, AnalysisEnvironment artifactFactory, + CppConfiguration cppConfiguration) { + + // Only create .dwo's for .o compilations (i.e. not .ii or .S). + boolean isObjectOutput = CppFileTypes.OBJECT_FILE.matches(outputFile.getExecPath()) + || CppFileTypes.PIC_OBJECT_FILE.matches(outputFile.getExecPath()); + + // Note configurations can be null for tests. + if (cppConfiguration != null && cppConfiguration.useFission() && isObjectOutput) { + return artifactFactory.getDerivedArtifact( + FileSystemUtils.replaceExtension(outputFile.getRootRelativePath(), ".dwo"), + outputFile.getRoot()); + } else { + return null; + } + } + + private static Predicate<String> getNocoptPredicate(Collection<Pattern> patterns) { + final ImmutableList<Pattern> finalPatterns = ImmutableList.copyOf(patterns); + if (finalPatterns.isEmpty()) { + return Predicates.alwaysTrue(); + } else { + return new Predicate<String>() { + @Override + public boolean apply(String option) { + for (Pattern pattern : finalPatterns) { + if (pattern.matcher(option).matches()) { + return false; + } + } + + return true; + } + }; + } + } + + private Iterable<IncludeScannable> getLipoScannables(NestedSet<Artifact> realMandatoryInputs) { + return lipoScannableMap == null ? ImmutableList.<IncludeScannable>of() : Iterables.filter( + Iterables.transform( + Iterables.filter( + FileType.filter( + realMandatoryInputs, + CppFileTypes.C_SOURCE, CppFileTypes.CPP_SOURCE, + CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR), + Predicates.not(Predicates.equalTo(getSourceFile()))), + Functions.forMap(lipoScannableMap, null)), + Predicates.notNull()); + } + + /** + * Builds the Action as configured and returns the to be generated Artifact. + * + * <p>This method may be called multiple times to create multiple compile + * actions (usually after calling some setters to modify the generated + * action). + */ + public CppCompileAction build() { + // Configuration can be null in tests. + NestedSetBuilder<Artifact> realMandatoryInputsBuilder = NestedSetBuilder.compileOrder(); + realMandatoryInputsBuilder.addTransitive(mandatoryInputsBuilder.build()); + if (tempOutputFile == null && configuration != null + && !configuration.getFragment(CppConfiguration.class).shouldScanIncludes()) { + realMandatoryInputsBuilder.addTransitive(context.getDeclaredIncludeSrcs()); + } + realMandatoryInputsBuilder.addTransitive(context.getAdditionalInputs()); + realMandatoryInputsBuilder.addTransitive(pluginInputsBuilder.build()); + realMandatoryInputsBuilder.add(sourceFile); + boolean fake = tempOutputFile != null; + + // Copying the collections is needed to make the builder reusable. + if (fake) { + return new FakeCppCompileAction(owner, ImmutableList.copyOf(features), featureConfiguration, + sourceFile, sourceLabel, realMandatoryInputsBuilder.build(), outputFile, tempOutputFile, + dotdFile, configuration, cppConfiguration, context, ImmutableList.copyOf(copts), + ImmutableList.copyOf(pluginOpts), getNocoptPredicate(nocopts), + extraSystemIncludePrefixes, enableLayeringCheck, fdoBuildStamp); + } else { + NestedSet<Artifact> realMandatoryInputs = realMandatoryInputsBuilder.build(); + + return new CppCompileAction(owner, ImmutableList.copyOf(features), featureConfiguration, + sourceFile, sourceLabel, realMandatoryInputs, outputFile, dotdFile, + gcnoFile, getDwoFile(outputFile, analysisEnvironment, cppConfiguration), + optionalSourceFile, configuration, cppConfiguration, context, + actionContext, ImmutableList.copyOf(copts), + ImmutableList.copyOf(pluginOpts), + getNocoptPredicate(nocopts), + extraSystemIncludePrefixes, enableLayeringCheck, fdoBuildStamp, + includeResolver, getLipoScannables(realMandatoryInputs), actionClassId, + compileHeaderModules); + } + } + + /** + * Sets the feature configuration to be used for the action. + */ + public CppCompileActionBuilder setFeatureConfiguration( + FeatureConfiguration featureConfiguration) { + this.featureConfiguration = featureConfiguration; + return this; + } + + public CppCompileActionBuilder setIncludeResolver(IncludeResolver includeResolver) { + this.includeResolver = includeResolver; + return this; + } + + public CppCompileActionBuilder setCppConfiguration(CppConfiguration cppConfiguration) { + this.cppConfiguration = cppConfiguration; + return this; + } + + public CppCompileActionBuilder setActionContext( + Class<? extends CppCompileActionContext> actionContext) { + this.actionContext = actionContext; + return this; + } + + public CppCompileActionBuilder setActionClassId(UUID uuid) { + this.actionClassId = uuid; + return this; + } + + public CppCompileActionBuilder setExtraSystemIncludePrefixes( + Collection<PathFragment> extraSystemIncludePrefixes) { + this.extraSystemIncludePrefixes = ImmutableList.copyOf(extraSystemIncludePrefixes); + return this; + } + + public CppCompileActionBuilder addPluginInput(Artifact artifact) { + pluginInputsBuilder.add(artifact); + return this; + } + + public CppCompileActionBuilder clearPluginInputs() { + pluginInputsBuilder = NestedSetBuilder.stableOrder(); + return this; + } + + /** + * Set an optional source file (usually with metadata of the main source file). The optional + * source file can only be set once, whether via this method or through the constructor + * {@link #CppCompileActionBuilder(CppCompileActionBuilder)}. + */ + public CppCompileActionBuilder addOptionalSourceFile(Artifact artifact) { + Preconditions.checkState(optionalSourceFile == null, "%s %s", optionalSourceFile, artifact); + optionalSourceFile = artifact; + return this; + } + + public CppCompileActionBuilder addMandatoryInputs(Iterable<Artifact> artifacts) { + mandatoryInputsBuilder.addAll(artifacts); + return this; + } + + public CppCompileActionBuilder addTransitiveMandatoryInputs(NestedSet<Artifact> artifacts) { + mandatoryInputsBuilder.addTransitive(artifacts); + return this; + } + + public CppCompileActionBuilder setOutputFile(Artifact outputFile) { + this.outputFile = outputFile; + return this; + } + + /** + * The temp output file is not an artifact, since it does not appear in the outputs of the + * action. + * + * <p>This is theoretically a problem if that file already existed before, since then Blaze + * does not delete it before executing the rule, but 1. that only applies for local + * execution which does not happen very often and 2. it is only a problem if the compiler is + * affected by the presence of this file, which it should not be. + */ + public CppCompileActionBuilder setTempOutputFile(PathFragment tempOutputFile) { + this.tempOutputFile = tempOutputFile; + return this; + } + + @VisibleForTesting + public CppCompileActionBuilder setDotdFileForTesting(Artifact dotdFile) { + this.dotdFile = new DotdFile(dotdFile); + return this; + } + + public CppCompileActionBuilder setDotdFile(PathFragment outputName, String extension, + RuleContext ruleContext) { + if (configuration.getFragment(CppConfiguration.class).getInmemoryDotdFiles()) { + // Just set the path, no artifact is constructed + PathFragment file = FileSystemUtils.replaceExtension(outputName, extension); + Root root = configuration.getBinDirectory(); + dotdFile = new DotdFile(root.getExecPath().getRelative(file)); + } else { + dotdFile = new DotdFile(ruleContext.getRelatedArtifact(outputName, extension)); + } + return this; + } + + public CppCompileActionBuilder setGcnoFile(Artifact gcnoFile) { + this.gcnoFile = gcnoFile; + return this; + } + + public CppCompileActionBuilder addCopt(String copt) { + copts.add(copt); + return this; + } + + public CppCompileActionBuilder addPluginOpt(String opt) { + pluginOpts.add(opt); + return this; + } + + public CppCompileActionBuilder clearPluginOpts() { + pluginOpts.clear(); + return this; + } + + public CppCompileActionBuilder addCopts(Iterable<? extends String> copts) { + Iterables.addAll(this.copts, copts); + return this; + } + + public CppCompileActionBuilder addCopts(int position, Iterable<? extends String> copts) { + this.copts.addAll(position, ImmutableList.copyOf(copts)); + return this; + } + + public CppCompileActionBuilder addNocopts(Pattern nocopts) { + this.nocopts.add(nocopts); + return this; + } + + public CppCompileActionBuilder setContext(CppCompilationContext context) { + this.context = context; + return this; + } + + public CppCompileActionBuilder setEnableLayeringCheck(boolean enableLayeringCheck) { + this.enableLayeringCheck = enableLayeringCheck; + return this; + } + + /** + * Sets whether the CompileAction should use header modules for its compilation. + */ + public CppCompileActionBuilder setCompileHeaderModules(boolean compileHeaderModules) { + this.compileHeaderModules = compileHeaderModules; + return this; + } + + public CppCompileActionBuilder setFdoBuildStamp(String fdoBuildStamp) { + this.fdoBuildStamp = fdoBuildStamp; + return this; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionContext.java new file mode 100644 index 0000000..4bbfa44 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionContext.java
@@ -0,0 +1,84 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.ActionContextMarker; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ResourceSet; + +import java.io.IOException; +import java.util.Collection; + +import javax.annotation.Nullable; + +/** + * Context for compiling plain C++. + */ +@ActionContextMarker(name = "C++") +public interface CppCompileActionContext extends ActionContext { + /** + * Reply for the execution of a C++ compilation. + */ + public interface Reply { + /** + * Returns the contents of the .d file. + */ + byte[] getContents() throws IOException; + } + + /** Does include scanning to find the list of files needed to execute the action. */ + public Collection<? extends ActionInput> findAdditionalInputs(CppCompileAction action, + ActionExecutionContext actionExecutionContext) + throws ExecException, InterruptedException, ActionExecutionException; + + /** + * Executes the given action and return the reply of the executor. + */ + Reply execWithReply(CppCompileAction action, + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException; + + /** + * Returns the executor reply from an exec exception, if available. + */ + @Nullable Reply getReplyFromException( + ExecException e, CppCompileAction action); + + /** + * Returns the estimated resource consumption of the action. + */ + ResourceSet estimateResourceConsumption(CppCompileAction action); + + /** + * Returns where the action actually runs. + */ + String strategyLocality(); + + /** + * Returns whether include scanning needs to be run. + */ + boolean needsIncludeScanning(); + + /** + * Returns the include files that should be shipped to the executor in addition the ones that + * were declared. + */ + Collection<Artifact> getScannedIncludeFiles( + CppCompileAction action, ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java new file mode 100644 index 0000000..c5cc9a5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java
@@ -0,0 +1,1691 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.PackageRootResolver; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.ViewCreationFailedException; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.CompilationMode; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.analysis.config.PerLabelOptions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.rules.cpp.CppConfigurationLoader.CppConfigurationParameters; +import com.google.devtools.build.lib.rules.cpp.FdoSupport.FdoException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.util.IncludeScanningUtil; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LinkingModeFlags; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode; +import com.google.devtools.build.skyframe.SkyFunction.Environment; +import com.google.devtools.common.options.OptionsParsingException; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipException; + +/** + * This class represents the C/C++ parts of the {@link BuildConfiguration}, + * including the host architecture, target architecture, compiler version, and + * a standard library version. It has information about the tools locations and + * the flags required for compiling. + */ +@SkylarkModule(name = "cpp", doc = "A configuration fragment for C++") +@Immutable +public class CppConfiguration extends BuildConfiguration.Fragment { + /** + * An enumeration of all the tools that comprise a toolchain. + */ + public enum Tool { + AR("ar"), + CPP("cpp"), + GCC("gcc"), + GCOV("gcov"), + GCOVTOOL("gcov-tool"), + LD("ld"), + NM("nm"), + OBJCOPY("objcopy"), + OBJDUMP("objdump"), + STRIP("strip"), + DWP("dwp"); + + private final String namePart; + + private Tool(String namePart) { + this.namePart = namePart; + } + + public String getNamePart() { + return namePart; + } + } + + /** + * Values for the --hdrs_check option. + */ + public static enum HeadersCheckingMode { + /** Legacy behavior: Silently allow undeclared headers. */ + LOOSE, + /** Warn about undeclared headers. */ + WARN, + /** Disallow undeclared headers. */ + STRICT + } + + /** + * --dynamic_mode parses to DynamicModeFlag, but AUTO will be translated based on platform, + * resulting in a DynamicMode value. + */ + public enum DynamicMode { OFF, DEFAULT, FULLY } + + /** + * This enumeration is used for the --strip option. + */ + public static enum StripMode { + + ALWAYS("always"), // Always strip. + SOMETIMES("sometimes"), // Strip iff compilationMode == FASTBUILD. + NEVER("never"); // Never strip. + + private final String mode; + + private StripMode(String mode) { + this.mode = mode; + } + + @Override + public String toString() { + return mode; + } + } + + /** Storage for the libc label, if given. */ + public static class LibcTop implements Serializable { + private final Label label; + + LibcTop(Label label) { + Preconditions.checkArgument(label != null); + this.label = label; + } + + public Label getLabel() { + return label; + } + + public PathFragment getSysroot() { + return label.getPackageFragment(); + } + + @Override + public String toString() { + return label.toString(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } else if (other instanceof LibcTop) { + return label.equals(((LibcTop) other).label); + } else { + return false; + } + } + + @Override + public int hashCode() { + return label.hashCode(); + } + } + + /** + * This macro will be passed as a command-line parameter (eg. -DBUILD_FDO_TYPE="LIPO"). + * For possible values see {@code CppModel.getFdoBuildStamp()}. + */ + public static final String FDO_STAMP_MACRO = "BUILD_FDO_TYPE"; + + /** + * Represents an optional flag that can be toggled using the package features mechanism. + */ + @VisibleForTesting + static class OptionalFlag implements Serializable { + private final String name; + private final List<String> flags; + + @VisibleForTesting + OptionalFlag(String name, List<String> flags) { + this.name = name; + this.flags = flags; + } + + private List<String> getFlags() { + return flags; + } + + private String getName() { + return name; + } + } + + @VisibleForTesting + static class FlagList implements Serializable { + private List<String> prefixFlags; + private List<OptionalFlag> optionalFlags; + private List<String> suffixFlags; + + @VisibleForTesting + FlagList(List<String> prefixFlags, + List<OptionalFlag> optionalFlags, + List<String> suffixFlags) { + this.prefixFlags = prefixFlags; + this.optionalFlags = optionalFlags; + this.suffixFlags = suffixFlags; + } + + @VisibleForTesting + List<String> evaluate(Collection<String> features) { + ImmutableList.Builder<String> result = ImmutableList.builder(); + result.addAll(prefixFlags); + for (OptionalFlag optionalFlag : optionalFlags) { + // The flag is added if the default is true and the flag is not specified, + // or if the default is false and the flag is specified. + if (features.contains(optionalFlag.getName())) { + result.addAll(optionalFlag.getFlags()); + } + } + + result.addAll(suffixFlags); + return result.build(); + } + } + + private final Label crosstoolTop; + private final String hostSystemName; + private final String compiler; + private final String targetCpu; + private final String targetSystemName; + private final String targetLibc; + private final LipoMode lipoMode; + private final PathFragment crosstoolTopPathFragment; + + private final String abi; + private final String abiGlibcVersion; + + private final String toolchainIdentifier; + private final String cacheKey; + + private final CcToolchainFeatures toolchainFeatures; + private final boolean supportsGoldLinker; + private final boolean supportsThinArchives; + private final boolean supportsStartEndLib; + private final boolean supportsInterfaceSharedObjects; + private final boolean supportsEmbeddedRuntimes; + private final boolean supportsFission; + + // We encode three states with two booleans: + // (1) (false false) -> no pic code + // (2) (true false) -> shared libraries as pic, but not binaries + // (3) (true true) -> both shared libraries and binaries as pic + private final boolean toolchainNeedsPic; + private final boolean usePicForBinaries; + + private final FdoSupport fdoSupport; + + // TODO(bazel-team): All these labels (except for ccCompilerRuleLabel) can be removed once the + // transition to the cc_compiler rule is complete. + private final Label libcLabel; + private final Label staticRuntimeLibsLabel; + private final Label dynamicRuntimeLibsLabel; + private final Label ccToolchainLabel; + + private final PathFragment sysroot; + private final PathFragment runtimeSysroot; + private final List<PathFragment> builtInIncludeDirectories; + + private final Map<String, PathFragment> toolPaths; + private final PathFragment ldExecutable; + + // Only used during construction. + private final List<String> commonLinkOptions; + private final ListMultimap<CompilationMode, String> linkOptionsFromCompilationMode; + private final ListMultimap<LipoMode, String> linkOptionsFromLipoMode; + private final ListMultimap<LinkingMode, String> linkOptionsFromLinkingMode; + + private final FlagList compilerFlags; + private final FlagList cxxFlags; + private final FlagList unfilteredCompilerFlags; + private final List<String> cOptions; + + private FlagList fullyStaticLinkFlags; + private FlagList mostlyStaticLinkFlags; + private FlagList mostlyStaticSharedLinkFlags; + private FlagList dynamicLinkFlags; + private FlagList dynamicLibraryLinkFlags; + private final List<String> testOnlyLinkFlags; + + private final List<String> linkOptions; + + private final List<String> objcopyOptions; + private final List<String> ldOptions; + private final List<String> arOptions; + private final List<String> arThinArchivesOptions; + + private final Map<String, String> additionalMakeVariables; + + private final CppOptions cppOptions; + + // The dynamic mode for linking. + private final DynamicMode dynamicMode; + private final boolean stripBinaries; + private final ImmutableMap<String, String> commandLineDefines; + private final String solibDirectory; + private final CompilationMode compilationMode; + private final Path execRoot; + /** + * If true, the ConfiguredTarget is only used to get the necessary cross-referenced + * CppCompilationContexts, but registering build actions is disabled. + */ + private final boolean lipoContextCollector; + private final Root greppedIncludesDirectory; + + protected CppConfiguration(CppConfigurationParameters params) + throws InvalidConfigurationException { + CrosstoolConfig.CToolchain toolchain = params.toolchain; + cppOptions = params.buildOptions.get(CppOptions.class); + this.hostSystemName = toolchain.getHostSystemName(); + this.compiler = toolchain.getCompiler(); + this.targetCpu = toolchain.getTargetCpu(); + this.lipoMode = cppOptions.getLipoMode(); + this.targetSystemName = toolchain.getTargetSystemName(); + this.targetLibc = toolchain.getTargetLibc(); + this.crosstoolTop = params.crosstoolTop; + this.ccToolchainLabel = params.ccToolchainLabel; + this.compilationMode = + params.buildOptions.get(BuildConfiguration.Options.class).compilationMode; + this.lipoContextCollector = cppOptions.lipoCollector; + this.execRoot = params.execRoot; + + // Note that the grepped includes directory is not configuration-specific; the paths of the + // files within that directory, however, are configuration-specific. + this.greppedIncludesDirectory = Root.asDerivedRoot(execRoot, + execRoot.getRelative(IncludeScanningUtil.GREPPED_INCLUDES)); + + this.crosstoolTopPathFragment = crosstoolTop.getPackageFragment(); + + try { + this.staticRuntimeLibsLabel = + crosstoolTop.getRelative(toolchain.hasStaticRuntimesFilegroup() ? + toolchain.getStaticRuntimesFilegroup() : "static-runtime-libs-" + targetCpu); + this.dynamicRuntimeLibsLabel = + crosstoolTop.getRelative(toolchain.hasDynamicRuntimesFilegroup() ? + toolchain.getDynamicRuntimesFilegroup() : "dynamic-runtime-libs-" + targetCpu); + } catch (SyntaxException e) { + // All of the above label.getRelative() calls are valid labels, and the crosstool_top + // was already checked earlier in the process. + throw new AssertionError(e); + } + + if (cppOptions.lipoMode == LipoMode.BINARY) { + // TODO(bazel-team): implement dynamic linking with LIPO + this.dynamicMode = DynamicMode.OFF; + } else { + switch (cppOptions.dynamicMode) { + case DEFAULT: + this.dynamicMode = DynamicMode.DEFAULT; break; + case OFF: this.dynamicMode = DynamicMode.OFF; break; + case FULLY: this.dynamicMode = DynamicMode.FULLY; break; + default: throw new IllegalStateException("Invalid dynamicMode."); + } + } + + this.fdoSupport = new FdoSupport( + params.buildOptions.get(CppOptions.class).fdoInstrument, params.fdoZip, + cppOptions.lipoMode, execRoot); + + this.stripBinaries = (cppOptions.stripBinaries == StripMode.ALWAYS || + (cppOptions.stripBinaries == StripMode.SOMETIMES && + compilationMode == CompilationMode.FASTBUILD)); + + CrosstoolConfigurationIdentifier crosstoolConfig = + CrosstoolConfigurationIdentifier.fromToolchain(toolchain); + Preconditions.checkState(crosstoolConfig.getCpu().equals(targetCpu)); + Preconditions.checkState(crosstoolConfig.getCompiler().equals(compiler)); + Preconditions.checkState(crosstoolConfig.getLibc().equals(targetLibc)); + + this.solibDirectory = "_solib_" + targetCpu; + + this.toolchainIdentifier = toolchain.getToolchainIdentifier(); + this.cacheKey = this + ":" + crosstoolTop + ":" + params.cacheKeySuffix + ":" + + lipoContextCollector; + + this.toolchainFeatures = new CcToolchainFeatures(toolchain); + this.supportsGoldLinker = toolchain.getSupportsGoldLinker(); + this.supportsThinArchives = toolchain.getSupportsThinArchives(); + this.supportsStartEndLib = toolchain.getSupportsStartEndLib(); + this.supportsInterfaceSharedObjects = toolchain.getSupportsInterfaceSharedObjects(); + this.supportsEmbeddedRuntimes = toolchain.getSupportsEmbeddedRuntimes(); + this.supportsFission = toolchain.getSupportsFission(); + this.toolchainNeedsPic = toolchain.getNeedsPic(); + this.usePicForBinaries = + toolchain.getNeedsPic() && compilationMode != CompilationMode.OPT; + + this.toolPaths = Maps.newHashMap(); + for (CrosstoolConfig.ToolPath tool : toolchain.getToolPathList()) { + PathFragment path = new PathFragment(tool.getPath()); + if (!path.isNormalized()) { + throw new IllegalArgumentException("The include path '" + tool.getPath() + + "' is not normalized."); + } + toolPaths.put(tool.getName(), crosstoolTopPathFragment.getRelative(path)); + } + + if (toolPaths.isEmpty()) { + // If no paths are specified, we just use the names of the tools as the path. + for (Tool tool : Tool.values()) { + toolPaths.put(tool.getNamePart(), + crosstoolTopPathFragment.getRelative(tool.getNamePart())); + } + } else { + Iterable<Tool> neededTools = Iterables.filter(EnumSet.allOf(Tool.class), + new Predicate<Tool>() { + @Override + public boolean apply(Tool tool) { + if (tool == Tool.DWP) { + // When fission is unsupported, don't check for the dwp tool. + return supportsFission(); + } else if (tool == Tool.GCOVTOOL) { + // gcov-tool is optional, don't check whether it's present + return false; + } else { + return true; + } + } + }); + for (Tool tool : neededTools) { + if (!toolPaths.containsKey(tool.getNamePart())) { + throw new IllegalArgumentException("Tool path for '" + tool.getNamePart() + + "' is missing"); + } + } + } + + // We can't use an ImmutableMap.Builder here; we need the ability (at least + // in tests) to add entries with keys that are already in the map, and only + // HashMap supports this (by replacing the existing entry under the key). + Map<String, String> commandLineDefinesBuilder = new HashMap<>(); + for (Map.Entry<String, String> define : cppOptions.commandLineDefinedVariables) { + commandLineDefinesBuilder.put(define.getKey(), define.getValue()); + } + commandLineDefines = ImmutableMap.copyOf(commandLineDefinesBuilder); + + ListMultimap<CompilationMode, String> cFlags = ArrayListMultimap.create(); + ListMultimap<CompilationMode, String> cxxFlags = ArrayListMultimap.create(); + linkOptionsFromCompilationMode = ArrayListMultimap.create(); + for (CrosstoolConfig.CompilationModeFlags flags : toolchain.getCompilationModeFlagsList()) { + // Remove this when CROSSTOOL files no longer contain 'coverage'. + if (flags.getMode() == CrosstoolConfig.CompilationMode.COVERAGE) { + continue; + } + CompilationMode realmode = importCompilationMode(flags.getMode()); + cFlags.putAll(realmode, flags.getCompilerFlagList()); + cxxFlags.putAll(realmode, flags.getCxxFlagList()); + linkOptionsFromCompilationMode.putAll(realmode, flags.getLinkerFlagList()); + } + + ListMultimap<LipoMode, String> lipoCFlags = ArrayListMultimap.create(); + ListMultimap<LipoMode, String> lipoCxxFlags = ArrayListMultimap.create(); + linkOptionsFromLipoMode = ArrayListMultimap.create(); + for (CrosstoolConfig.LipoModeFlags flags : toolchain.getLipoModeFlagsList()) { + LipoMode realmode = flags.getMode(); + lipoCFlags.putAll(realmode, flags.getCompilerFlagList()); + lipoCxxFlags.putAll(realmode, flags.getCxxFlagList()); + linkOptionsFromLipoMode.putAll(realmode, flags.getLinkerFlagList()); + } + + linkOptionsFromLinkingMode = ArrayListMultimap.create(); + for (LinkingModeFlags flags : toolchain.getLinkingModeFlagsList()) { + LinkingMode realmode = importLinkingMode(flags.getMode()); + linkOptionsFromLinkingMode.putAll(realmode, flags.getLinkerFlagList()); + } + + this.commonLinkOptions = ImmutableList.copyOf(toolchain.getLinkerFlagList()); + dynamicLibraryLinkFlags = new FlagList( + ImmutableList.copyOf(toolchain.getDynamicLibraryLinkerFlagList()), + convertOptionalOptions(toolchain.getOptionalDynamicLibraryLinkerFlagList()), + Collections.<String>emptyList()); + this.objcopyOptions = ImmutableList.copyOf(toolchain.getObjcopyEmbedFlagList()); + this.ldOptions = ImmutableList.copyOf(toolchain.getLdEmbedFlagList()); + this.arOptions = copyOrDefaultIfEmpty(toolchain.getArFlagList(), "rcsD"); + this.arThinArchivesOptions = copyOrDefaultIfEmpty( + toolchain.getArThinArchivesFlagList(), "rcsDT"); + + this.abi = toolchain.getAbiVersion(); + this.abiGlibcVersion = toolchain.getAbiLibcVersion(); + + // The default value for optional string attributes is the empty string. + PathFragment defaultSysroot = toolchain.getBuiltinSysroot().length() == 0 + ? null + : new PathFragment(toolchain.getBuiltinSysroot()); + if ((defaultSysroot != null) && !defaultSysroot.isNormalized()) { + throw new IllegalArgumentException("The built-in sysroot '" + defaultSysroot + + "' is not normalized."); + } + + if ((cppOptions.libcTop != null) && (defaultSysroot == null)) { + throw new InvalidConfigurationException("The selected toolchain " + toolchainIdentifier + + " does not support setting --grte_top."); + } + LibcTop libcTop = cppOptions.libcTop; + if ((libcTop == null) && !toolchain.getDefaultGrteTop().isEmpty()) { + try { + libcTop = new CppOptions.LibcTopConverter().convert(toolchain.getDefaultGrteTop()); + } catch (OptionsParsingException e) { + throw new InvalidConfigurationException(e.getMessage(), e); + } + } + if ((libcTop != null) && (libcTop.getLabel() != null)) { + libcLabel = libcTop.getLabel(); + } else { + libcLabel = null; + } + + ImmutableList.Builder<PathFragment> builtInIncludeDirectoriesBuilder + = ImmutableList.builder(); + sysroot = libcTop == null ? defaultSysroot : libcTop.getSysroot(); + for (String s : toolchain.getCxxBuiltinIncludeDirectoryList()) { + builtInIncludeDirectoriesBuilder.add( + resolveIncludeDir(s, sysroot, crosstoolTopPathFragment)); + } + builtInIncludeDirectories = builtInIncludeDirectoriesBuilder.build(); + + // The runtime sysroot should really be set from --grte_top. However, currently libc has no + // way to set the sysroot. The CROSSTOOL file does set the runtime sysroot, in the + // builtin_sysroot field. This implies that you can not arbitrarily mix and match Crosstool + // and libc versions, you must always choose compatible ones. + runtimeSysroot = defaultSysroot; + + String sysrootFlag; + if (sysroot != null && !sysroot.equals(defaultSysroot)) { + // Only specify the --sysroot option if it is different from the built-in one. + sysrootFlag = "--sysroot=" + sysroot; + } else { + sysrootFlag = null; + } + + ImmutableList.Builder<String> unfilteredCoptsBuilder = ImmutableList.builder(); + if (sysrootFlag != null) { + unfilteredCoptsBuilder.add(sysrootFlag); + } + unfilteredCoptsBuilder.addAll(toolchain.getUnfilteredCxxFlagList()); + unfilteredCompilerFlags = new FlagList( + unfilteredCoptsBuilder.build(), + convertOptionalOptions(toolchain.getOptionalUnfilteredCxxFlagList()), + Collections.<String>emptyList()); + + ImmutableList.Builder<String> linkoptsBuilder = ImmutableList.builder(); + linkoptsBuilder.addAll(cppOptions.linkoptList); + if (cppOptions.experimentalOmitfp) { + linkoptsBuilder.add("-Wl,--eh-frame-hdr"); + } + if (sysrootFlag != null) { + linkoptsBuilder.add(sysrootFlag); + } + this.linkOptions = linkoptsBuilder.build(); + + ImmutableList.Builder<String> coptsBuilder = ImmutableList.<String>builder() + .addAll(toolchain.getCompilerFlagList()) + .addAll(cFlags.get(compilationMode)) + .addAll(lipoCFlags.get(cppOptions.getLipoMode())); + if (cppOptions.experimentalOmitfp) { + coptsBuilder.add("-fomit-frame-pointer"); + coptsBuilder.add("-fasynchronous-unwind-tables"); + coptsBuilder.add("-DNO_FRAME_POINTER"); + } + this.compilerFlags = new FlagList( + coptsBuilder.build(), + convertOptionalOptions(toolchain.getOptionalCompilerFlagList()), + cppOptions.coptList); + + this.cOptions = ImmutableList.copyOf(cppOptions.conlyoptList); + + ImmutableList.Builder<String> cxxOptsBuilder = ImmutableList.<String>builder() + .addAll(toolchain.getCxxFlagList()) + .addAll(cxxFlags.get(compilationMode)) + .addAll(lipoCxxFlags.get(cppOptions.getLipoMode())); + + this.cxxFlags = new FlagList( + cxxOptsBuilder.build(), + convertOptionalOptions(toolchain.getOptionalCxxFlagList()), + cppOptions.cxxoptList); + + this.ldExecutable = getToolPathFragment(CppConfiguration.Tool.LD); + + boolean stripBinaries = (cppOptions.stripBinaries == StripMode.ALWAYS) || + ((cppOptions.stripBinaries == StripMode.SOMETIMES) && + (compilationMode == CompilationMode.FASTBUILD)); + + fullyStaticLinkFlags = new FlagList( + configureLinkerOptions(compilationMode, lipoMode, LinkingMode.FULLY_STATIC, + ldExecutable, stripBinaries), + convertOptionalOptions(toolchain.getOptionalLinkerFlagList()), + Collections.<String>emptyList()); + mostlyStaticLinkFlags = new FlagList( + configureLinkerOptions(compilationMode, lipoMode, LinkingMode.MOSTLY_STATIC, + ldExecutable, stripBinaries), + convertOptionalOptions(toolchain.getOptionalLinkerFlagList()), + Collections.<String>emptyList()); + mostlyStaticSharedLinkFlags = new FlagList( + configureLinkerOptions(compilationMode, lipoMode, + LinkingMode.MOSTLY_STATIC_LIBRARIES, ldExecutable, stripBinaries), + convertOptionalOptions(toolchain.getOptionalLinkerFlagList()), + Collections.<String>emptyList()); + dynamicLinkFlags = new FlagList( + configureLinkerOptions(compilationMode, lipoMode, LinkingMode.DYNAMIC, + ldExecutable, stripBinaries), + convertOptionalOptions(toolchain.getOptionalLinkerFlagList()), + Collections.<String>emptyList()); + testOnlyLinkFlags = ImmutableList.copyOf(toolchain.getTestOnlyLinkerFlagList()); + + Map<String, String> makeVariablesBuilder = new HashMap<>(); + // The following are to be used to allow some build rules to avoid the limits on stack frame + // sizes and variable-length arrays. Ensure that these are always set. + makeVariablesBuilder.put("STACK_FRAME_UNLIMITED", ""); + makeVariablesBuilder.put("CC_FLAGS", ""); + for (CrosstoolConfig.MakeVariable variable : toolchain.getMakeVariableList()) { + makeVariablesBuilder.put(variable.getName(), variable.getValue()); + } + if (sysrootFlag != null) { + String ccFlags = makeVariablesBuilder.get("CC_FLAGS"); + ccFlags = ccFlags.isEmpty() ? sysrootFlag : ccFlags + " " + sysrootFlag; + makeVariablesBuilder.put("CC_FLAGS", ccFlags); + } + this.additionalMakeVariables = ImmutableMap.copyOf(makeVariablesBuilder); + } + + private List<OptionalFlag> convertOptionalOptions( + List<CrosstoolConfig.CToolchain.OptionalFlag> optionalFlagList) + throws IllegalArgumentException { + List<OptionalFlag> result = new ArrayList<>(); + + for (CrosstoolConfig.CToolchain.OptionalFlag crosstoolOptionalFlag : optionalFlagList) { + String name = crosstoolOptionalFlag.getDefaultSettingName(); + result.add(new OptionalFlag( + name, + ImmutableList.copyOf(crosstoolOptionalFlag.getFlagList()))); + } + + return result; + } + + private static ImmutableList<String> copyOrDefaultIfEmpty(List<String> list, + String defaultValue) { + return list.isEmpty() ? ImmutableList.of(defaultValue) : ImmutableList.copyOf(list); + } + + @VisibleForTesting + static CompilationMode importCompilationMode(CrosstoolConfig.CompilationMode mode) { + return CompilationMode.valueOf(mode.name()); + } + + @VisibleForTesting + static LinkingMode importLinkingMode(CrosstoolConfig.LinkingMode mode) { + return LinkingMode.valueOf(mode.name()); + } + + private static final PathFragment SYSROOT_FRAGMENT = new PathFragment("%sysroot%"); + + /** + * Resolve the given include directory. If it is not absolute, it is + * interpreted relative to the crosstool top. If it starts with %sysroot%/, + * that part is replaced with the actual sysroot. + */ + static PathFragment resolveIncludeDir(String s, PathFragment sysroot, + PathFragment crosstoolTopPathFragment) { + PathFragment path = new PathFragment(s); + if (!path.isNormalized()) { + throw new IllegalArgumentException("The include path '" + s + "' is not normalized."); + } + if (path.startsWith(SYSROOT_FRAGMENT)) { + if (sysroot == null) { + throw new IllegalArgumentException("A %sysroot% prefix is only allowed if the " + + "default_sysroot option is set"); + } + return sysroot.getRelative(path.relativeTo(SYSROOT_FRAGMENT)); + } else { + return crosstoolTopPathFragment.getRelative(path); + } + } + + /** + * Returns the configuration-independent grepped-includes directory. + */ + public Root getGreppedIncludesDirectory() { + return greppedIncludesDirectory; + } + + @VisibleForTesting + List<String> configureLinkerOptions( + CompilationMode compilationMode, LipoMode lipoMode, LinkingMode linkingMode, + PathFragment ldExecutable, boolean stripBinaries) { + List<String> result = new ArrayList<>(); + result.addAll(commonLinkOptions); + + result.add("-B" + ldExecutable.getParentDirectory().getPathString()); + if (stripBinaries) { + result.add("-Wl,-S"); + } + + result.addAll(linkOptionsFromCompilationMode.get(compilationMode)); + result.addAll(linkOptionsFromLipoMode.get(lipoMode)); + result.addAll(linkOptionsFromLinkingMode.get(linkingMode)); + return ImmutableList.copyOf(result); + } + + /** + * Returns the toolchain identifier, which uniquely identifies the compiler + * version, target libc version, target cpu, and LIPO linkage. + */ + public String getToolchainIdentifier() { + return toolchainIdentifier; + } + + /** + * Returns the system name which is required by the toolchain to run. + */ + public String getHostSystemName() { + return hostSystemName; + } + + @Override + public String toString() { + return toolchainIdentifier; + } + + /** + * Returns the compiler version string (e.g. "gcc-4.1.1"). + */ + @SkylarkCallable(name = "compiler", structField = true, doc = "C++ compiler.") + public String getCompiler() { + return compiler; + } + + /** + * Returns the libc version string (e.g. "glibc-2.2.2"). + */ + public String getTargetLibc() { + return targetLibc; + } + + /** + * Returns the target architecture using blaze-specific constants (e.g. "piii"). + */ + @SkylarkCallable(name = "cpu", structField = true, doc = "Target CPU of the C++ toolchain.") + public String getTargetCpu() { + return targetCpu; + } + + /** + * Returns the path fragment that is either absolute or relative to the + * execution root that can be used to execute the given tool. + * + * <p>Note that you must not use this method to get the linker location, but + * use {@link #getLdExecutable} instead! + */ + public PathFragment getToolPathFragment(CppConfiguration.Tool tool) { + return toolPaths.get(tool.getNamePart()); + } + + /** + * Returns a label that forms a dependency to the files required for the + * sysroot that is used. + */ + public Label getLibcLabel() { + return libcLabel; + } + + /** + * Returns a label that references the library files needed to statically + * link the C++ runtime (i.e. libgcc.a, libgcc_eh.a, libstdc++.a) for the + * target architecture. + */ + public Label getStaticRuntimeLibsLabel() { + return supportsEmbeddedRuntimes() ? staticRuntimeLibsLabel : null; + } + + /** + * Returns a label that references the library files needed to dynamically + * link the C++ runtime (i.e. libgcc_s.so, libstdc++.so) for the target + * architecture. + */ + public Label getDynamicRuntimeLibsLabel() { + return supportsEmbeddedRuntimes() ? dynamicRuntimeLibsLabel : null; + } + + /** + * Returns the label of the <code>cc_compiler</code> rule for the C++ configuration. + */ + public Label getCcToolchainRuleLabel() { + return ccToolchainLabel; + } + + /** + * Returns the abi we're using, which is a gcc version. E.g.: "gcc-3.4". + * Note that in practice we might be using gcc-3.4 as ABI even when compiling + * with gcc-4.1.0, because ABIs are backwards compatible. + */ + // TODO(bazel-team): The javadoc should clarify how this is used in Blaze. + public String getAbi() { + return abi; + } + + /** + * Returns the glibc version used by the abi we're using. This is a + * glibc version number (e.g., "2.2.2"). Note that in practice we + * might be using glibc 2.2.2 as ABI even when compiling with + * gcc-4.2.2, gcc-4.3.1, or gcc-4.4.0 (which use glibc 2.3.6), + * because ABIs are backwards compatible. + */ + // TODO(bazel-team): The javadoc should clarify how this is used in Blaze. + public String getAbiGlibcVersion() { + return abiGlibcVersion; + } + + /** + * Returns the configured features of the toolchain. Rules should not call this directly, but + * instead use {@code CcToolchainProvider.getFeatures}. + */ + public CcToolchainFeatures getFeatures() { + return toolchainFeatures; + } + + /** + * Returns whether the toolchain supports the gold linker. + */ + public boolean supportsGoldLinker() { + return supportsGoldLinker; + } + + /** + * Returns whether the toolchain supports thin archives. + */ + public boolean supportsThinArchives() { + return supportsThinArchives; + } + + /** + * Returns whether the toolchain supports the --start-lib/--end-lib options. + */ + public boolean supportsStartEndLib() { + return supportsStartEndLib; + } + + /** + * Returns whether build_interface_so can build interface shared objects for this toolchain. + * Should be true if this toolchain generates ELF objects. + */ + public boolean supportsInterfaceSharedObjects() { + return supportsInterfaceSharedObjects; + } + + /** + * Returns whether the toolchain supports linking C/C++ runtime libraries + * supplied inside the toolchain distribution. + */ + public boolean supportsEmbeddedRuntimes() { + return supportsEmbeddedRuntimes; + } + + /** + * Returns whether the toolchain supports EXEC_ORIGIN libraries resolution. + */ + public boolean supportsExecOrigin() { + // We're rolling out support for this in the same release that also supports embedded runtimes. + return supportsEmbeddedRuntimes; + } + + /** + * Returns whether the toolchain supports "Fission" C++ builds, i.e. builds + * where compilation partitions object code and debug symbols into separate + * output files. + */ + public boolean supportsFission() { + return supportsFission; + } + + /** + * Returns whether shared libraries must be compiled with position + * independent code on this platform. + */ + public boolean toolchainNeedsPic() { + return toolchainNeedsPic; + } + + /** + * Returns whether binaries must be compiled with position independent code. + */ + public boolean usePicForBinaries() { + return usePicForBinaries; + } + + /** + * Returns the type of archives being used. + */ + public Link.ArchiveType archiveType() { + if (useStartEndLib()) { + return Link.ArchiveType.START_END_LIB; + } + if (useThinArchives()) { + return Link.ArchiveType.THIN; + } + return Link.ArchiveType.FAT; + } + + /** + * Returns the ar flags to be used. + */ + public List<String> getArFlags(boolean thinArchives) { + return thinArchives ? arThinArchivesOptions : arOptions; + } + + /** + * Returns a string that uniquely identifies the toolchain. + */ + @Override + public String cacheKey() { + return cacheKey; + } + + /** + * Returns the built-in list of system include paths for the toolchain + * compiler. All paths in this list should be relative to the exec directory. + * They may be absolute if they are also installed on the remote build nodes or + * for local compilation. + */ + public List<PathFragment> getBuiltInIncludeDirectories() { + return builtInIncludeDirectories; + } + + /** + * Returns the sysroot to be used. If the toolchain compiler does not support + * different sysroots, or the sysroot is the same as the default sysroot, then + * this method returns <code>null</code>. + */ + public PathFragment getSysroot() { + return sysroot; + } + + /** + * Returns the run time sysroot, which is where the dynamic linker + * and system libraries are found at runtime. This is usually an absolute path. If the + * toolchain compiler does not support sysroots, then this method returns <code>null</code>. + */ + public PathFragment getRuntimeSysroot() { + return runtimeSysroot; + } + + /** + * Returns the default options to use for compiling C, C++, and assembler. + * This is just the options that should be used for all three languages. + * There may be additional C-specific or C++-specific options that should be used, + * in addition to the ones returned by this method; + */ + public List<String> getCompilerOptions(Collection<String> features) { + return compilerFlags.evaluate(features); + } + + /** + * Returns the list of additional C-specific options to use for compiling + * C. These should be go on the command line after the common options + * returned by {@link #getCompilerOptions}. + */ + public List<String> getCOptions() { + return cOptions; + } + + /** + * Returns the list of additional C++-specific options to use for compiling + * C++. These should be go on the command line after the common options + * returned by {@link #getCompilerOptions}. + */ + public List<String> getCxxOptions(Collection<String> features) { + return cxxFlags.evaluate(features); + } + + /** + * Returns the default list of options which cannot be filtered by BUILD + * rules. These should be appended to the command line after filtering. + */ + public List<String> getUnfilteredCompilerOptions(Collection<String> features) { + return unfilteredCompilerFlags.evaluate(features); + } + + /** + * Returns the set of command-line linker options, including any flags + * inferred from the command-line options. + * + * @see Link + */ + // TODO(bazel-team): Clean up the linker options computation! + public List<String> getLinkOptions() { + return linkOptions; + } + + /** + * Returns the immutable list of linker options for fully statically linked + * outputs. Does not include command-line options passed via --linkopt or + * --linkopts. + * + * @param features default settings affecting this link + * @param sharedLib true if the output is a shared lib, false if it's an executable + */ + public List<String> getFullyStaticLinkOptions(Collection<String> features, + boolean sharedLib) { + if (sharedLib) { + return getSharedLibraryLinkOptions(mostlyStaticLinkFlags, features); + } else { + return fullyStaticLinkFlags.evaluate(features); + } + } + + /** + * Returns the immutable list of linker options for mostly statically linked + * outputs. Does not include command-line options passed via --linkopt or + * --linkopts. + * + * @param features default settings affecting this link + * @param sharedLib true if the output is a shared lib, false if it's an executable + */ + public List<String> getMostlyStaticLinkOptions(Collection<String> features, + boolean sharedLib) { + if (sharedLib) { + return getSharedLibraryLinkOptions( + supportsEmbeddedRuntimes ? mostlyStaticSharedLinkFlags : dynamicLinkFlags, + features); + } else { + return mostlyStaticLinkFlags.evaluate(features); + } + } + + /** + * Returns the immutable list of linker options for artifacts that are not + * fully or mostly statically linked. Does not include command-line options + * passed via --linkopt or --linkopts. + * + * @param features default settings affecting this link + * @param sharedLib true if the output is a shared lib, false if it's an executable + */ + public List<String> getDynamicLinkOptions(Collection<String> features, + boolean sharedLib) { + if (sharedLib) { + return getSharedLibraryLinkOptions(dynamicLinkFlags, features); + } else { + return dynamicLinkFlags.evaluate(features); + } + } + + /** + * Returns link options for the specified flag list, combined with universal options + * for all shared libraries (regardless of link staticness). + */ + private List<String> getSharedLibraryLinkOptions(FlagList flags, + Collection<String> features) { + return ImmutableList.<String>builder() + .addAll(flags.evaluate(features)) + .addAll(dynamicLibraryLinkFlags.evaluate(features)) + .build(); + } + + /** + * Returns test-only link options such that certain test-specific features can be configured + * separately (e.g. lazy binding). + */ + public List<String> getTestOnlyLinkOptions() { + return testOnlyLinkFlags; + } + + + /** + * Returns the list of options to be used with 'objcopy' when converting + * binary files to object files, or {@code null} if this operation is not + * supported. + */ + public List<String> getObjCopyOptionsForEmbedding() { + return objcopyOptions; + } + + /** + * Returns the list of options to be used with 'ld' when converting + * binary files to object files, or {@code null} if this operation is not + * supported. + */ + public List<String> getLdOptionsForEmbedding() { + return ldOptions; + } + + /** + * Returns a map of additional make variables for use by {@link + * BuildConfiguration}. These are to used to allow some build rules to + * avoid the limits on stack frame sizes and variable-length arrays. + * + * <p>The returned map must contain an entry for {@code STACK_FRAME_UNLIMITED}, + * though the entry may be an empty string. + */ + @VisibleForTesting + public Map<String, String> getAdditionalMakeVariables() { + return additionalMakeVariables; + } + + /** + * Returns the execution path to the linker binary to use for this build. + * Relative paths are relative to the execution root. + */ + public PathFragment getLdExecutable() { + return ldExecutable; + } + + /** + * Returns the dynamic linking mode (full, off, or default). + */ + public DynamicMode getDynamicMode() { + return dynamicMode; + } + + /* + * If true then the directory name for non-LIPO targets will have a '-lipodata' suffix in + * AutoFDO mode. + */ + public boolean getAutoFdoLipoData() { + return cppOptions.autoFdoLipoData; + } + + /** + * Returns the STL label if given on the command line. {@code null} + * otherwise. + */ + public Label getStl() { + return cppOptions.stl; + } + + /* + * Returns the command-line "Make" variable overrides. + */ + @Override + public ImmutableMap<String, String> getCommandLineDefines() { + return commandLineDefines; + } + + /** + * Returns the command-line override value for the specified "Make" variable + * for this configuration, or null if none. + */ + public String getMakeVariableOverride(String var) { + return commandLineDefines.get(var); + } + + public boolean shouldScanIncludes() { + return cppOptions.scanIncludes; + } + + /** + * Returns the currently active LIPO compilation mode. + */ + public LipoMode getLipoMode() { + return cppOptions.lipoMode; + } + + public boolean isFdo() { + return cppOptions.isFdo(); + } + + public boolean isLipoOptimization() { + // The LIPO optimization bits are set in the LIPO context collector configuration, too. + return cppOptions.isLipoOptimization() && !isLipoContextCollector(); + } + + public boolean isLipoOptimizationOrInstrumentation() { + return cppOptions.isLipoOptimizationOrInstrumentation(); + } + + /** + * Returns true if it is AutoFDO LIPO build. + */ + public boolean isAutoFdoLipo() { + return cppOptions.fdoOptimize != null && FdoSupport.isAutoFdo(cppOptions.fdoOptimize) + && getLipoMode() != LipoMode.OFF; + } + + /** + * Returns the default header check mode. + */ + public HeadersCheckingMode getHeadersCheckingMode() { + return cppOptions.headersCheckingMode; + } + + /** + * Returns whether or not to strip the binaries. + */ + public boolean shouldStripBinaries() { + return stripBinaries; + } + + /** + * Returns the additional options to pass to strip when generating a + * {@code <name>.stripped} binary by this build. + */ + public List<String> getStripOpts() { + return cppOptions.stripoptList; + } + + /** + * Returns whether temporary outputs from gcc will be saved. + */ + public boolean getSaveTemps() { + return cppOptions.saveTemps; + } + + /** + * Returns the {@link PerLabelOptions} to apply to the gcc command line, if + * the label of the compiled file matches the regular expression. + */ + public List<PerLabelOptions> getPerFileCopts() { + return cppOptions.perFileCopts; + } + + public Label getLipoContextLabel() { + return cppOptions.getLipoContextLabel(); + } + + /** + * Returns the custom malloc library label. + */ + public Label customMalloc() { + return cppOptions.customMalloc; + } + + /** + * Returns the extra warnings enabled for C compilation. + */ + public List<String> getCWarns() { + return cppOptions.cWarns; + } + + /** + * Returns true if mostly-static C++ binaries should be skipped. + */ + public boolean skipStaticOutputs() { + return cppOptions.skipStaticOutputs; + } + + /** + * Returns true if Fission is specified for this build and supported by the crosstool. + */ + public boolean useFission() { + return cppOptions.fissionModes.contains(compilationMode) && supportsFission(); + } + + /** + * Returns true if all C++ compilations should produce position-independent code, links should + * produce position-independent executables, and dependencies with equivalent pre-built pic and + * nopic versions should apply the pic versions. Returns false if default settings should be + * applied (i.e. make no special provisions for pic code). + */ + public boolean forcePic() { + return cppOptions.forcePic; + } + + public boolean useStartEndLib() { + return cppOptions.useStartEndLib && supportsStartEndLib(); + } + + public boolean useThinArchives() { + return cppOptions.useThinArchives && supportsThinArchives(); + } + + /** + * Returns true if interface shared objects should be used. + */ + public boolean useInterfaceSharedObjects() { + return supportsInterfaceSharedObjects() && cppOptions.useInterfaceSharedObjects; + } + + public boolean forceIgnoreDashStatic() { + return cppOptions.forceIgnoreDashStatic; + } + + /** + * Returns true iff this build configuration requires inclusion extraction + * (for include scanning) in the action graph. + */ + public boolean needsIncludeScanning() { + return cppOptions.extractInclusions; + } + + public boolean createCppModuleMaps() { + return cppOptions.cppModuleMaps; + } + + /** + * Returns true if shared libraries must be compiled with position independent code + * on this platform or in this configuration. + */ + public boolean needsPic() { + return forcePic() || toolchainNeedsPic(); + } + + /** + * Returns true iff we should use ".pic.o" files when linking executables. + */ + public boolean usePicObjectsForBinaries() { + return forcePic() || usePicForBinaries(); + } + + public boolean legacyWholeArchive() { + return cppOptions.legacyWholeArchive; + } + + public boolean getSymbolCounts() { + return cppOptions.symbolCounts; + } + + public boolean getInmemoryDotdFiles() { + return cppOptions.inmemoryDotdFiles; + } + + public boolean useIsystemForIncludes() { + return cppOptions.useIsystemForIncludes; + } + + public LibcTop getLibcTop() { + return cppOptions.libcTop; + } + + public boolean getUseInterfaceSharedObjects() { + return cppOptions.useInterfaceSharedObjects; + } + + /** + * Returns the FDO support object. + */ + public FdoSupport getFdoSupport() { + return fdoSupport; + } + + /** + * Return the name of the directory (relative to the bin directory) that + * holds mangled links to shared libraries. This name is always set to + * the '{@code _solib_<cpu_archictecture_name>}. + */ + public String getSolibDirectory() { + return solibDirectory; + } + + /** + * Returns the path to the GNU binutils 'objcopy' binary to use for this + * build. (Corresponds to $(OBJCOPY) in make-dbg.) Relative paths are + * relative to the execution root. + */ + public PathFragment getObjCopyExecutable() { + return getToolPathFragment(CppConfiguration.Tool.OBJCOPY); + } + + /** + * Returns the path to the GNU binutils 'gcc' binary that should be used + * by this build. This binary should support compilation of both C (*.c) + * and C++ (*.cc) files. Relative paths are relative to the execution root. + */ + public PathFragment getCppExecutable() { + return getToolPathFragment(CppConfiguration.Tool.GCC); + } + + /** + * Returns the path to the GNU binutils 'g++' binary that should be used + * by this build. This binary should support linking of both C (*.c) + * and C++ (*.cc) files. Relative paths are relative to the execution root. + */ + public PathFragment getCppLinkExecutable() { + return getToolPathFragment(CppConfiguration.Tool.GCC); + } + + /** + * Returns the path to the GNU binutils 'cpp' binary that should be used + * by this build. Relative paths are relative to the execution root. + */ + public PathFragment getCpreprocessorExecutable() { + return getToolPathFragment(CppConfiguration.Tool.CPP); + } + + /** + * Returns the path to the GNU binutils 'gcov' binary that should be used + * by this build to analyze C++ coverage data. Relative paths are relative to + * the execution root. + */ + public PathFragment getGcovExecutable() { + return getToolPathFragment(CppConfiguration.Tool.GCOV); + } + + /** + * Returns the path to the 'gcov-tool' executable that should be used + * by this build. Relative paths are relative to the execution root. + */ + public PathFragment getGcovToolExecutable() { + return getToolPathFragment(CppConfiguration.Tool.GCOVTOOL); + } + + /** + * Returns the path to the GNU binutils 'nm' executable that should be used + * by this build. Used only for testing. Relative paths are relative to the + * execution root. + */ + public PathFragment getNmExecutable() { + return getToolPathFragment(CppConfiguration.Tool.NM); + } + + /** + * Returns the path to the GNU binutils 'objdump' executable that should be + * used by this build. Used only for testing. Relative paths are relative to + * the execution root. + */ + public PathFragment getObjdumpExecutable() { + return getToolPathFragment(CppConfiguration.Tool.OBJDUMP); + } + + /** + * Returns the path to the GNU binutils 'ar' binary to use for this build. + * Relative paths are relative to the execution root. + */ + public PathFragment getArExecutable() { + return getToolPathFragment(CppConfiguration.Tool.AR); + } + + /** + * Returns the path to the GNU binutils 'strip' executable that should be used + * by this build. Relative paths are relative to the execution root. + */ + public PathFragment getStripExecutable() { + return getToolPathFragment(CppConfiguration.Tool.STRIP); + } + + /** + * Returns the path to the GNU binutils 'dwp' binary that should be used by this + * build to combine debug info output from individual C++ compilations (i.e. .dwo + * files) into aggregate target-level debug packages. Relative paths are relative to the + * execution root. See https://gcc.gnu.org/wiki/DebugFission . + */ + public PathFragment getDwpExecutable() { + return getToolPathFragment(CppConfiguration.Tool.DWP); + } + + /** + * Returns the GNU System Name + */ + public String getTargetGnuSystemName() { + return targetSystemName; + } + + /** + * Returns the architecture component of the GNU System Name + */ + public String getGnuSystemArch() { + if (targetSystemName.indexOf('-') == -1) { + return targetSystemName; + } + return targetSystemName.substring(0, targetSystemName.indexOf('-')); + } + + /** + * Returns whether the configuration's purpose is only to collect LIPO-related data. + */ + public boolean isLipoContextCollector() { + return lipoContextCollector; + } + + @Override + public String getName() { + return "cpp"; + } + + @Override + public void reportInvalidOptions(EventHandler reporter, BuildOptions buildOptions) { + CppOptions cppOptions = buildOptions.get(CppOptions.class); + if (stripBinaries) { + boolean warn = cppOptions.coptList.contains("-g"); + for (PerLabelOptions opt : cppOptions.perFileCopts) { + warn |= opt.getOptions().contains("-g"); + } + if (warn) { + reporter.handle(Event.warn("Stripping enabled, but '--copt=-g' (or --per_file_copt=...@-g) " + + "specified. Debug information will be generated and then stripped away. This is " + + "probably not what you want! Use '-c dbg' for debug mode, or use '--strip=never' " + + "to disable stripping")); + } + } + + if (cppOptions.fdoInstrument != null && cppOptions.fdoOptimize != null) { + reporter.handle(Event.error("Cannot instrument and optimize for FDO at the same time. " + + "Remove one of the '--fdo_instrument' and '--fdo_optimize' options")); + } + + if (cppOptions.lipoContext != null) { + if (cppOptions.lipoMode != LipoMode.BINARY || cppOptions.fdoOptimize == null) { + reporter.handle(Event.warn("The --lipo_context option can only be used together with " + + "--fdo_optimize=<profile zip> and --lipo=binary. LIPO context will be ignored.")); + } + } else { + if (cppOptions.lipoMode == LipoMode.BINARY && cppOptions.fdoOptimize != null) { + reporter.handle(Event.error("The --lipo_context option must be specified when using " + + "--fdo_optimize=<profile zip> and --lipo=binary")); + } + } + if (cppOptions.lipoMode == LipoMode.BINARY && + compilationMode != CompilationMode.OPT) { + reporter.handle(Event.error( + "'--lipo=binary' can only be used with '--compilation_mode=opt' (or '-c opt')")); + } + + if (cppOptions.fissionModes.contains(compilationMode) && !supportsFission()) { + reporter.handle( + Event.warn("Fission is not supported by this crosstool. Please use a supporting " + + "crosstool to enable fission")); + } + } + + @Override + public void addGlobalMakeVariables(Builder<String, String> globalMakeEnvBuilder) { + // hardcoded CC->gcc setting for unit tests + globalMakeEnvBuilder.put("CC", getCppExecutable().getPathString()); + + // Make variables provided by crosstool/gcc compiler suite. + globalMakeEnvBuilder.put("AR", getArExecutable().getPathString()); + globalMakeEnvBuilder.put("NM", getNmExecutable().getPathString()); + globalMakeEnvBuilder.put("OBJCOPY", getObjCopyExecutable().getPathString()); + globalMakeEnvBuilder.put("STRIP", getStripExecutable().getPathString()); + + PathFragment gcovtool = getGcovToolExecutable(); + if (gcovtool != null) { + // gcov-tool is optional in Crosstool + globalMakeEnvBuilder.put("GCOVTOOL", gcovtool.getPathString()); + } + + if (getTargetLibc().startsWith("glibc-")) { + globalMakeEnvBuilder.put("GLIBC_VERSION", + getTargetLibc().substring("glibc-".length())); + } else { + globalMakeEnvBuilder.put("GLIBC_VERSION", getTargetLibc()); + } + + globalMakeEnvBuilder.put("C_COMPILER", getCompiler()); + globalMakeEnvBuilder.put("TARGET_CPU", getTargetCpu()); + + // Deprecated variables + + // TODO(bazel-team): (2009) These variables are so rarely used we should try to eliminate + // them entirely. see: "cs -f=BUILD -noi GNU_TARGET" and "cs -f=build_defs -noi + // GNU_TARGET" + globalMakeEnvBuilder.put("CROSSTOOLTOP", crosstoolTopPathFragment.getPathString()); + globalMakeEnvBuilder.put("GLIBC", getTargetLibc()); + globalMakeEnvBuilder.put("GNU_TARGET", targetSystemName); + + globalMakeEnvBuilder.putAll(getAdditionalMakeVariables()); + + globalMakeEnvBuilder.put("ABI_GLIBC_VERSION", getAbiGlibcVersion()); + globalMakeEnvBuilder.put("ABI", abi); + } + + @Override + public void addImplicitLabels(Multimap<String, Label> implicitLabels) { + if (getLibcLabel() != null) { + implicitLabels.put("crosstool", getLibcLabel()); + } + + implicitLabels.put("crosstool", crosstoolTop); + } + + @Override + public void prepareHook(Path execRoot, ArtifactFactory artifactFactory, PathFragment genfilesPath, + PackageRootResolver resolver) throws ViewCreationFailedException { + try { + getFdoSupport().prepareToBuild(execRoot, genfilesPath, artifactFactory, resolver); + } catch (ZipException e) { + throw new ViewCreationFailedException("Error reading provided FDO zip file", e); + } catch (FdoException | IOException e) { + throw new ViewCreationFailedException("Error while initializing FDO support", e); + } + } + + @Override + public void declareSkyframeDependencies(Environment env) { + getFdoSupport().declareSkyframeDependencies(env, execRoot); + } + + @Override + public void addRoots(List<Root> roots) { + // Fdo root can only exist for the target configuration. + FdoSupport fdoSupport = getFdoSupport(); + if (fdoSupport.getFdoRoot() != null) { + roots.add(fdoSupport.getFdoRoot()); + } + + // Grepped header includes; this root is not configuration specific. + roots.add(getGreppedIncludesDirectory()); + } + + @Override + public Map<String, String> getCoverageEnvironment() { + ImmutableMap.Builder<String, String> env = ImmutableMap.builder(); + env.put("COVERAGE_GCOV_PATH", getGcovExecutable().getPathString()); + PathFragment fdoInstrument = getFdoSupport().getFdoInstrument(); + if (fdoInstrument != null) { + env.put("FDO_DIR", fdoInstrument.getPathString()); + } + return env.build(); + } + + @Override + public ImmutableList<Label> getCoverageLabels() { + // TODO(bazel-team): Using a gcov-specific crosstool filegroup here could reduce the number of + // inputs significantly. We'd also need to add logic in tools/coverage/collect_coverage.sh to + // drop crosstool dependency if metadataFiles does not contain *.gcno artifacts. + return ImmutableList.of(crosstoolTop); + } + + @Override + public String getOutputDirectoryName() { + String lipoSuffix; + if (getLipoMode() != LipoMode.OFF && !isAutoFdoLipo()) { + lipoSuffix = "-lipo"; + } else if (getAutoFdoLipoData()) { + lipoSuffix = "-lipodata"; + } else { + lipoSuffix = ""; + } + return toolchainIdentifier + lipoSuffix; + } + + @Override + public String getConfigurationNameSuffix() { + return isLipoContextCollector() ? "collector" : null; + } + + @Override + public String getPlatformName() { + return getToolchainIdentifier(); + } + + @Override + public boolean supportsIncrementalBuild() { + return !isLipoOptimization(); + } + + @Override + public boolean performsStaticLink() { + return getLinkOptions().contains("-static"); + } + + /** + * Returns true if we should share identical native libraries between different targets. + */ + public boolean shareNativeDeps() { + return cppOptions.shareNativeDeps; + } + + @Override + public void prepareForExecutionPhase() throws IOException { + // _fdo has a prefix of "_", but it should nevertheless be deleted. Detailed description + // of the structure of the symlinks / directories can be found at FdoSupport.extractFdoZip(). + // We actually create a directory named "blaze-fdo" under the exec root, the previous version + // of which is deleted in FdoSupport.prepareToBuildExec(). We cannot do that just before the + // execution phase because that needs to happen before the analysis phase (in order to create + // the artifacts corresponding to the .gcda files). + Path tempPath = execRoot.getRelative("_fdo"); + if (tempPath.exists()) { + FileSystemUtils.deleteTree(tempPath); + } + } + + @Override + public Map<String, Object> lateBoundOptionDefaults() { + // --cpu defaults to null. With that default, the actual target cpu string gets picked up + // by the "default_target_cpu" crosstool parameter. + return ImmutableMap.<String, Object>of("cpu", getTargetCpu()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfigurationLoader.java new file mode 100644 index 0000000..de20283 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfigurationLoader.java
@@ -0,0 +1,174 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.RedirectChaser; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig; + +import javax.annotation.Nullable; + +/** + * Loader for C++ configurations. + */ +public class CppConfigurationLoader implements ConfigurationFragmentFactory { + @Override + public Class<? extends Fragment> creates() { + return CppConfiguration.class; + } + + private final Function<String, String> cpuTransformer; + + /** + * Creates a new CrosstoolConfigurationLoader instance with the given + * configuration provider. The configuration provider is used to perform + * caller-specific configuration file lookup. + */ + public CppConfigurationLoader(Function<String, String> cpuTransformer) { + this.cpuTransformer = cpuTransformer; + } + + @Override + public CppConfiguration create(ConfigurationEnvironment env, BuildOptions options) + throws InvalidConfigurationException { + CppConfigurationParameters params = createParameters(env, options); + if (params == null) { + return null; + } + return new CppConfiguration(params); + } + + /** + * Value class for all the data needed to create a {@link CppConfiguration}. + */ + public static class CppConfigurationParameters { + protected final CrosstoolConfig.CToolchain toolchain; + protected final String cacheKeySuffix; + protected final BuildOptions buildOptions; + protected final Label crosstoolTop; + protected final Label ccToolchainLabel; + protected final Path fdoZip; + protected final Path execRoot; + + CppConfigurationParameters(CrosstoolConfig.CToolchain toolchain, + String cacheKeySuffix, + BuildOptions buildOptions, + Path fdoZip, + Path execRoot, + Label crosstoolTop, + Label ccToolchainLabel) { + this.toolchain = toolchain; + this.cacheKeySuffix = cacheKeySuffix; + this.buildOptions = buildOptions; + this.fdoZip = fdoZip; + this.execRoot = execRoot; + this.crosstoolTop = crosstoolTop; + this.ccToolchainLabel = ccToolchainLabel; + } + } + + @Nullable + protected CppConfigurationParameters createParameters( + ConfigurationEnvironment env, BuildOptions options) throws InvalidConfigurationException { + BlazeDirectories directories = env.getBlazeDirectories(); + if (directories == null) { + return null; + } + Label crosstoolTop = RedirectChaser.followRedirects(env, + options.get(CppOptions.class).crosstoolTop, "crosstool_top"); + if (crosstoolTop == null) { + return null; + } + CrosstoolConfigurationLoader.CrosstoolFile file = + CrosstoolConfigurationLoader.readCrosstool(env, crosstoolTop); + if (file == null) { + return null; + } + CrosstoolConfig.CToolchain toolchain = + CrosstoolConfigurationLoader.selectToolchain(file.getProto(), options, cpuTransformer); + + // FDO + // TODO(bazel-team): move this to CppConfiguration.prepareHook + CppOptions cppOptions = options.get(CppOptions.class); + Path fdoZip; + if (cppOptions.fdoOptimize == null) { + fdoZip = null; + } else if (cppOptions.fdoOptimize.startsWith("//")) { + try { + Target target = env.getTarget(Label.parseAbsolute(cppOptions.fdoOptimize)); + if (target == null) { + return null; + } + if (!(target instanceof InputFile)) { + throw new InvalidConfigurationException( + "--fdo_optimize cannot accept targets that do not refer to input files"); + } + fdoZip = env.getPath(target.getPackage(), target.getName()); + if (fdoZip == null) { + throw new InvalidConfigurationException( + "The --fdo_optimize parameter you specified resolves to a file that does not exist"); + } + } catch (NoSuchPackageException | NoSuchTargetException | SyntaxException e) { + throw new InvalidConfigurationException(e); + } + } else { + fdoZip = directories.getWorkspace().getRelative(cppOptions.fdoOptimize); + } + + Label ccToolchainLabel; + try { + ccToolchainLabel = crosstoolTop.getRelative("cc-compiler-" + toolchain.getTargetCpu()); + } catch (Label.SyntaxException e) { + throw new InvalidConfigurationException(String.format( + "'%s' is not a valid CPU. It should only consist of characters valid in labels", + toolchain.getTargetCpu())); + } + + Target ccToolchain; + try { + ccToolchain = env.getTarget(ccToolchainLabel); + if (ccToolchain == null) { + return null; + } + } catch (NoSuchThingException e) { + throw new InvalidConfigurationException(String.format( + "The toolchain rule '%s' does not exist", ccToolchainLabel)); + } + + if (!(ccToolchain instanceof Rule) + || !((Rule) ccToolchain).getRuleClass().equals("cc_toolchain")) { + throw new InvalidConfigurationException(String.format( + "The label '%s' is not a cc_toolchain rule", ccToolchainLabel)); + } + + return new CppConfigurationParameters(toolchain, file.getMd5(), options, + fdoZip, directories.getExecRoot(), crosstoolTop, ccToolchainLabel); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugFileProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugFileProvider.java new file mode 100644 index 0000000..c0fcb11 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugFileProvider.java
@@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A target that provides .dwo files which can be combined into a .dwp packaging step. See + * https://gcc.gnu.org/wiki/DebugFission for details. + */ +@Immutable +public final class CppDebugFileProvider implements TransitiveInfoProvider { + + private final NestedSet<Artifact> transitiveDwoFiles; + private final NestedSet<Artifact> transitivePicDwoFiles; + + public CppDebugFileProvider(NestedSet<Artifact> transitiveDwoFiles, + NestedSet<Artifact> transitivePicDwoFiles) { + this.transitiveDwoFiles = transitiveDwoFiles; + this.transitivePicDwoFiles = transitivePicDwoFiles; + } + + /** + * Returns the .dwo files that should be included in this target's .dwp packaging (if this + * target is linked) or passed through to a dependant's .dwp packaging (e.g. if this is a + * cc_library depended on by a statically linked cc_binary). + * + * Assumes the corresponding link consumes .o files (vs. .pic.o files). + */ + public NestedSet<Artifact> getTransitiveDwoFiles() { + return transitiveDwoFiles; + } + + /** + * Same as above, but assumes the corresponding link consumes pic.o files. + */ + public NestedSet<Artifact> getTransitivePicDwoFiles() { + return transitivePicDwoFiles; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugPackageProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugPackageProvider.java new file mode 100644 index 0000000..864a4d5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugPackageProvider.java
@@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import javax.annotation.Nullable; + +/** + * Provides the binary artifact and its associated .dwp files, if fission is enabled. + * If Fission ({@link https://gcc.gnu.org/wiki/DebugFission}) is not enabled, the + * dwp file will be null. + */ +@Immutable +public final class CppDebugPackageProvider implements TransitiveInfoProvider { + + private final Artifact strippedArtifact; + private final Artifact unstrippedArtifact; + @Nullable private final Artifact dwpArtifact; + + public CppDebugPackageProvider( + Artifact strippedArtifact, + Artifact unstrippedArtifact, + @Nullable Artifact dwpArtifact) { + Preconditions.checkNotNull(strippedArtifact); + Preconditions.checkNotNull(unstrippedArtifact); + this.strippedArtifact = strippedArtifact; + this.unstrippedArtifact = unstrippedArtifact; + this.dwpArtifact = dwpArtifact; + } + + /** + * Returns the stripped file (the explicit ".stripped" target). + */ + public final Artifact getStrippedArtifact() { + return strippedArtifact; + } + + /** + * Returns the unstripped file (the default executable target). + */ + public final Artifact getUnstrippedArtifact() { + return unstrippedArtifact; + } + + /** + * Returns the .dwp file (for fission builds) or null if --fission=no. + */ + @Nullable + public final Artifact getDwpArtifact() { + return dwpArtifact; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java new file mode 100644 index 0000000..d9bb7b6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java
@@ -0,0 +1,141 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.util.FileType; + +import java.util.List; +import java.util.regex.Pattern; + +/** + * C++-related file type definitions. + */ +public final class CppFileTypes { + public static final FileType CPP_SOURCE = FileType.of(".cc", ".cpp", ".cxx", ".C"); + public static final FileType C_SOURCE = FileType.of(".c"); + public static final FileType CPP_HEADER = FileType.of(".h", ".hh", ".hpp", ".hxx", ".inc"); + public static final FileType CPP_TEXTUAL_INCLUDE = FileType.of(".inc"); + + public static final FileType PIC_PREPROCESSED_C = FileType.of(".pic.i"); + public static final FileType PREPROCESSED_C = new FileType() { + final String ext = ".i"; + @Override + public boolean apply(String filename) { + return filename.endsWith(ext) && !PIC_PREPROCESSED_C.matches(filename); + } + @Override + public List<String> getExtensions() { + return ImmutableList.of(ext); + } + }; + public static final FileType PIC_PREPROCESSED_CPP = FileType.of(".pic.ii"); + public static final FileType PREPROCESSED_CPP = new FileType() { + final String ext = ".ii"; + @Override + public boolean apply(String filename) { + return filename.endsWith(ext) && !PIC_PREPROCESSED_CPP.matches(filename); + } + @Override + public List<String> getExtensions() { + return ImmutableList.of(ext); + } + }; + + public static final FileType ASSEMBLER_WITH_C_PREPROCESSOR = FileType.of(".S"); + public static final FileType PIC_ASSEMBLER = FileType.of(".pic.s"); + public static final FileType ASSEMBLER = new FileType() { + final String ext = ".s"; + @Override + public boolean apply(String filename) { + return filename.endsWith(ext) && !PIC_ASSEMBLER.matches(filename); + } + @Override + public List<String> getExtensions() { + return ImmutableList.of(ext); + } + }; + + public static final FileType PIC_ARCHIVE = FileType.of(".pic.a"); + public static final FileType ARCHIVE = new FileType() { + final String ext = ".a"; + @Override + public boolean apply(String filename) { + return filename.endsWith(ext) && !PIC_ARCHIVE.matches(filename); + } + @Override + public List<String> getExtensions() { + return ImmutableList.of(ext); + } + }; + + public static final FileType ALWAYS_LINK_PIC_LIBRARY = FileType.of(".pic.lo"); + public static final FileType ALWAYS_LINK_LIBRARY = new FileType() { + final String ext = ".lo"; + @Override + public boolean apply(String filename) { + return filename.endsWith(ext) && !ALWAYS_LINK_PIC_LIBRARY.matches(filename); + } + @Override + public List<String> getExtensions() { + return ImmutableList.of(ext); + } + }; + + public static final FileType PIC_OBJECT_FILE = FileType.of(".pic.o"); + public static final FileType OBJECT_FILE = new FileType() { + final String ext = ".o"; + @Override + public boolean apply(String filename) { + return filename.endsWith(ext) && !PIC_OBJECT_FILE.matches(filename); + } + @Override + public List<String> getExtensions() { + return ImmutableList.of(ext); + } + }; + + + public static final FileType SHARED_LIBRARY = FileType.of(".so"); + public static final FileType INTERFACE_SHARED_LIBRARY = FileType.of(".ifso"); + public static final FileType LINKER_SCRIPT = FileType.of(".lds"); + // Matches shared libraries with version names in the extension, i.e. + // libmylib.so.2 or libmylib.so.2.10. + private static final Pattern VERSIONED_SHARED_LIBRARY_PATTERN = + Pattern.compile("^.+\\.so(\\.\\d+)+$"); + public static final FileType VERSIONED_SHARED_LIBRARY = new FileType() { + @Override + public boolean apply(String filename) { + // Because regex matching can be slow, we first do a quick digit check on the final + // character before risking the full-on regex match. This should eliminate the performance + // hit on practically every non-qualifying file type. + if (Character.isDigit(filename.charAt(filename.length() - 1))) { + return VERSIONED_SHARED_LIBRARY_PATTERN.matcher(filename).matches(); + } else { + return false; + } + } + }; + + public static final FileType COVERAGE_NOTES = FileType.of(".gcno"); + public static final FileType COVERAGE_DATA = FileType.of(".gcda"); + public static final FileType COVERAGE_DATA_IMPORTS = FileType.of(".gcda.imports"); + public static final FileType GCC_AUTO_PROFILE = FileType.of(".afdo"); + + public static final FileType CPP_MODULE_MAP = FileType.of(".cppmap"); + public static final FileType CPP_MODULE = FileType.of(".pcm"); + + // Output of the dwp tool + public static final FileType DEBUG_INFO_PACKAGE = FileType.of(".dwp"); +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java new file mode 100644 index 0000000..5bc1363 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java
@@ -0,0 +1,529 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.MiddlemanFactory; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CcLinkParams.Linkstamp; +import com.google.devtools.build.lib.rules.cpp.CppCompilationContext.Builder; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.util.IncludeScanningUtil; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Helper class for functionality shared by cpp related rules. + * + * <p>This class can be used only after the loading phase. + */ +public class CppHelper { + // TODO(bazel-team): should this use Link.SHARED_LIBRARY_FILETYPES? + public static final FileTypeSet SHARED_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY); + + private static final FileTypeSet CPP_FILETYPES = FileTypeSet.of( + CppFileTypes.CPP_HEADER, + CppFileTypes.CPP_SOURCE); + + private CppHelper() { + // prevents construction + } + + /** + * Merges the STL and toolchain contexts into context builder. The STL is automatically determined + * using the ":stl" attribute. + */ + public static void mergeToolchainDependentContext(RuleContext ruleContext, + Builder contextBuilder) { + TransitiveInfoCollection stl = ruleContext.getPrerequisite(":stl", Mode.TARGET); + if (stl != null) { + // TODO(bazel-team): Clean this up. + contextBuilder.addSystemIncludeDir(stl.getLabel().getPackageFragment().getRelative("gcc3")); + contextBuilder.mergeDependentContext(stl.getProvider(CppCompilationContext.class)); + } + CcToolchainProvider toolchain = getToolchain(ruleContext); + if (toolchain != null) { + contextBuilder.mergeDependentContext(toolchain.getCppCompilationContext()); + } + } + + /** + * Returns the malloc implementation for the given target. + */ + public static TransitiveInfoCollection mallocForTarget(RuleContext ruleContext) { + if (ruleContext.getFragment(CppConfiguration.class).customMalloc() != null) { + return ruleContext.getPrerequisite(":default_malloc", Mode.TARGET); + } else { + return ruleContext.getPrerequisite("malloc", Mode.TARGET); + } + } + + /** + * Expands Make variables in a list of string and tokenizes the result. If the package feature + * no_copts_tokenization is set, tokenize only items consisting of a single make variable. + * + * @param ruleContext the ruleContext to be used as the context of Make variable expansion + * @param attributeName the name of the attribute to use in error reporting + * @param input the list of strings to expand + * @return a list of strings containing the expanded and tokenized values for the + * attribute + */ + // TODO(bazel-team): Move to CcCommon; refactor CcPlugin to use either CcLibraryHelper or + // CcCommon. + static List<String> expandMakeVariables( + RuleContext ruleContext, String attributeName, List<String> input) { + boolean tokenization = + !ruleContext.getFeatures().contains("no_copts_tokenization"); + + List<String> tokens = new ArrayList<>(); + for (String token : input) { + try { + // Legacy behavior: tokenize all items. + if (tokenization) { + ruleContext.tokenizeAndExpandMakeVars(tokens, attributeName, token); + } else { + String exp = ruleContext.expandSingleMakeVariable(attributeName, token); + if (exp != null) { + ShellUtils.tokenize(tokens, exp); + } else { + tokens.add(ruleContext.expandMakeVariables(attributeName, token)); + } + } + } catch (ShellUtils.TokenizationException e) { + ruleContext.attributeError(attributeName, e.getMessage()); + } + } + return ImmutableList.copyOf(tokens); + } + + /** + * Appends the tokenized values of the copts attribute to copts. + */ + public static ImmutableList<String> getAttributeCopts(RuleContext ruleContext, String attr) { + Preconditions.checkArgument(ruleContext.getRule().isAttrDefined(attr, Type.STRING_LIST)); + List<String> unexpanded = ruleContext.attributes().get(attr, Type.STRING_LIST); + + return ImmutableList.copyOf(expandMakeVariables(ruleContext, attr, unexpanded)); + } + + /** + * Expands attribute value either using label expansion + * (if attemptLabelExpansion == {@code true} and it does not look like make + * variable or flag) or tokenizes and expands make variables. + */ + public static void expandAttribute(RuleContext ruleContext, + List<String> values, String attrName, String attrValue, boolean attemptLabelExpansion) { + if (attemptLabelExpansion && CppHelper.isLinkoptLabel(attrValue)) { + if (!CppHelper.expandLabel(ruleContext, values, attrValue)) { + ruleContext.attributeError(attrName, "could not resolve label '" + attrValue + "'"); + } + } else { + ruleContext.tokenizeAndExpandMakeVars(values, attrName, attrValue); + } + } + + /** + * Determines if a linkopt can be a label. Linkopts come in 2 varieties: + * literals -- flags like -Xl and makefile vars like $(LD) -- and labels, + * which we should expand into filenames. + * + * @param linkopt the link option to test. + * @return true if the linkopt is not a flag (starting with "-") or a makefile + * variable (starting with "$"); + */ + private static boolean isLinkoptLabel(String linkopt) { + return !linkopt.startsWith("$") && !linkopt.startsWith("-"); + } + + /** + * Expands a label against the target's deps, adding the expanded path strings + * to the linkopts. + * + * @param linkopts the linkopts to add the expanded label to + * @param labelName the name of the label to expand + * @return true if the label was expanded successfully, false otherwise + */ + private static boolean expandLabel(RuleContext ruleContext, List<String> linkopts, + String labelName) { + try { + Label label = ruleContext.getLabel().getRelative(labelName); + for (FileProvider target : ruleContext + .getPrerequisites("deps", Mode.TARGET, FileProvider.class)) { + if (target.getLabel().equals(label)) { + for (Artifact artifact : target.getFilesToBuild()) { + linkopts.add(artifact.getExecPathString()); + } + return true; + } + } + } catch (SyntaxException e) { + // Quietly ignore and fall through. + } + linkopts.add(labelName); + return false; + } + + /** + * This almost trivial method looks up the :cc_toolchain attribute on the rule context, makes sure + * that it refers to a rule that has a {@link CcToolchainProvider} (gives an error otherwise), and + * returns a reference to that {@link CcToolchainProvider}. The method only returns {@code null} + * if there is no such attribute (this is currently not an error). + */ + @Nullable public static CcToolchainProvider getToolchain(RuleContext ruleContext) { + if (ruleContext.attributes().getAttributeDefinition(":cc_toolchain") == null) { + // TODO(bazel-team): Report an error or throw an exception in this case. + return null; + } + TransitiveInfoCollection dep = ruleContext.getPrerequisite(":cc_toolchain", Mode.TARGET); + return getToolchain(ruleContext, dep); + } + + /** + * This almost trivial method makes sure that the given info collection has a {@link + * CcToolchainProvider} (gives an error otherwise), and returns a reference to that {@link + * CcToolchainProvider}. The method never returns {@code null}, even if there is no toolchain. + */ + public static CcToolchainProvider getToolchain(RuleContext ruleContext, + TransitiveInfoCollection dep) { + // TODO(bazel-team): Consider checking this generally at the attribute level. + if ((dep == null) || (dep.getProvider(CcToolchainProvider.class) == null)) { + ruleContext.ruleError("The selected C++ toolchain is not a cc_toolchain rule"); + return CcToolchainProvider.EMPTY_TOOLCHAIN_IS_ERROR; + } + return dep.getProvider(CcToolchainProvider.class); + } + + /** + * Returns the directory where object files are created. + */ + public static PathFragment getObjDirectory(Label ruleLabel) { + return AnalysisUtils.getUniqueDirectory(ruleLabel, new PathFragment("_objs")); + } + + /** + * Creates a grep-includes ExtractInclusions action for generated sources/headers in the + * needsIncludeScanning() BuildConfiguration case. Returns a map from original header + * Artifact to the output Artifact of grepping over it. The return value only includes + * entries for generated sources or headers when --extract_generated_inclusions is enabled. + * + * <p>Previously, incremental rebuilds redid all include scanning work + * for a given .cc source in serial. For high-latency file systems, this could cause + * performance problems if many headers are generated. + */ + @Nullable + public static final Map<Artifact, Artifact> createExtractInclusions(RuleContext ruleContext, + Iterable<Artifact> prerequisites) { + Map<Artifact, Artifact> extractions = new HashMap<>(); + for (Artifact prerequisite : prerequisites) { + Artifact scanned = createExtractInclusions(ruleContext, prerequisite); + if (scanned != null) { + extractions.put(prerequisite, scanned); + } + } + return extractions; + } + + /** + * Creates a grep-includes ExtractInclusions action for generated sources/headers in the + * needsIncludeScanning() BuildConfiguration case. + * + * <p>Previously, incremental rebuilds redid all include scanning work for a given + * .cc source in serial. For high-latency file systems, this could cause + * performance problems if many headers are generated. + */ + private static final Artifact createExtractInclusions(RuleContext ruleContext, + Artifact prerequisite) { + if (ruleContext != null && + ruleContext.getFragment(CppConfiguration.class).needsIncludeScanning() && + !prerequisite.isSourceArtifact() && + CPP_FILETYPES.matches(prerequisite.getFilename())) { + Artifact scanned = getIncludesOutput(ruleContext, prerequisite); + ruleContext.registerAction( + new ExtractInclusionAction(ruleContext.getActionOwner(), prerequisite, scanned)); + return scanned; + } + return null; + } + + private static Artifact getIncludesOutput(RuleContext ruleContext, Artifact src) { + Root root = ruleContext.getFragment(CppConfiguration.class).getGreppedIncludesDirectory(); + PathFragment relOut = IncludeScanningUtil.getRootRelativeOutputPath(src.getExecPath()); + return ruleContext.getAnalysisEnvironment().getDerivedArtifact(relOut, root); + } + + /** + * Returns the workspace-relative filename for the linked artifact. + */ + public static PathFragment getLinkedFilename(RuleContext ruleContext, + LinkTargetType linkType) { + PathFragment relativePath = Util.getWorkspaceRelativePath(ruleContext.getTarget()); + PathFragment linkedFileName = (linkType == LinkTargetType.EXECUTABLE) ? + relativePath : + relativePath.replaceName("lib" + relativePath.getBaseName() + linkType.getExtension()); + return linkedFileName; + } + + /** + * Resolves the linkstamp collection from the {@code CcLinkParams} into a map. + * + * <p>Emits a warning on the rule if there are identical linkstamp artifacts with different + * compilation contexts. + */ + public static Map<Artifact, ImmutableList<Artifact>> resolveLinkstamps(RuleContext ruleContext, + CcLinkParams linkParams) { + Map<Artifact, ImmutableList<Artifact>> result = new LinkedHashMap<>(); + for (Linkstamp pair : linkParams.getLinkstamps()) { + Artifact artifact = pair.getArtifact(); + if (result.containsKey(artifact)) { + ruleContext.ruleWarning("rule inherits the '" + artifact.toDetailString() + + "' linkstamp file from more than one cc_library rule"); + } + result.put(artifact, pair.getDeclaredIncludeSrcs()); + } + return result; + } + + public static void addTransitiveLipoInfoForCommonAttributes( + RuleContext ruleContext, + CcCompilationOutputs outputs, + NestedSetBuilder<IncludeScannable> scannableBuilder) { + + TransitiveLipoInfoProvider stl = null; + if (ruleContext.getRule().getAttributeDefinition(":stl") != null && + ruleContext.getPrerequisite(":stl", Mode.TARGET) != null) { + // If the attribute is defined, it is never null. + stl = ruleContext.getPrerequisite(":stl", Mode.TARGET) + .getProvider(TransitiveLipoInfoProvider.class); + } + if (stl != null) { + scannableBuilder.addTransitive(stl.getTransitiveIncludeScannables()); + } + + for (TransitiveLipoInfoProvider dep : + ruleContext.getPrerequisites("deps", Mode.TARGET, TransitiveLipoInfoProvider.class)) { + scannableBuilder.addTransitive(dep.getTransitiveIncludeScannables()); + } + + if (ruleContext.getRule().getRuleClassObject().hasAttr("malloc", Type.LABEL)) { + TransitiveInfoCollection malloc = mallocForTarget(ruleContext); + TransitiveLipoInfoProvider provider = malloc.getProvider(TransitiveLipoInfoProvider.class); + if (provider != null) { + scannableBuilder.addTransitive(provider.getTransitiveIncludeScannables()); + } + } + + for (IncludeScannable scannable : outputs.getLipoScannables()) { + Preconditions.checkState(scannable.getIncludeScannerSources().size() == 1); + scannableBuilder.add(scannable); + } + } + + // TODO(bazel-team): figure out a way to merge these 2 methods. See the Todo in + // CcCommonConfiguredTarget.noCoptsMatches(). + /** + * Determines if we should apply -fPIC for this rule's C++ compilations. This determination + * is generally made by the global C++ configuration settings "needsPic" and + * and "usePicForBinaries". However, an individual rule may override these settings by applying + * -fPIC" to its "nocopts" attribute. This allows incompatible rules to "opt out" of global PIC + * settings (see bug: "Provide a way to turn off -fPIC for targets that can't be built that way"). + * + * @param ruleContext the context of the rule to check + * @param forBinary true if compiling for a binary, false if for a shared library + * @return true if this rule's compilations should apply -fPIC, false otherwise + */ + public static boolean usePic(RuleContext ruleContext, boolean forBinary) { + if (CcCommon.noCoptsMatches("-fPIC", ruleContext)) { + return false; + } + CppConfiguration config = ruleContext.getFragment(CppConfiguration.class); + return forBinary ? config.usePicObjectsForBinaries() : config.needsPic(); + } + + /** + * Returns the LIPO context provider for configured target, + * or null if such a provider doesn't exist. + */ + public static LipoContextProvider getLipoContextProvider(RuleContext ruleContext) { + if (ruleContext.getRule().getAttributeDefinition(":lipo_context_collector") == null) { + return null; + } + + TransitiveInfoCollection dep = + ruleContext.getPrerequisite(":lipo_context_collector", Mode.DONT_CHECK); + return (dep != null) ? dep.getProvider(LipoContextProvider.class) : null; + } + + // Creates CppModuleMap object, and adds it to C++ compilation context. + public static CppModuleMap addCppModuleMapToContext(RuleContext ruleContext, + CppCompilationContext.Builder contextBuilder) { + if (!ruleContext.getFragment(CppConfiguration.class).createCppModuleMaps()) { + return null; + } + if (getToolchain(ruleContext).getCppCompilationContext().getCppModuleMap() == null) { + return null; + } + // Create the module map artifact as a genfile. + PathFragment mapPath = FileSystemUtils.appendExtension(ruleContext.getLabel().toPathFragment(), + Iterables.getOnlyElement(CppFileTypes.CPP_MODULE_MAP.getExtensions())); + Artifact mapFile = ruleContext.getAnalysisEnvironment().getDerivedArtifact(mapPath, + ruleContext.getConfiguration().getGenfilesDirectory()); + CppModuleMap moduleMap = + new CppModuleMap(mapFile, ruleContext.getLabel().toString()); + contextBuilder.setCppModuleMap(moduleMap); + return moduleMap; + } + + /** + * Returns a middleman for all files to build for the given configured target, + * substituting shared library artifacts with corresponding solib symlinks. If + * multiple calls are made, then it returns the same artifact for configurations + * with the same internal directory. + * + * <p>The resulting middleman only aggregates the inputs and must be expanded + * before populating the set of files necessary to execute an action. + */ + static List<Artifact> getAggregatingMiddlemanForCppRuntimes(RuleContext ruleContext, + String purpose, TransitiveInfoCollection dep, String solibDirOverride, + BuildConfiguration configuration) { + return getMiddlemanInternal( + ruleContext.getAnalysisEnvironment(), ruleContext, ruleContext.getActionOwner(), purpose, + dep, true, true, solibDirOverride, configuration); + } + + @VisibleForTesting + public static List<Artifact> getAggregatingMiddlemanForTesting(AnalysisEnvironment env, + RuleContext ruleContext, ActionOwner owner, String purpose, TransitiveInfoCollection dep, + boolean useSolibSymlinks, BuildConfiguration configuration) { + return getMiddlemanInternal( + env, ruleContext, owner, purpose, dep, useSolibSymlinks, false, null, configuration); + } + + /** + * Internal implementation for getAggregatingMiddlemanForCppRuntimes. + */ + private static List<Artifact> getMiddlemanInternal(AnalysisEnvironment env, + RuleContext ruleContext, ActionOwner actionOwner, String purpose, + TransitiveInfoCollection dep, boolean useSolibSymlinks, boolean isCppRuntime, + String solibDirOverride, BuildConfiguration configuration) { + if (dep == null) { + return ImmutableList.of(); + } + MiddlemanFactory factory = env.getMiddlemanFactory(); + Iterable<Artifact> artifacts = dep.getProvider(FileProvider.class).getFilesToBuild(); + if (useSolibSymlinks) { + List<Artifact> symlinkedArtifacts = new ArrayList<>(); + for (Artifact artifact : artifacts) { + symlinkedArtifacts.add(solibArtifactMaybe( + ruleContext, artifact, isCppRuntime, solibDirOverride, configuration)); + } + artifacts = symlinkedArtifacts; + purpose += "_with_solib"; + } + return ImmutableList.of(factory.createMiddlemanAllowMultiple( + env, actionOwner, purpose, artifacts, configuration.getMiddlemanDirectory())); + } + + /** + * If the artifact is a shared library, returns the solib symlink artifact associated with it. + * + * @param ruleContext the context of the rule that creates the symlink + * @param artifact the library the solib symlink should point to + * @param isCppRuntime whether the library is a C++ runtime + * @param solibDirOverride if not null, forces the solib symlink to be in this directory + */ + private static Artifact solibArtifactMaybe(RuleContext ruleContext, Artifact artifact, + boolean isCppRuntime, String solibDirOverride, BuildConfiguration configuration) { + if (SHARED_LIBRARY_FILETYPES.matches(artifact.getFilename())) { + return isCppRuntime + ? SolibSymlinkAction.getCppRuntimeSymlink( + ruleContext, artifact, solibDirOverride, configuration) + .getArtifact() + : SolibSymlinkAction.getDynamicLibrarySymlink( + ruleContext, artifact, false, true, configuration) + .getArtifact(); + } else { + return artifact; + } + } + + /** + * Returns the type of archives being used. + */ + public static Link.ArchiveType archiveType(BuildConfiguration config) { + CppConfiguration cppConfig = config.getFragment(CppConfiguration.class); + return cppConfig.archiveType(); + } + + /** + * Returns the FDO build subtype. + */ + public static String getFdoBuildStamp(CppConfiguration cppConfiguration) { + if (cppConfiguration.getFdoSupport().isAutoFdoEnabled()) { + return (cppConfiguration.getLipoMode() == LipoMode.BINARY) ? "ALIPO" : "AFDO"; + } + if (cppConfiguration.isFdo()) { + return (cppConfiguration.getLipoMode() == LipoMode.BINARY) ? "LIPO" : "FDO"; + } + return null; + } + + /** + * Returns a relative path to the bin directory for data in AutoFDO LIPO mode. + */ + public static PathFragment getLipoDataBinFragment(BuildConfiguration configuration) { + PathFragment parent = configuration.getBinFragment().getParentDirectory(); + return parent.replaceName(parent.getBaseName() + "-lipodata") + .getChild(configuration.getBinFragment().getBaseName()); + } + + /** + * Returns a relative path to the genfiles directory for data in AutoFDO LIPO mode. + */ + public static PathFragment getLipoDataGenfilesFragment(BuildConfiguration configuration) { + PathFragment parent = configuration.getGenfilesFragment().getParentDirectory(); + return parent.replaceName(parent.getBaseName() + "-lipodata") + .getChild(configuration.getGenfilesFragment().getBaseName()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java new file mode 100644 index 0000000..ecf3431 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java
@@ -0,0 +1,1074 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ParameterFile; +import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.extra.CppLinkInfo; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.collect.ImmutableIterable; +import com.google.devtools.build.lib.collect.IterablesChain; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Action that represents an ELF linking step. + */ +@ThreadCompatible +public final class CppLinkAction extends AbstractAction { + private static final String LINK_GUID = "58ec78bd-1176-4e36-8143-439f656b181d"; + private static final String FAKE_LINK_GUID = "da36f819-5a15-43a9-8a45-e01b60e10c8b"; + + private final CppConfiguration cppConfiguration; + private final LibraryToLink outputLibrary; + private final LibraryToLink interfaceOutputLibrary; + + private final LinkCommandLine linkCommandLine; + + /** True for cc_fake_binary targets. */ + private final boolean fake; + + private final Iterable<Artifact> mandatoryInputs; + + // Linking uses a lot of memory; estimate 1 MB per input file, min 1.5 Gib. + // It is vital to not underestimate too much here, + // because running too many concurrent links can + // thrash the machine to the point where it stops + // responding to keystrokes or mouse clicks. + // CPU and IO do not scale similarly and still use the static minimum estimate. + public static final ResourceSet LINK_RESOURCES_PER_INPUT = new ResourceSet(1, 0, 0); + + // This defines the minimum of each resource that will be reserved. + public static final ResourceSet MIN_STATIC_LINK_RESOURCES = new ResourceSet(1536, 1, 0.3); + + // Dynamic linking should be cheaper than static linking. + public static final ResourceSet MIN_DYNAMIC_LINK_RESOURCES = new ResourceSet(1024, 0.3, 0.2); + + /** + * Use {@link Builder} to create instances of this class. Also see there for + * the documentation of all parameters. + * + * <p>This constructor is intentionally private and is only to be called from + * {@link Builder#build()}. + */ + private CppLinkAction(ActionOwner owner, + Iterable<Artifact> inputs, + ImmutableList<Artifact> outputs, + CppConfiguration cppConfiguration, + LibraryToLink outputLibrary, + LibraryToLink interfaceOutputLibrary, + boolean fake, + LinkCommandLine linkCommandLine) { + super(owner, inputs, outputs); + this.mandatoryInputs = inputs; + this.cppConfiguration = cppConfiguration; + this.outputLibrary = outputLibrary; + this.interfaceOutputLibrary = interfaceOutputLibrary; + this.fake = fake; + + this.linkCommandLine = linkCommandLine; + } + + private static Iterable<LinkerInput> filterLinkerInputs(Iterable<LinkerInput> inputs) { + return Iterables.filter(inputs, new Predicate<LinkerInput>() { + @Override + public boolean apply(LinkerInput input) { + return Link.VALID_LINKER_INPUTS.matches(input.getArtifact().getFilename()); + } + }); + } + + private static Iterable<Artifact> filterLinkerInputArtifacts(Iterable<Artifact> inputs) { + return Iterables.filter(inputs, new Predicate<Artifact>() { + @Override + public boolean apply(Artifact input) { + return Link.VALID_LINKER_INPUTS.matches(input.getFilename()); + } + }); + } + + private CppConfiguration getCppConfiguration() { + return cppConfiguration; + } + + @VisibleForTesting + public String getTargetCpu() { + return getCppConfiguration().getTargetCpu(); + } + + public String getHostSystemName() { + return getCppConfiguration().getHostSystemName(); + } + + /** + * Returns the link configuration; for correctness you should not call this method during + * execution - only the argv is part of the action cache key, and we therefore don't guarantee + * that the action will be re-executed if the contents change in a way that does not affect the + * argv. + */ + @VisibleForTesting + public LinkCommandLine getLinkCommandLine() { + return linkCommandLine; + } + + public LibraryToLink getOutputLibrary() { + return outputLibrary; + } + + public LibraryToLink getInterfaceOutputLibrary() { + return interfaceOutputLibrary; + } + + /** + * Returns the path to the output artifact produced by the linker. + */ + public Path getOutputFile() { + return outputLibrary.getArtifact().getPath(); + } + + @VisibleForTesting + public List<String> getRawLinkArgv() { + return linkCommandLine.getRawLinkArgv(); + } + + @VisibleForTesting + public List<String> getArgv() { + return linkCommandLine.arguments(); + } + + /** + * Prepares and returns the command line specification for this link. + * Splits appropriate parts into a .params file and adds any required + * linkstamp compilation steps. + * + * @return a finalized command line suitable for execution + */ + public final List<String> prepareCommandLine(Path execRoot, List<String> inputFiles) + throws ExecException { + List<String> commandlineArgs; + // Try to shorten the command line by use of a parameter file. + // This makes the output with --subcommands (et al) more readable. + if (linkCommandLine.canBeSplit()) { + PathFragment paramExecPath = ParameterFile.derivePath( + outputLibrary.getArtifact().getExecPath()); + Pair<List<String>, List<String>> split = linkCommandLine.splitCommandline(paramExecPath); + commandlineArgs = split.first; + writeToParamFile(execRoot, paramExecPath, split.second); + if (inputFiles != null) { + inputFiles.add(paramExecPath.getPathString()); + } + } else { + commandlineArgs = linkCommandLine.getRawLinkArgv(); + } + return linkCommandLine.finalizeWithLinkstampCommands(commandlineArgs); + } + + private static void writeToParamFile(Path workingDir, PathFragment paramExecPath, + List<String> paramFileArgs) throws ExecException { + // Create parameter file. + ParameterFile paramFile = new ParameterFile(workingDir, paramExecPath, ISO_8859_1, + ParameterFileType.UNQUOTED); + Path paramFilePath = paramFile.getPath(); + try { + // writeContent() fails for existing files that are marked readonly. + paramFilePath.delete(); + } catch (IOException e) { + throw new EnvironmentalExecException("could not delete file '" + paramFilePath + "'", e); + } + paramFile.writeContent(paramFileArgs); + + // Normally Blaze chmods all output files automatically (see + // SkyframeActionExecutor#setOutputsReadOnlyAndExecutable), but this params file is created + // out-of-band and is not declared as an output. By chmodding the file, other processes + // can observe this file being created. + try { + paramFilePath.setWritable(false); + paramFilePath.setExecutable(true); // for consistency with other action outputs + } catch (IOException e) { + throw new EnvironmentalExecException("could not chmod param file '" + paramFilePath + "'", e); + } + } + + @Override + @ThreadCompatible + public void execute( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + if (fake) { + executeFake(); + } else { + Executor executor = actionExecutionContext.getExecutor(); + + try { + executor.getContext(CppLinkActionContext.class).exec( + this, actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException("Linking of rule '" + getOwner().getLabel() + "'", + executor.getVerboseFailures(), this); + } + } + } + + @Override + public String describeStrategy(Executor executor) { + return fake + ? "fake,local" + : executor.getContext(CppLinkActionContext.class).strategyLocality(this); + } + + // Don't forget to update FAKE_LINK_GUID if you modify this method. + @ThreadCompatible + private void executeFake() + throws ActionExecutionException { + // The uses of getLinkConfiguration in this method may not be consistent with the computed key. + // I.e., this may be incrementally incorrect. + final Collection<Artifact> linkstampOutputs = getLinkCommandLine().getLinkstamps().values(); + + // Prefix all fake output files in the command line with $TEST_TMPDIR/. + final String outputPrefix = "$TEST_TMPDIR/"; + List<String> escapedLinkArgv = escapeLinkArgv(linkCommandLine.getRawLinkArgv(), + linkstampOutputs, outputPrefix); + // Write the commands needed to build the real target to the fake target + // file. + StringBuilder s = new StringBuilder(); + Joiner.on('\n').appendTo(s, + "# This is a fake target file, automatically generated.", + "# Do not edit by hand!", + "echo $0 is a fake target file and not meant to be executed.", + "exit 0", + "EOS", + "", + "makefile_dir=.", + ""); + + try { + // Concatenate all the (fake) .o files into the result. + for (LinkerInput linkerInput : getLinkCommandLine().getLinkerInputs()) { + Artifact objectFile = linkerInput.getArtifact(); + if (CppFileTypes.OBJECT_FILE.matches(objectFile.getFilename()) + && linkerInput.isFake()) { + s.append(FileSystemUtils.readContentAsLatin1(objectFile.getPath())); // (IOException) + } + } + + s.append(getOutputFile().getBaseName()).append(": "); + for (Artifact linkstamp : linkstampOutputs) { + s.append("mkdir -p " + outputPrefix + + linkstamp.getExecPath().getParentDirectory() + " && "); + } + Joiner.on(' ').appendTo(s, + ShellEscaper.escapeAll(linkCommandLine.finalizeAlreadyEscapedWithLinkstampCommands( + escapedLinkArgv, outputPrefix))); + s.append('\n'); + if (getOutputFile().exists()) { + getOutputFile().setWritable(true); // (IOException) + } + FileSystemUtils.writeContent(getOutputFile(), ISO_8859_1, s.toString()); + getOutputFile().setExecutable(true); // (IOException) + for (Artifact linkstamp : linkstampOutputs) { + FileSystemUtils.touchFile(linkstamp.getPath()); + } + } catch (IOException e) { + throw new ActionExecutionException("failed to create fake link command for rule '" + + getOwner().getLabel() + ": " + e.getMessage(), + this, false); + } + } + + /** + * Shell-escapes the raw link command line. + * + * @param rawLinkArgv raw link command line + * @param linkstampOutputs linkstamp artifacts + * @param outputPrefix to be prepended to any outputs + * @return escaped link command line + */ + private List<String> escapeLinkArgv(List<String> rawLinkArgv, + final Collection<Artifact> linkstampOutputs, final String outputPrefix) { + final List<String> linkstampExecPaths = Artifact.asExecPaths(linkstampOutputs); + ImmutableList.Builder<String> escapedArgs = ImmutableList.builder(); + for (String rawArg : rawLinkArgv) { + String escapedArg; + if (rawArg.equals(getPrimaryOutput().getExecPathString()) + || linkstampExecPaths.contains(rawArg)) { + escapedArg = outputPrefix + ShellEscaper.escapeString(rawArg); + } else if (rawArg.startsWith(Link.FAKE_OBJECT_PREFIX)) { + escapedArg = outputPrefix + ShellEscaper.escapeString( + rawArg.substring(Link.FAKE_OBJECT_PREFIX.length())); + } else { + escapedArg = ShellEscaper.escapeString(rawArg); + } + escapedArgs.add(escapedArg); + } + return escapedArgs.build(); + } + + @Override + public ExtraActionInfo.Builder getExtraActionInfo() { + // The uses of getLinkConfiguration in this method may not be consistent with the computed key. + // I.e., this may be incrementally incorrect. + CppLinkInfo.Builder info = CppLinkInfo.newBuilder(); + info.addAllInputFile(Artifact.toExecPaths( + LinkerInputs.toLibraryArtifacts(getLinkCommandLine().getLinkerInputs()))); + info.addAllInputFile(Artifact.toExecPaths( + LinkerInputs.toLibraryArtifacts(getLinkCommandLine().getRuntimeInputs()))); + info.setOutputFile(getPrimaryOutput().getExecPathString()); + if (interfaceOutputLibrary != null) { + info.setInterfaceOutputFile(interfaceOutputLibrary.getArtifact().getExecPathString()); + } + info.setLinkTargetType(getLinkCommandLine().getLinkTargetType().name()); + info.setLinkStaticness(getLinkCommandLine().getLinkStaticness().name()); + info.addAllLinkStamp(Artifact.toExecPaths(getLinkCommandLine().getLinkstamps().values())); + info.addAllBuildInfoHeaderArtifact( + Artifact.toExecPaths(getLinkCommandLine().getBuildInfoHeaderArtifacts())); + info.addAllLinkOpt(getLinkCommandLine().getLinkopts()); + + return super.getExtraActionInfo() + .setExtension(CppLinkInfo.cppLinkInfo, info.build()); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(fake ? FAKE_LINK_GUID : LINK_GUID); + f.addString(getCppConfiguration().getLdExecutable().getPathString()); + f.addStrings(linkCommandLine.arguments()); + // TODO(bazel-team): For correctness, we need to ensure the invariant that all values accessed + // during the execution phase are also covered by the key. Above, we add the argv to the key, + // which covers most cases. Unfortunately, the extra action and fake support methods above also + // sometimes directly access settings from the link configuration that may or may not affect the + // key. We either need to change the code to cover them in the key computation, or change the + // LinkConfiguration to disallow the combinations where the value of a setting does not affect + // the argv. + f.addBoolean(linkCommandLine.isNativeDeps()); + f.addBoolean(linkCommandLine.useTestOnlyFlags()); + if (linkCommandLine.getRuntimeSolibDir() != null) { + f.addPath(linkCommandLine.getRuntimeSolibDir()); + } + return f.hexDigestAndReset(); + } + + @Override + public String describeKey() { + StringBuilder message = new StringBuilder(); + if (fake) { + message.append("Fake "); + } + message.append(getProgressMessage()); + message.append('\n'); + message.append(" Command: "); + message.append(ShellEscaper.escapeString( + getCppConfiguration().getLdExecutable().getPathString())); + message.append('\n'); + // Outputting one argument per line makes it easier to diff the results. + for (String argument : ShellEscaper.escapeAll(linkCommandLine.arguments())) { + message.append(" Argument: "); + message.append(argument); + message.append('\n'); + } + return message.toString(); + } + + @Override + public String getMnemonic() { return "CppLink"; } + + @Override + protected String getRawProgressMessage() { + return "Linking " + outputLibrary.getArtifact().prettyPrint(); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return executor.getContext(CppLinkActionContext.class).estimateResourceConsumption(this); + } + + /** + * Estimate the resources consumed when this action is run locally. + */ + public ResourceSet estimateResourceConsumptionLocal() { + // It's ok if this behaves differently even if the key is identical. + ResourceSet minLinkResources = + getLinkCommandLine().getLinkStaticness() == Link.LinkStaticness.DYNAMIC + ? MIN_DYNAMIC_LINK_RESOURCES + : MIN_STATIC_LINK_RESOURCES; + + final int inputSize = Iterables.size(getLinkCommandLine().getLinkerInputs()) + + Iterables.size(getLinkCommandLine().getRuntimeInputs()); + + return new ResourceSet( + Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getMemoryMb(), + minLinkResources.getMemoryMb()), + Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getCpuUsage(), + minLinkResources.getCpuUsage()), + Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getIoUsage(), + minLinkResources.getIoUsage()) + ); + } + + @Override + public Iterable<Artifact> getMandatoryInputs() { + return mandatoryInputs; + } + + /** + * Determines whether or not this link should output a symbol counts file. + */ + private static boolean enableSymbolsCounts(CppConfiguration cppConfiguration, boolean fake, + LinkTargetType linkType) { + return cppConfiguration.getSymbolCounts() + && cppConfiguration.supportsGoldLinker() + && linkType == LinkTargetType.EXECUTABLE + && !fake; + } + + /** + * Builder class to construct {@link CppLinkAction}s. + */ + public static class Builder { + // Builder-only + private final RuleContext ruleContext; + private final AnalysisEnvironment analysisEnvironment; + private final PathFragment outputPath; + private final CcToolchainProvider toolchain; + private PathFragment interfaceOutputPath; + private PathFragment runtimeSolibDir; + protected final BuildConfiguration configuration; + private final CppConfiguration cppConfiguration; + + // Morally equivalent with {@link Context}, except these are mutable. + // Keep these in sync with {@link Context}. + private final Set<LinkerInput> nonLibraries = new LinkedHashSet<>(); + private final NestedSetBuilder<LibraryToLink> libraries = NestedSetBuilder.linkOrder(); + private NestedSet<Artifact> crosstoolInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + private Artifact runtimeMiddleman; + private NestedSet<Artifact> runtimeInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + private final NestedSetBuilder<Artifact> compilationInputs = NestedSetBuilder.stableOrder(); + private final Set<Artifact> linkstamps = new LinkedHashSet<>(); + private List<String> linkstampOptions = new ArrayList<>(); + private final List<String> linkopts = new ArrayList<>(); + private LinkTargetType linkType = LinkTargetType.STATIC_LIBRARY; + private LinkStaticness linkStaticness = LinkStaticness.FULLY_STATIC; + private boolean fake; + private boolean isNativeDeps; + private boolean useTestOnlyFlags; + private boolean wholeArchive; + private boolean supportsParamFiles = true; + + /** + * Creates a builder that builds {@link CppLinkAction} instances. + * + * @param ruleContext the rule that owns the action + * @param outputPath the path of the ELF file to be created, relative to the + * 'bin' directory + */ + public Builder(RuleContext ruleContext, PathFragment outputPath) { + this(ruleContext, outputPath, ruleContext.getConfiguration(), + ruleContext.getAnalysisEnvironment(), CppHelper.getToolchain(ruleContext)); + } + + /** + * Creates a builder that builds {@link CppLinkAction} instances. + * + * @param ruleContext the rule that owns the action + * @param outputPath the path of the ELF file to be created, relative to the + * 'bin' directory + */ + public Builder(RuleContext ruleContext, PathFragment outputPath, + BuildConfiguration configuration, CcToolchainProvider toolchain) { + this(ruleContext, outputPath, configuration, + ruleContext.getAnalysisEnvironment(), toolchain); + } + + /** + * Creates a builder that builds {@link CppLinkAction}s. + * + * @param ruleContext the rule that owns the action + * @param outputPath the path of the ELF file to be created, relative to the + * 'bin' directory + * @param configuration the configuration used to determine the tool chain + * and the default link options + */ + private Builder(RuleContext ruleContext, PathFragment outputPath, + BuildConfiguration configuration, AnalysisEnvironment analysisEnvironment, + CcToolchainProvider toolchain) { + this.ruleContext = ruleContext; + this.analysisEnvironment = Preconditions.checkNotNull(analysisEnvironment); + this.outputPath = Preconditions.checkNotNull(outputPath); + this.configuration = Preconditions.checkNotNull(configuration); + this.cppConfiguration = configuration.getFragment(CppConfiguration.class); + this.toolchain = toolchain; + + // The toolchain != null is here for CppLinkAction.createTestBuilder(). Meh. + if (cppConfiguration.supportsEmbeddedRuntimes() && toolchain != null) { + runtimeSolibDir = toolchain.getDynamicRuntimeSolibDir(); + } + if (toolchain != null) { + supportsParamFiles = toolchain.supportsParamFiles(); + } + } + + /** + * Given a Context, creates a Builder that builds {@link CppLinkAction}s. + * Note well: Keep the Builder->Context and Context->Builder transforms consistent! + * @param ruleContext the rule that owns the action + * @param outputPath the path of the ELF file to be created, relative to the + * 'bin' directory + * @param linkContext an immutable CppLinkAction.Context from the original builder + */ + public Builder(RuleContext ruleContext, PathFragment outputPath, Context linkContext, + BuildConfiguration configuration) { + // These Builder-only fields get set in the constructor: + // ruleContext, analysisEnvironment, outputPath, configuration, runtimeSolibDir + this(ruleContext, outputPath, configuration, ruleContext.getAnalysisEnvironment(), + CppHelper.getToolchain(ruleContext)); + Preconditions.checkNotNull(linkContext); + + // All linkContext fields should be transferred to this Builder. + this.nonLibraries.addAll(linkContext.nonLibraries); + this.libraries.addTransitive(linkContext.libraries); + this.crosstoolInputs = linkContext.crosstoolInputs; + this.runtimeMiddleman = linkContext.runtimeMiddleman; + this.runtimeInputs = linkContext.runtimeInputs; + this.compilationInputs.addTransitive(linkContext.compilationInputs); + this.linkstamps.addAll(linkContext.linkstamps); + this.linkopts.addAll(linkContext.linkopts); + this.linkType = linkContext.linkType; + this.linkStaticness = linkContext.linkStaticness; + this.fake = linkContext.fake; + this.isNativeDeps = linkContext.isNativeDeps; + this.useTestOnlyFlags = linkContext.useTestOnlyFlags; + } + + /** + * Builds the Action as configured and returns it. + * + * <p>This method may only be called once. + */ + public CppLinkAction build() { + if (interfaceOutputPath != null && (fake || linkType != LinkTargetType.DYNAMIC_LIBRARY)) { + throw new RuntimeException("Interface output can only be used " + + "with non-fake DYNAMIC_LIBRARY targets"); + } + + final Artifact output = createArtifact(outputPath); + final Artifact interfaceOutput = (interfaceOutputPath != null) + ? createArtifact(interfaceOutputPath) + : null; + + final ImmutableList<Artifact> buildInfoHeaderArtifacts = !linkstamps.isEmpty() + ? ruleContext.getAnalysisEnvironment().getBuildInfo(ruleContext, CppBuildInfo.KEY) + : ImmutableList.<Artifact>of(); + + final Artifact symbolCountOutput = enableSymbolsCounts(cppConfiguration, fake, linkType) + ? createArtifact(output.getRootRelativePath().replaceName( + output.getExecPath().getBaseName() + ".sc")) + : null; + + boolean needWholeArchive = wholeArchive || needWholeArchive( + linkStaticness, linkType, linkopts, isNativeDeps, cppConfiguration); + + NestedSet<LibraryToLink> uniqueLibraries = libraries.build(); + final Iterable<Artifact> filteredNonLibraryArtifacts = filterLinkerInputArtifacts( + LinkerInputs.toLibraryArtifacts(nonLibraries)); + final Iterable<LinkerInput> linkerInputs = IterablesChain.<LinkerInput>builder() + .add(ImmutableList.copyOf(filterLinkerInputs(nonLibraries))) + .add(ImmutableIterable.from(Link.mergeInputsCmdLine( + uniqueLibraries, needWholeArchive, cppConfiguration.archiveType()))) + .build(); + + // ruleContext can only be null during testing. This is kind of ugly. + final ImmutableSet<String> features = (ruleContext == null) + ? ImmutableSet.<String>of() + : ruleContext.getFeatures(); + + final LibraryToLink outputLibrary = + LinkerInputs.newInputLibrary(output, filteredNonLibraryArtifacts); + final LibraryToLink interfaceOutputLibrary = interfaceOutput == null ? null : + LinkerInputs.newInputLibrary(interfaceOutput, filteredNonLibraryArtifacts); + + final ImmutableMap<Artifact, Artifact> linkstampMap = + mapLinkstampsToOutputs(linkstamps, ruleContext, output); + + final ImmutableList<Artifact> actionOutputs = constructOutputs( + outputLibrary.getArtifact(), + linkstampMap.values(), + interfaceOutputLibrary == null ? null : interfaceOutputLibrary.getArtifact(), + symbolCountOutput); + + LinkCommandLine linkCommandLine = new LinkCommandLine.Builder(configuration, getOwner()) + .setOutput(outputLibrary.getArtifact()) + .setInterfaceOutput(interfaceOutput) + .setSymbolCountsOutput(symbolCountOutput) + .setBuildInfoHeaderArtifacts(buildInfoHeaderArtifacts) + .setLinkerInputs(linkerInputs) + .setRuntimeInputs(ImmutableList.copyOf(LinkerInputs.simpleLinkerInputs(runtimeInputs))) + .setLinkTargetType(linkType) + .setLinkStaticness(linkStaticness) + .setLinkopts(ImmutableList.copyOf(linkopts)) + .setFeatures(features) + .setLinkstamps(linkstampMap) + .addLinkstampCompileOptions(linkstampOptions) + .setRuntimeSolibDir(linkType.isStaticLibraryLink() ? null : runtimeSolibDir) + .setNativeDeps(isNativeDeps) + .setUseTestOnlyFlags(useTestOnlyFlags) + .setNeedWholeArchive(needWholeArchive) + .setInterfaceSoBuilder(getInterfaceSoBuilder()) + .setSupportsParamFiles(supportsParamFiles) + .build(); + + // Compute the set of inputs - we only need stable order here. + NestedSetBuilder<Artifact> dependencyInputsBuilder = NestedSetBuilder.stableOrder(); + dependencyInputsBuilder.addAll(buildInfoHeaderArtifacts); + dependencyInputsBuilder.addAll(linkstamps); + dependencyInputsBuilder.addTransitive(crosstoolInputs); + if (runtimeMiddleman != null) { + dependencyInputsBuilder.add(runtimeMiddleman); + } + dependencyInputsBuilder.addTransitive(compilationInputs.build()); + + Iterable<Artifact> expandedInputs = + LinkerInputs.toLibraryArtifacts(Link.mergeInputsDependencies(uniqueLibraries, + needWholeArchive, cppConfiguration.archiveType())); + // getPrimaryInput returns the first element, and that is a public interface - therefore the + // order here is important. + Iterable<Artifact> inputs = IterablesChain.<Artifact>builder() + .add(ImmutableList.copyOf(LinkerInputs.toLibraryArtifacts(nonLibraries))) + .add(dependencyInputsBuilder.build()) + .add(ImmutableIterable.from(expandedInputs)) + .deduplicate() + .build(); + + return new CppLinkAction( + getOwner(), + inputs, + actionOutputs, + cppConfiguration, + outputLibrary, + interfaceOutputLibrary, + fake, + linkCommandLine); + } + + /** + * The default heuristic on whether we need to use whole-archive for the link. + */ + private static boolean needWholeArchive(LinkStaticness staticness, + LinkTargetType type, Collection<String> linkopts, boolean isNativeDeps, + CppConfiguration cppConfig) { + boolean fullyStatic = (staticness == LinkStaticness.FULLY_STATIC); + boolean mostlyStatic = (staticness == LinkStaticness.MOSTLY_STATIC); + boolean sharedLinkopts = type == LinkTargetType.DYNAMIC_LIBRARY + || linkopts.contains("-shared") + || cppConfig.getLinkOptions().contains("-shared"); + return (isNativeDeps || cppConfig.legacyWholeArchive()) + && (fullyStatic || mostlyStatic) + && sharedLinkopts; + } + + private static ImmutableList<Artifact> constructOutputs(Artifact primaryOutput, + Collection<Artifact> outputList, Artifact... outputs) { + return new ImmutableList.Builder<Artifact>() + .add(primaryOutput) + .addAll(outputList) + .addAll(CollectionUtils.asListWithoutNulls(outputs)) + .build(); + } + + /** + * Translates a collection of linkstamp source files to an immutable + * mapping from source files to object files. In other words, given a + * set of source files, this method determines the output path to which + * each file should be compiled. + * + * @param linkstamps collection of linkstamp source files + * @param ruleContext the rule for which this link is being performed + * @param outputBinary the binary output path for this link + * @return an immutable map that pairs each source file with the + * corresponding object file that should be fed into the link + */ + public static ImmutableMap<Artifact, Artifact> mapLinkstampsToOutputs( + Collection<Artifact> linkstamps, RuleContext ruleContext, Artifact outputBinary) { + ImmutableMap.Builder<Artifact, Artifact> mapBuilder = ImmutableMap.builder(); + + PathFragment outputBinaryPath = outputBinary.getRootRelativePath(); + PathFragment stampOutputDirectory = outputBinaryPath.getParentDirectory(). + getRelative("_objs").getRelative(outputBinaryPath.getBaseName()); + + for (Artifact linkstamp : linkstamps) { + PathFragment stampOutputPath = stampOutputDirectory.getRelative( + FileSystemUtils.replaceExtension(linkstamp.getRootRelativePath(), ".o")); + mapBuilder.put(linkstamp, + ruleContext.getAnalysisEnvironment().getDerivedArtifact( + stampOutputPath, outputBinary.getRoot())); + } + return mapBuilder.build(); + } + + protected ActionOwner getOwner() { + return ruleContext.getActionOwner(); + } + + protected Artifact createArtifact(PathFragment path) { + return analysisEnvironment.getDerivedArtifact(path, configuration.getBinDirectory()); + } + + protected Artifact getInterfaceSoBuilder() { + return analysisEnvironment.getEmbeddedToolArtifact(CppRuleClasses.BUILD_INTERFACE_SO); + } + + /** + * Set the crosstool inputs required for the action. + */ + public Builder setCrosstoolInputs(NestedSet<Artifact> inputs) { + this.crosstoolInputs = inputs; + return this; + } + + /** + * Sets the C++ runtime library inputs for the action. + */ + public Builder setRuntimeInputs(Artifact middleman, NestedSet<Artifact> inputs) { + Preconditions.checkArgument((middleman == null) == inputs.isEmpty()); + this.runtimeMiddleman = middleman; + this.runtimeInputs = inputs; + return this; + } + + /** + * Sets the interface output of the link. A non-null argument can + * only be provided if the link type is {@code DYNAMIC_LIBRARY} + * and fake is false. + */ + public Builder setInterfaceOutputPath(PathFragment path) { + this.interfaceOutputPath = path; + return this; + } + + /** + * Add additional inputs needed for the linkstamp compilation that is being done as part of the + * link. + */ + public Builder addCompilationInputs(Iterable<Artifact> inputs) { + this.compilationInputs.addAll(inputs); + return this; + } + + public Builder addTransitiveCompilationInputs(NestedSet<Artifact> inputs) { + this.compilationInputs.addTransitive(inputs); + return this; + } + + private void addNonLibraryInput(LinkerInput input) { + String name = input.getArtifact().getFilename(); + Preconditions.checkArgument( + !Link.ARCHIVE_LIBRARY_FILETYPES.matches(name) + && !Link.SHARED_LIBRARY_FILETYPES.matches(name), + "'%s' is a library file", input); + this.nonLibraries.add(input); + } + /** + * Adds a single artifact to the set of inputs (C++ source files, header files, etc). Artifacts + * that are not of recognized types will be used for dependency checking but will not be passed + * to the linker. The artifact must not be an archive or a shared library. + */ + public Builder addNonLibraryInput(Artifact input) { + addNonLibraryInput(LinkerInputs.simpleLinkerInput(input)); + return this; + } + + /** + * Adds multiple artifacts to the set of inputs (C++ source files, header files, etc). + * Artifacts that are not of recognized types will be used for dependency checking but will + * not be passed to the linker. The artifacts must not be archives or shared libraries. + */ + public Builder addNonLibraryInputs(Iterable<Artifact> inputs) { + for (Artifact input : inputs) { + addNonLibraryInput(LinkerInputs.simpleLinkerInput(input)); + } + return this; + } + + public Builder addFakeNonLibraryInputs(Iterable<Artifact> inputs) { + for (Artifact input : inputs) { + addNonLibraryInput(LinkerInputs.fakeLinkerInput(input)); + } + return this; + } + + private void checkLibrary(LibraryToLink input) { + String name = input.getArtifact().getFilename(); + Preconditions.checkArgument( + Link.ARCHIVE_LIBRARY_FILETYPES.matches(name) || + Link.SHARED_LIBRARY_FILETYPES.matches(name), + "'%s' is not a library file", input); + } + + /** + * Adds a single artifact to the set of inputs. The artifact must be an archive or a shared + * library. Note that all directly added libraries are implicitly ordered before all nested + * sets added with {@link #addLibraries}, even if added in the opposite order. + */ + public Builder addLibrary(LibraryToLink input) { + checkLibrary(input); + libraries.add(input); + return this; + } + + /** + * Adds multiple artifact to the set of inputs. The artifacts must be archives or shared + * libraries. + */ + public Builder addLibraries(NestedSet<LibraryToLink> inputs) { + for (LibraryToLink input : inputs) { + checkLibrary(input); + } + this.libraries.addTransitive(inputs); + return this; + } + + /** + * Sets the type of ELF file to be created (.a, .so, .lo, executable). The + * default is {@link LinkTargetType#STATIC_LIBRARY}. + */ + public Builder setLinkType(LinkTargetType linkType) { + this.linkType = linkType; + return this; + } + + /** + * Sets the degree of "staticness" of the link: fully static (static binding + * of all symbols), mostly static (use dynamic binding only for symbols from + * glibc), dynamic (use dynamic binding wherever possible). The default is + * {@link LinkStaticness#FULLY_STATIC}. + */ + public Builder setLinkStaticness(LinkStaticness linkStaticness) { + this.linkStaticness = linkStaticness; + return this; + } + + /** + * Adds a C++ source file which will be compiled at link time. This is used + * to embed various values from the build system into binaries to identify + * their provenance. + * + * <p>Link stamps are also automatically added to the inputs. + */ + public Builder addLinkstamps(Map<Artifact, ImmutableList<Artifact>> linkstamps) { + this.linkstamps.addAll(linkstamps.keySet()); + // Add inputs for linkstamping. + if (!linkstamps.isEmpty()) { + // This will just be the compiler unless include scanning is disabled, in which case it will + // include all header files. Since we insist that linkstamps declare all their headers, all + // header files would be overkill, but that only happens when include scanning is disabled. + addTransitiveCompilationInputs(toolchain.getCompile()); + for (Map.Entry<Artifact, ImmutableList<Artifact>> entry : linkstamps.entrySet()) { + addCompilationInputs(entry.getValue()); + } + } + return this; + } + + public Builder addLinkstampCompilerOptions(ImmutableList<String> linkstampOptions) { + this.linkstampOptions = linkstampOptions; + return this; + } + + /** + * Adds an additional linker option. + */ + public Builder addLinkopt(String linkopt) { + this.linkopts.add(linkopt); + return this; + } + + /** + * Adds multiple linker options at once. + * + * @see #addLinkopt(String) + */ + public Builder addLinkopts(Collection<String> linkopts) { + this.linkopts.addAll(linkopts); + return this; + } + + /** + * Sets whether this link action will be used for a cc_fake_binary; false by + * default. + */ + public Builder setFake(boolean fake) { + this.fake = fake; + return this; + } + + /** + * Sets whether this link action is used for a native dependency library. + */ + public Builder setNativeDeps(boolean isNativeDeps) { + this.isNativeDeps = isNativeDeps; + return this; + } + + /** + * Setting this to true overrides the default whole-archive computation and force-enables + * whole archives for every archive in the link. This is only necessary for linking executable + * binaries that are supposed to export symbols. + * + * <p>Usually, the link action while use whole archives for dynamic libraries that are native + * deps (or the legacy whole archive flag is enabled), and that are not dynamically linked. + * + * <p>(Note that it is possible to build dynamic libraries with cc_binary rules by specifying + * linkshared = 1, and giving the rule a name that matches the pattern {@code + * lib<name>.so}.) + */ + public Builder setWholeArchive(boolean wholeArchive) { + this.wholeArchive = wholeArchive; + return this; + } + + /** + * Sets whether this link action should use test-specific flags (e.g. $EXEC_ORIGIN instead of + * $ORIGIN for the solib search path or lazy binding); false by default. + */ + public Builder setUseTestOnlyFlags(boolean useTestOnlyFlags) { + this.useTestOnlyFlags = useTestOnlyFlags; + return this; + } + + /** + * Sets the name of the directory where the solib symlinks for the dynamic runtime libraries + * live. This is usually automatically set from the cc_toolchain. + */ + public Builder setRuntimeSolibDir(PathFragment runtimeSolibDir) { + this.runtimeSolibDir = runtimeSolibDir; + return this; + } + + /** + * Creates a builder without the need for a {@link RuleContext}. + * This is to be used exclusively for testing purposes. + * + * <p>Link stamping is not supported if using this method. + */ + @VisibleForTesting + public static Builder createTestBuilder( + final ActionOwner owner, final AnalysisEnvironment analysisEnvironment, + final PathFragment outputPath, BuildConfiguration config) { + return new Builder(null, outputPath, config, analysisEnvironment, null) { + @Override + protected Artifact createArtifact(PathFragment path) { + return new Artifact(configuration.getBinDirectory().getPath().getRelative(path), + configuration.getBinDirectory(), configuration.getBinFragment().getRelative(path), + analysisEnvironment.getOwner()); + } + @Override + protected ActionOwner getOwner() { + return owner; + } + }; + } + } + + /** + * Immutable ELF linker context, suitable for serialization. + */ + @Immutable @ThreadSafe + public static final class Context implements TransitiveInfoProvider { + // Morally equivalent with {@link Builder}, except these are immutable. + // Keep these in sync with {@link Builder}. + private final ImmutableSet<LinkerInput> nonLibraries; + private final NestedSet<LibraryToLink> libraries; + private final NestedSet<Artifact> crosstoolInputs; + private final Artifact runtimeMiddleman; + private final NestedSet<Artifact> runtimeInputs; + private final NestedSet<Artifact> compilationInputs; + private final ImmutableSet<Artifact> linkstamps; + private final ImmutableList<String> linkopts; + private final LinkTargetType linkType; + private final LinkStaticness linkStaticness; + private final boolean fake; + private final boolean isNativeDeps; + private final boolean useTestOnlyFlags; + + /** + * Given a {@link Builder}, creates a {@code Context} to pass to another target. + * Note well: Keep the Builder->Context and Context->Builder transforms consistent! + * @param builder a mutable {@link CppLinkAction.Builder} to clone from + */ + public Context(Builder builder) { + this.nonLibraries = ImmutableSet.copyOf(builder.nonLibraries); + this.libraries = NestedSetBuilder.<LibraryToLink>linkOrder() + .addTransitive(builder.libraries.build()).build(); + this.crosstoolInputs = + NestedSetBuilder.<Artifact>stableOrder().addTransitive(builder.crosstoolInputs).build(); + this.runtimeMiddleman = builder.runtimeMiddleman; + this.runtimeInputs = + NestedSetBuilder.<Artifact>stableOrder().addTransitive(builder.runtimeInputs).build(); + this.compilationInputs = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(builder.compilationInputs.build()).build(); + this.linkstamps = ImmutableSet.copyOf(builder.linkstamps); + this.linkopts = ImmutableList.copyOf(builder.linkopts); + this.linkType = builder.linkType; + this.linkStaticness = builder.linkStaticness; + this.fake = builder.fake; + this.isNativeDeps = builder.isNativeDeps; + this.useTestOnlyFlags = builder.useTestOnlyFlags; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionContext.java new file mode 100644 index 0000000..24a936b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionContext.java
@@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.ActionContextMarker; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ResourceSet; + +/** + * Context for executing {@link CppLinkAction}s. + */ +@ActionContextMarker(name = "C++ link") +public interface CppLinkActionContext extends ActionContext { + /** + * Returns where the action actually runs. + */ + String strategyLocality(CppLinkAction action); + + /** + * Returns the estimated resource consumption of the action. + */ + ResourceSet estimateResourceConsumption(CppLinkAction action); + + /** + * Executes the specified action. + */ + void exec(CppLinkAction action, + ActionExecutionContext actionExecutionContext) + throws ExecException, ActionExecutionException, InterruptedException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModel.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModel.java new file mode 100644 index 0000000..44258a5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModel.java
@@ -0,0 +1,707 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CcCompilationOutputs.Builder; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.RegexFilter; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +/** + * Representation of a C/C++ compilation. Its purpose is to share the code that creates compilation + * actions between all classes that need to do so. It follows the builder pattern - load up the + * necessary settings and then call {@link #createCcCompileActions}. + * + * <p>This class is not thread-safe, and it should only be used once for each set of source files, + * i.e. calling {@link #createCcCompileActions} will throw an Exception if called twice. + */ +public final class CppModel { + private final CppSemantics semantics; + private final RuleContext ruleContext; + private final BuildConfiguration configuration; + private final CppConfiguration cppConfiguration; + + // compile model + private CppCompilationContext context; + private final List<Pair<Artifact, Label>> sourceFiles = new ArrayList<>(); + private final List<String> copts = new ArrayList<>(); + private final List<PathFragment> additionalIncludes = new ArrayList<>(); + @Nullable private Pattern nocopts; + private boolean fake; + private boolean maySaveTemps; + private boolean onlySingleOutput; + private CcCompilationOutputs compilationOutputs; + private boolean enableLayeringCheck; + private boolean compileHeaderModules; + + // link model + private final List<String> linkopts = new ArrayList<>(); + private LinkTargetType linkType = LinkTargetType.STATIC_LIBRARY; + private boolean neverLink; + private boolean allowInterfaceSharedObjects; + private boolean createDynamicLibrary = true; + private PathFragment soImplFilename; + private FeatureConfiguration featureConfiguration; + + public CppModel(RuleContext ruleContext, CppSemantics semantics) { + this.ruleContext = ruleContext; + this.semantics = semantics; + configuration = ruleContext.getConfiguration(); + cppConfiguration = configuration.getFragment(CppConfiguration.class); + } + + /** + * If the cpp compilation is a fake, then it creates only a single compile action without PIC. + * Defaults to false. + */ + public CppModel setFake(boolean fake) { + this.fake = fake; + return this; + } + + /** + * If set, the CppModel only creates a single .o output that can be linked into a dynamic library, + * i.e., it never generates both PIC and non-PIC outputs. Otherwise it creates outputs that can be + * linked into both static binaries and dynamic libraries (if both require PIC or both require + * non-PIC, then it still only creates a single output). Defaults to false. + */ + public CppModel setOnlySingleOutput(boolean onlySingleOutput) { + this.onlySingleOutput = onlySingleOutput; + return this; + } + + /** + * If set, use compiler flags to enable compiler based layering checks. + */ + public CppModel setEnableLayeringCheck(boolean enableLayeringCheck) { + this.enableLayeringCheck = enableLayeringCheck; + return this; + } + + /** + * If set, add actions that compile header modules to the build. + * See http://clang.llvm.org/docs/Modules.html for more information. + */ + public CppModel setCompileHeaderModules(boolean compileHeaderModules) { + this.compileHeaderModules = compileHeaderModules; + return this; + } + + /** + * Whether to create actions for temps. This defaults to false. + */ + public CppModel setSaveTemps(boolean maySaveTemps) { + this.maySaveTemps = maySaveTemps; + return this; + } + + /** + * Sets the compilation context, i.e. include directories and allowed header files inclusions. + */ + public CppModel setContext(CppCompilationContext context) { + this.context = context; + return this; + } + + /** + * Adds a single source file to be compiled. Note that this should only be called for primary + * compilation units, not for header files or files that are otherwise included. + */ + public CppModel addSources(Iterable<Artifact> sourceFiles, Label sourceLabel) { + for (Artifact sourceFile : sourceFiles) { + this.sourceFiles.add(Pair.of(sourceFile, sourceLabel)); + } + return this; + } + + /** + * Adds all the source files. Note that this should only be called for primary compilation units, + * not for header files or files that are otherwise included. + */ + public CppModel addSources(Iterable<Pair<Artifact, Label>> sources) { + Iterables.addAll(this.sourceFiles, sources); + return this; + } + + /** + * Adds the given copts. + */ + public CppModel addCopts(Collection<String> copts) { + this.copts.addAll(copts); + return this; + } + + /** + * Sets the nocopts pattern. This is used to filter out flags from the system defined set of + * flags. By default no filter is applied. + */ + public CppModel setNoCopts(@Nullable Pattern nocopts) { + this.nocopts = nocopts; + return this; + } + + /** + * This can be used to specify additional include directories, without modifying the compilation + * context. + */ + public CppModel addAdditionalIncludes(Collection<PathFragment> additionalIncludes) { + // TODO(bazel-team): Maybe this could be handled by the compilation context instead? + this.additionalIncludes.addAll(additionalIncludes); + return this; + } + + /** + * Adds the given linkopts to the optional dynamic library link command. + */ + public CppModel addLinkopts(Collection<String> linkopts) { + this.linkopts.addAll(linkopts); + return this; + } + + /** + * Sets the link type used for the link actions. Note that only static links are supported at this + * time. + */ + public CppModel setLinkTargetType(LinkTargetType linkType) { + this.linkType = linkType; + return this; + } + + public CppModel setNeverLink(boolean neverLink) { + this.neverLink = neverLink; + return this; + } + + /** + * Whether to allow interface dynamic libraries. Note that setting this to true only has an effect + * if the configuration allows it. Defaults to false. + */ + public CppModel setAllowInterfaceSharedObjects(boolean allowInterfaceSharedObjects) { + // TODO(bazel-team): Set the default to true, and require explicit action to disable it. + this.allowInterfaceSharedObjects = allowInterfaceSharedObjects; + return this; + } + + public CppModel setCreateDynamicLibrary(boolean createDynamicLibrary) { + this.createDynamicLibrary = createDynamicLibrary; + return this; + } + + public CppModel setDynamicLibraryPath(PathFragment soImplFilename) { + this.soImplFilename = soImplFilename; + return this; + } + + /** + * Sets the feature configuration to be used for C/C++ actions. + */ + public CppModel setFeatureConfiguration(FeatureConfiguration featureConfiguration) { + this.featureConfiguration = featureConfiguration; + return this; + } + + /** + * @return the non-pic header module artifact for the current target. + */ + public Artifact getHeaderModule(Artifact moduleMapArtifact) { + PathFragment objectDir = CppHelper.getObjDirectory(ruleContext.getLabel()); + PathFragment outputName = objectDir.getRelative( + semantics.getEffectiveSourcePath(moduleMapArtifact)); + return ruleContext.getRelatedArtifact(outputName, ".pcm"); + } + + /** + * @return the pic header module artifact for the current target. + */ + public Artifact getPicHeaderModule(Artifact moduleMapArtifact) { + PathFragment objectDir = CppHelper.getObjDirectory(ruleContext.getLabel()); + PathFragment outputName = objectDir.getRelative( + semantics.getEffectiveSourcePath(moduleMapArtifact)); + return ruleContext.getRelatedArtifact(outputName, ".pic.pcm"); + } + + /** + * @return whether this target needs to generate pic actions. + */ + public boolean getGeneratePicActions() { + return CppHelper.usePic(ruleContext, false); + } + + /** + * @return whether this target needs to generate non-pic actions. + */ + public boolean getGenerateNoPicActions() { + return + // If we always need pic for everything, then don't bother to create a no-pic action. + (!CppHelper.usePic(ruleContext, true) || !CppHelper.usePic(ruleContext, false)) + // onlySingleOutput guarantees that the code is only ever linked into a dynamic library - so + // we don't need a no-pic action even if linking into a binary would require it. + && !((onlySingleOutput && getGeneratePicActions())); + } + + /** + * @return whether this target needs to generate a pic header module. + */ + public boolean getGeneratesPicHeaderModule() { + // TODO(bazel-team): Make sure cc_fake_binary works with header module support. + return compileHeaderModules && !fake && getGeneratePicActions(); + } + + /** + * @return whether this target needs to generate a non-pic header module. + */ + public boolean getGeratesNoPicHeaderModule() { + return compileHeaderModules && !fake && getGenerateNoPicActions(); + } + + /** + * Returns a {@code CppCompileActionBuilder} with the common fields for a C++ compile action + * being initialized. + */ + private CppCompileActionBuilder initializeCompileAction(Artifact sourceArtifact, + Label sourceLabel) { + CppCompileActionBuilder builder = createCompileActionBuilder(sourceArtifact, sourceLabel); + if (nocopts != null) { + builder.addNocopts(nocopts); + } + + builder.setEnableLayeringCheck(enableLayeringCheck); + builder.setCompileHeaderModules(compileHeaderModules); + builder.setExtraSystemIncludePrefixes(additionalIncludes); + builder.setFdoBuildStamp(CppHelper.getFdoBuildStamp(cppConfiguration)); + builder.setFeatureConfiguration(featureConfiguration); + return builder; + } + + /** + * Constructs the C++ compiler actions. It generally creates one action for every specified source + * file. It takes into account LIPO, fake-ness, coverage, and PIC, in addition to using the + * settings specified on the current object. This method should only be called once. + */ + public CcCompilationOutputs createCcCompileActions() { + CcCompilationOutputs.Builder result = new CcCompilationOutputs.Builder(); + Preconditions.checkNotNull(context); + AnalysisEnvironment env = ruleContext.getAnalysisEnvironment(); + PathFragment objectDir = CppHelper.getObjDirectory(ruleContext.getLabel()); + + if (compileHeaderModules) { + Artifact moduleMapArtifact = context.getCppModuleMap().getArtifact(); + Label moduleMapLabel = Label.parseAbsoluteUnchecked(context.getCppModuleMap().getName()); + PathFragment outputName = getObjectOutputPath(moduleMapArtifact, objectDir); + CppCompileActionBuilder builder = initializeCompileAction(moduleMapArtifact, moduleMapLabel); + + // A header module compile action is just like a normal compile action, but: + // - the compiled source file is the module map + // - it creates a header module (.pcm file). + createSourceAction(outputName, result, env, moduleMapArtifact, builder, ".pcm"); + } + + for (Pair<Artifact, Label> source : sourceFiles) { + Artifact sourceArtifact = source.getFirst(); + Label sourceLabel = source.getSecond(); + PathFragment outputName = getObjectOutputPath(sourceArtifact, objectDir); + CppCompileActionBuilder builder = initializeCompileAction(sourceArtifact, sourceLabel); + + if (CppFileTypes.CPP_HEADER.matches(source.first.getExecPath())) { + createHeaderAction(outputName, result, env, builder); + } else { + createSourceAction(outputName, result, env, sourceArtifact, builder, ".o"); + } + } + + compilationOutputs = result.build(); + return compilationOutputs; + } + + private void createHeaderAction(PathFragment outputName, Builder result, AnalysisEnvironment env, + CppCompileActionBuilder builder) { + builder.setOutputFile(ruleContext.getRelatedArtifact(outputName, ".h.processed")).setDotdFile( + outputName, ".h.d", ruleContext); + semantics.finalizeCompileActionBuilder(ruleContext, builder); + CppCompileAction compileAction = builder.build(); + env.registerAction(compileAction); + Artifact tokenFile = compileAction.getOutputFile(); + result.addHeaderTokenFile(tokenFile); + } + + private void createSourceAction(PathFragment outputName, + CcCompilationOutputs.Builder result, + AnalysisEnvironment env, + Artifact sourceArtifact, + CppCompileActionBuilder builder, + String outputExtension) { + PathFragment ccRelativeName = semantics.getEffectiveSourcePath(sourceArtifact); + LipoContextProvider lipoProvider = null; + if (cppConfiguration.isLipoOptimization()) { + // TODO(bazel-team): we shouldn't be needing this, merging context with the binary + // is a superset of necessary information. + lipoProvider = Preconditions.checkNotNull(CppHelper.getLipoContextProvider(ruleContext), + outputName); + builder.setContext(CppCompilationContext.mergeForLipo(lipoProvider.getLipoContext(), + context)); + } + if (fake) { + // For cc_fake_binary, we only create a single fake compile action. It's + // not necessary to use -fPIC for negative compilation tests, and using + // .pic.o files in cc_fake_binary would break existing uses of + // cc_fake_binary. + Artifact outputFile = ruleContext.getRelatedArtifact(outputName, outputExtension); + PathFragment tempOutputName = + FileSystemUtils.replaceExtension(outputFile.getExecPath(), ".temp" + outputExtension); + builder + .setOutputFile(outputFile) + .setDotdFile(outputName, ".d", ruleContext) + .setTempOutputFile(tempOutputName); + semantics.finalizeCompileActionBuilder(ruleContext, builder); + CppCompileAction action = builder.build(); + env.registerAction(action); + result.addObjectFile(action.getOutputFile()); + } else { + boolean generatePicAction = getGeneratePicActions(); + // If we always need pic for everything, then don't bother to create a no-pic action. + boolean generateNoPicAction = getGenerateNoPicActions(); + Preconditions.checkState(generatePicAction || generateNoPicAction); + + // Create PIC compile actions (same as non-PIC, but use -fPIC and + // generate .pic.o, .pic.d, .pic.gcno instead of .o, .d, .gcno.) + if (generatePicAction) { + CppCompileActionBuilder picBuilder = copyAsPicBuilder(builder, outputName, outputExtension); + cppConfiguration.getFdoSupport().configureCompilation(picBuilder, ruleContext, env, + ruleContext.getLabel(), ccRelativeName, nocopts, /*usePic=*/true, + lipoProvider); + + if (maySaveTemps) { + result.addTemps( + createTempsActions(sourceArtifact, outputName, picBuilder, /*usePic=*/true)); + } + + if (isCodeCoverageEnabled()) { + picBuilder.setGcnoFile(ruleContext.getRelatedArtifact(outputName, ".pic.gcno")); + } + + semantics.finalizeCompileActionBuilder(ruleContext, picBuilder); + CppCompileAction picAction = picBuilder.build(); + env.registerAction(picAction); + result.addPicObjectFile(picAction.getOutputFile()); + if (picAction.getDwoFile() != null) { + // Host targets don't produce .dwo files. + result.addPicDwoFile(picAction.getDwoFile()); + } + if (cppConfiguration.isLipoContextCollector() && !generateNoPicAction) { + result.addLipoScannable(picAction); + } + } + + if (generateNoPicAction) { + builder + .setOutputFile(ruleContext.getRelatedArtifact(outputName, outputExtension)) + .setDotdFile(outputName, ".d", ruleContext); + // Create non-PIC compile actions + cppConfiguration.getFdoSupport().configureCompilation(builder, ruleContext, env, + ruleContext.getLabel(), ccRelativeName, nocopts, /*usePic=*/false, + lipoProvider); + + if (maySaveTemps) { + result.addTemps( + createTempsActions(sourceArtifact, outputName, builder, /*usePic=*/false)); + } + + if (!cppConfiguration.isLipoOptimization() && isCodeCoverageEnabled()) { + builder.setGcnoFile(ruleContext.getRelatedArtifact(outputName, ".gcno")); + } + + semantics.finalizeCompileActionBuilder(ruleContext, builder); + CppCompileAction compileAction = builder.build(); + env.registerAction(compileAction); + Artifact objectFile = compileAction.getOutputFile(); + result.addObjectFile(objectFile); + if (compileAction.getDwoFile() != null) { + // Host targets don't produce .dwo files. + result.addDwoFile(compileAction.getDwoFile()); + } + if (cppConfiguration.isLipoContextCollector()) { + result.addLipoScannable(compileAction); + } + } + } + } + + /** + * Constructs the C++ linker actions. It generally generates two actions, one for a static library + * and one for a dynamic library. If PIC is required for shared libraries, but not for binaries, + * it additionally creates a third action to generate a PIC static library. + * + * <p>For dynamic libraries, this method can additionally create an interface shared library that + * can be used for linking, but doesn't contain any executable code. This increases the number of + * cache hits for link actions. Call {@link #setAllowInterfaceSharedObjects(boolean)} to enable + * this behavior. + */ + public CcLinkingOutputs createCcLinkActions(CcCompilationOutputs ccOutputs) { + // For now only handle static links. Note that the dynamic library link below ignores linkType. + // TODO(bazel-team): Either support non-static links or move this check to setLinkType(). + Preconditions.checkState(linkType.isStaticLibraryLink(), "can only handle static links"); + + CcLinkingOutputs.Builder result = new CcLinkingOutputs.Builder(); + if (cppConfiguration.isLipoContextCollector()) { + // Don't try to create LIPO link actions in collector mode, + // because it needs some data that's not available at this point. + return result.build(); + } + + AnalysisEnvironment env = ruleContext.getAnalysisEnvironment(); + boolean usePicForBinaries = CppHelper.usePic(ruleContext, true); + boolean usePicForSharedLibs = CppHelper.usePic(ruleContext, false); + + // Create static library (.a). The linkType only reflects whether the library is alwayslink or + // not. The PIC-ness is determined by whether we need to use PIC or not. There are three cases + // for (usePicForSharedLibs usePicForBinaries): + // + // (1) (false false) -> no pic code + // (2) (true false) -> shared libraries as pic, but not binaries + // (3) (true true) -> both shared libraries and binaries as pic + // + // In case (3), we always need PIC, so only create one static library containing the PIC object + // files. The name therefore does not match the content. + // + // Presumably, it is done this way because the .a file is an implicit output of every cc_library + // rule, so we can't use ".pic.a" that in the always-PIC case. + PathFragment linkedFileName = CppHelper.getLinkedFilename(ruleContext, linkType); + CppLinkAction maybePicAction = newLinkActionBuilder(linkedFileName) + .addNonLibraryInputs(ccOutputs.getObjectFiles(usePicForBinaries)) + .addNonLibraryInputs(ccOutputs.getHeaderTokenFiles()) + .setLinkType(linkType) + .setLinkStaticness(LinkStaticness.FULLY_STATIC) + .build(); + env.registerAction(maybePicAction); + result.addStaticLibrary(maybePicAction.getOutputLibrary()); + + // Create a second static library (.pic.a). Only in case (2) do we need both PIC and non-PIC + // static libraries. In that case, the first static library contains the non-PIC code, and this + // one contains the PIC code, so the names match the content. + if (!usePicForBinaries && usePicForSharedLibs) { + LinkTargetType picLinkType = (linkType == LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY) + ? LinkTargetType.ALWAYS_LINK_PIC_STATIC_LIBRARY + : LinkTargetType.PIC_STATIC_LIBRARY; + + PathFragment picFileName = CppHelper.getLinkedFilename(ruleContext, picLinkType); + CppLinkAction picAction = newLinkActionBuilder(picFileName) + .addNonLibraryInputs(ccOutputs.getObjectFiles(true)) + .addNonLibraryInputs(ccOutputs.getHeaderTokenFiles()) + .setLinkType(picLinkType) + .setLinkStaticness(LinkStaticness.FULLY_STATIC) + .build(); + env.registerAction(picAction); + result.addPicStaticLibrary(picAction.getOutputLibrary()); + } + + if (!createDynamicLibrary) { + return result.build(); + } + + // Create dynamic library. + if (soImplFilename == null) { + soImplFilename = CppHelper.getLinkedFilename(ruleContext, LinkTargetType.DYNAMIC_LIBRARY); + } + List<String> sonameLinkopts = ImmutableList.of(); + PathFragment soInterfaceFilename = null; + if (cppConfiguration.useInterfaceSharedObjects() && allowInterfaceSharedObjects) { + soInterfaceFilename = + CppHelper.getLinkedFilename(ruleContext, LinkTargetType.INTERFACE_DYNAMIC_LIBRARY); + Artifact dynamicLibrary = env.getDerivedArtifact( + soImplFilename, configuration.getBinDirectory()); + sonameLinkopts = ImmutableList.of("-Wl,-soname=" + + SolibSymlinkAction.getDynamicLibrarySoname(dynamicLibrary.getRootRelativePath(), false)); + } + + // Should we also link in any libraries that this library depends on? + // That is required on some systems... + CppLinkAction action = newLinkActionBuilder(soImplFilename) + .setInterfaceOutputPath(soInterfaceFilename) + .addNonLibraryInputs(ccOutputs.getObjectFiles(usePicForSharedLibs)) + .addNonLibraryInputs(ccOutputs.getHeaderTokenFiles()) + .setLinkType(LinkTargetType.DYNAMIC_LIBRARY) + .setLinkStaticness(LinkStaticness.DYNAMIC) + .addLinkopts(linkopts) + .addLinkopts(sonameLinkopts) + .setRuntimeInputs( + CppHelper.getToolchain(ruleContext).getDynamicRuntimeLinkMiddleman(), + CppHelper.getToolchain(ruleContext).getDynamicRuntimeLinkInputs()) + .build(); + env.registerAction(action); + + LibraryToLink dynamicLibrary = action.getOutputLibrary(); + LibraryToLink interfaceLibrary = action.getInterfaceOutputLibrary(); + if (interfaceLibrary == null) { + interfaceLibrary = dynamicLibrary; + } + + // If shared library has neverlink=1, then leave it untouched. Otherwise, + // create a mangled symlink for it and from now on reference it through + // mangled name only. + if (neverLink) { + result.addDynamicLibrary(interfaceLibrary); + result.addExecutionDynamicLibrary(dynamicLibrary); + } else { + LibraryToLink libraryLink = SolibSymlinkAction.getDynamicLibrarySymlink( + ruleContext, interfaceLibrary.getArtifact(), false, false, + ruleContext.getConfiguration()); + result.addDynamicLibrary(libraryLink); + LibraryToLink implLibraryLink = SolibSymlinkAction.getDynamicLibrarySymlink( + ruleContext, dynamicLibrary.getArtifact(), false, false, + ruleContext.getConfiguration()); + result.addExecutionDynamicLibrary(implLibraryLink); + } + return result.build(); + } + + private CppLinkAction.Builder newLinkActionBuilder(PathFragment outputPath) { + return new CppLinkAction.Builder(ruleContext, outputPath) + .setCrosstoolInputs(CppHelper.getToolchain(ruleContext).getLink()) + .addNonLibraryInputs(context.getCompilationPrerequisites()); + } + + /** + * Returns the output artifact path relative to the object directory. + */ + private PathFragment getObjectOutputPath(Artifact source, PathFragment objectDirectory) { + return objectDirectory.getRelative(semantics.getEffectiveSourcePath(source)); + } + + /** + * Creates a basic cpp compile action builder for source file. Configures options, + * crosstool inputs, output and dotd file names, compilation context and copts. + */ + private CppCompileActionBuilder createCompileActionBuilder( + Artifact source, Label label) { + CppCompileActionBuilder builder = new CppCompileActionBuilder( + ruleContext, source, label); + + builder + .setContext(context) + .addCopts(copts); + return builder; + } + + /** + * Creates cpp PIC compile action builder from the given builder by adding necessary copt and + * changing output and dotd file names. + */ + private CppCompileActionBuilder copyAsPicBuilder(CppCompileActionBuilder builder, + PathFragment outputName, String outputExtension) { + CppCompileActionBuilder picBuilder = new CppCompileActionBuilder(builder); + picBuilder.addCopt("-fPIC") + .setOutputFile(ruleContext.getRelatedArtifact(outputName, ".pic" + outputExtension)) + .setDotdFile(outputName, ".pic.d", ruleContext); + return picBuilder; + } + + /** + * Create the actions for "--save_temps". + */ + private ImmutableList<Artifact> createTempsActions(Artifact source, PathFragment outputName, + CppCompileActionBuilder builder, boolean usePic) { + if (!cppConfiguration.getSaveTemps()) { + return ImmutableList.of(); + } + + String path = source.getFilename(); + boolean isCFile = CppFileTypes.C_SOURCE.matches(path); + boolean isCppFile = CppFileTypes.CPP_SOURCE.matches(path); + + if (!isCFile && !isCppFile) { + return ImmutableList.of(); + } + + String iExt = isCFile ? ".i" : ".ii"; + String picExt = usePic ? ".pic" : ""; + CppCompileActionBuilder dBuilder = new CppCompileActionBuilder(builder); + CppCompileActionBuilder sdBuilder = new CppCompileActionBuilder(builder); + + dBuilder + .setOutputFile(ruleContext.getRelatedArtifact(outputName, picExt + iExt)) + .setDotdFile(outputName, picExt + iExt + ".d", ruleContext); + semantics.finalizeCompileActionBuilder(ruleContext, dBuilder); + CppCompileAction dAction = dBuilder.build(); + ruleContext.registerAction(dAction); + + sdBuilder + .setOutputFile(ruleContext.getRelatedArtifact(outputName, picExt + ".s")) + .setDotdFile(outputName, picExt + ".s.d", ruleContext); + semantics.finalizeCompileActionBuilder(ruleContext, sdBuilder); + CppCompileAction sdAction = sdBuilder.build(); + ruleContext.registerAction(sdAction); + return ImmutableList.of( + dAction.getOutputFile(), + sdAction.getOutputFile()); + } + + /** + * Returns true iff code coverage is enabled for the given target. + */ + private boolean isCodeCoverageEnabled() { + if (configuration.isCodeCoverageEnabled()) { + final RegexFilter filter = configuration.getInstrumentationFilter(); + // If rule is matched by the instrumentation filter, enable instrumentation + if (filter.isIncluded(ruleContext.getLabel().toString())) { + return true; + } + // At this point the rule itself is not matched by the instrumentation filter. However, we + // might still want to instrument C++ rules if one of the targets listed in "deps" is + // instrumented and, therefore, can supply header files that we would want to collect code + // coverage for. For example, think about cc_test rule that tests functionality defined in a + // header file that is supplied by the cc_library. + // + // Note that we only check direct prerequisites and not the transitive closure. This is done + // for two reasons: + // a) It is a good practice to declare libraries which you directly rely on. Including headers + // from a library hidden deep inside the transitive closure makes build dependencies less + // readable and can lead to unexpected breakage. + // b) Traversing the transitive closure for each C++ compile action would require more complex + // implementation (with caching results of this method) to avoid O(N^2) slowdown. + if (ruleContext.getRule().isAttrDefined("deps", Type.LABEL_LIST)) { + for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps", Mode.TARGET)) { + if (dep.getProvider(CppCompilationContext.class) != null + && filter.isIncluded(dep.getLabel().toString())) { + return true; + } + } + } + } + return false; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMap.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMap.java new file mode 100644 index 0000000..bb27209 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMap.java
@@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Structure for C++ module maps. Stores the name of the module and a .cppmap artifact. + */ +@Immutable +public class CppModuleMap { + private final Artifact artifact; + private final String name; + + public CppModuleMap(Artifact artifact, String name) { + this.artifact = artifact; + this.name = name; + } + + public Artifact getArtifact() { + return artifact; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return name + "@" + artifact; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMapAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMapAction.java new file mode 100644 index 0000000..a350fc4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMapAction.java
@@ -0,0 +1,185 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Creates C++ module map artifact genfiles. These are then passed to Clang to + * do dependency checking. + */ +public class CppModuleMapAction extends AbstractFileWriteAction { + + private static final String GUID = "4f407081-1951-40c1-befc-d6b4daff5de3"; + + // C++ module map of the current target + private final CppModuleMap cppModuleMap; + + /** + * If set, the paths in the module map are relative to the current working directory instead + * of relative to the module map file's location. + */ + private final boolean moduleMapHomeIsCwd; + + // Headers and dependencies list + private final ImmutableList<Artifact> privateHeaders; + private final ImmutableList<Artifact> publicHeaders; + private final ImmutableList<CppModuleMap> dependencies; + private final ImmutableList<PathFragment> additionalExportedHeaders; + private final boolean compiledModule; + + public CppModuleMapAction(ActionOwner owner, CppModuleMap cppModuleMap, + Iterable<Artifact> privateHeaders, Iterable<Artifact> publicHeaders, + Iterable<CppModuleMap> dependencies, Iterable<PathFragment> additionalExportedHeaders, + boolean compiledModule, boolean moduleMapHomeIsCwd) { + super(owner, ImmutableList.<Artifact>of(), cppModuleMap.getArtifact(), + /*makeExecutable=*/false); + this.cppModuleMap = cppModuleMap; + this.moduleMapHomeIsCwd = moduleMapHomeIsCwd; + this.privateHeaders = ImmutableList.copyOf(privateHeaders); + this.publicHeaders = ImmutableList.copyOf(publicHeaders); + this.dependencies = ImmutableList.copyOf(dependencies); + this.additionalExportedHeaders = ImmutableList.copyOf(additionalExportedHeaders); + this.compiledModule = compiledModule; + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) { + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + StringBuilder content = new StringBuilder(); + PathFragment fragment = cppModuleMap.getArtifact().getExecPath(); + int segmentsToExecPath = fragment.segmentCount() - 1; + + // For details about the different header types, see: + // http://clang.llvm.org/docs/Modules.html#header-declaration + String leadingPeriods = moduleMapHomeIsCwd ? "" : Strings.repeat("../", segmentsToExecPath); + content.append("module \"").append(cppModuleMap.getName()).append("\" {\n"); + content.append(" export *\n"); + for (Artifact artifact : privateHeaders) { + appendHeader(content, "private", artifact.getExecPath(), leadingPeriods, + /*canCompile=*/true); + } + for (Artifact artifact : publicHeaders) { + appendHeader(content, "", artifact.getExecPath(), leadingPeriods, /*canCompile=*/true); + } + for (PathFragment additionalExportedHeader : additionalExportedHeaders) { + appendHeader(content, "", additionalExportedHeader, leadingPeriods, /*canCompile*/false); + } + for (CppModuleMap dep : dependencies) { + content.append(" use \"").append(dep.getName()).append("\"\n"); + } + content.append("}"); + for (CppModuleMap dep : dependencies) { + content.append("\nextern module \"") + .append(dep.getName()) + .append("\" \"") + .append(leadingPeriods) + .append(dep.getArtifact().getExecPath()) + .append("\""); + } + out.write(content.toString().getBytes(StandardCharsets.ISO_8859_1)); + } + }; + } + + private void appendHeader(StringBuilder content, String visibilitySpecifier, PathFragment path, + String leadingPeriods, boolean canCompile) { + content.append(" "); + if (!visibilitySpecifier.isEmpty()) { + content.append(visibilitySpecifier).append(" "); + } + if (!canCompile || !shouldCompileHeader(path)) { + content.append("textual "); + } + content.append("header \"").append(leadingPeriods).append(path).append("\"\n"); + } + + private boolean shouldCompileHeader(PathFragment path) { + return compiledModule && !CppFileTypes.CPP_TEXTUAL_INCLUDE.matches(path); + } + + @Override + public String getMnemonic() { + return "CppModuleMap"; + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addInt(privateHeaders.size()); + for (Artifact artifact : privateHeaders) { + f.addPath(artifact.getRootRelativePath()); + } + f.addInt(publicHeaders.size()); + for (Artifact artifact : publicHeaders) { + f.addPath(artifact.getRootRelativePath()); + } + f.addInt(dependencies.size()); + for (CppModuleMap dep : dependencies) { + f.addPath(dep.getArtifact().getExecPath()); + } + f.addPath(cppModuleMap.getArtifact().getExecPath()); + f.addString(cppModuleMap.getName()); + return f.hexDigestAndReset(); + } + + @Override + public ResourceSet estimateResourceConsumptionLocal() { + return new ResourceSet(/*memoryMb=*/0, /*cpuUsage=*/0, /*ioUsage=*/0.02); + } + + @VisibleForTesting + public Collection<Artifact> getPublicHeaders() { + return publicHeaders; + } + + @VisibleForTesting + public Collection<Artifact> getPrivateHeaders() { + return privateHeaders; + } + + @VisibleForTesting + public ImmutableList<PathFragment> getAdditionalExportedHeaders() { + return additionalExportedHeaders; + } + + @VisibleForTesting + public Collection<Artifact> getDependencyArtifacts() { + List<Artifact> artifacts = new ArrayList<>(); + for (CppModuleMap map : dependencies) { + artifacts.add(map.getArtifact()); + } + return artifacts; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java new file mode 100644 index 0000000..b0f2e82 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java
@@ -0,0 +1,646 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.LabelConverter; +import com.google.devtools.build.lib.analysis.config.CompilationMode; +import com.google.devtools.build.lib.analysis.config.FragmentOptions; +import com.google.devtools.build.lib.analysis.config.PerLabelOptions; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.HeadersCheckingMode; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.LibcTop; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.StripMode; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.EnumConverter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * Command-line options for C++. + */ +public class CppOptions extends FragmentOptions { + /** + * Label of a filegroup that contains all crosstool files for all configurations. + */ + @VisibleForTesting + public static final String DEFAULT_CROSSTOOL_TARGET = "//tools/cpp:toolchain"; + + + /** + * Converter for --cwarn flag + */ + public static class GccWarnConverter implements Converter<String> { + @Override + public String convert(String input) throws OptionsParsingException { + if (input.startsWith("no-") || input.startsWith("-W")) { + throw new OptionsParsingException("Not a valid gcc warning to enable"); + } + return input; + } + + @Override + public String getTypeDescription() { + return "A gcc warning to enable"; + } + } + + /** + * Converts a comma-separated list of compilation mode settings to a properly typed List. + */ + public static class FissionOptionConverter implements Converter<List<CompilationMode>> { + @Override + public List<CompilationMode> convert(String input) throws OptionsParsingException { + ImmutableSet.Builder<CompilationMode> modes = ImmutableSet.builder(); + if (input.equals("yes")) { // Special case: enable all modes. + modes.add(CompilationMode.values()); + } else if (!input.equals("no")) { // "no" is another special case that disables all modes. + CompilationMode.Converter modeConverter = new CompilationMode.Converter(); + for (String mode : Splitter.on(',').split(input)) { + modes.add(modeConverter.convert(mode)); + } + } + return modes.build().asList(); + } + + @Override + public String getTypeDescription() { + return "a set of compilation modes"; + } + } + + /** + * The same as DynamicMode, but on command-line we also allow AUTO. + */ + public enum DynamicModeFlag { OFF, DEFAULT, FULLY, AUTO } + + /** + * Converter for DynamicModeFlag + */ + public static class DynamicModeConverter extends EnumConverter<DynamicModeFlag> { + public DynamicModeConverter() { + super(DynamicModeFlag.class, "dynamic mode"); + } + } + + /** + * Converter for the --strip option. + */ + public static class StripModeConverter extends EnumConverter<StripMode> { + public StripModeConverter() { + super(StripMode.class, "strip mode"); + } + } + + private static final String LIBC_RELATIVE_LABEL = ":everything"; + + /** + * Converts a String, which is an absolute path or label into a LibcTop + * object. + */ + public static class LibcTopConverter implements Converter<LibcTop> { + @Override + public LibcTop convert(String input) throws OptionsParsingException { + if (!input.startsWith("//")) { + throw new OptionsParsingException("Not a label"); + } + try { + Label label = Label.parseAbsolute(input).getRelative(LIBC_RELATIVE_LABEL); + return new LibcTop(label); + } catch (SyntaxException e) { + throw new OptionsParsingException(e.getMessage()); + } + } + + @Override + public String getTypeDescription() { + return "a label"; + } + } + + /** + * Converter for the --hdrs_check option. + */ + public static class HdrsCheckConverter extends EnumConverter<HeadersCheckingMode> { + public HdrsCheckConverter() { + super(HeadersCheckingMode.class, "Headers check mode"); + } + } + + /** + * Checks whether a string is a valid regex pattern and compiles it. + */ + public static class NullableRegexPatternConverter implements Converter<Pattern> { + + @Override + public Pattern convert(String input) throws OptionsParsingException { + if (input.isEmpty()) { + return null; + } + try { + return Pattern.compile(input); + } catch (PatternSyntaxException e) { + throw new OptionsParsingException("Not a valid regular expression: " + e.getMessage()); + } + } + + @Override + public String getTypeDescription() { + return "a valid Java regular expression"; + } + } + + /** + * Converter for the --lipo option. + */ + public static class LipoModeConverter extends EnumConverter<LipoMode> { + public LipoModeConverter() { + super(LipoMode.class, "LIPO mode"); + } + } + + @Option(name = "lipo input collector", + defaultValue = "false", + category = "undocumented", + help = "Internal flag, only used to create configurations with the LIPO-collector flag set.") + public boolean lipoCollector; + + @Option(name = "crosstool_top", + defaultValue = CppOptions.DEFAULT_CROSSTOOL_TARGET, + category = "version", + converter = LabelConverter.class, + help = "The label of the crosstool package to be used for compiling C++ code.") + public Label crosstoolTop; + + @Option(name = "compiler", + defaultValue = "null", + category = "version", + help = "The C++ compiler to use for compiling the target.") + public String cppCompiler; + + @Option(name = "glibc", + defaultValue = "null", + category = "version", + help = "The version of glibc the target should be linked against. " + + "By default, a suitable version is chosen based on --cpu.") + public String glibc; + + @Option(name = "thin_archives", + defaultValue = "false", + category = "strategy", // but also adds edges to the action graph + help = "Pass the 'T' flag to ar if supported by the toolchain. " + + "All supported toolchains support this setting.") + public boolean useThinArchives; + + // O intrepid reaper of unused options: Be warned that the [no]start_end_lib + // option, however tempting to remove, has a use case. Look in our telemetry data. + @Option(name = "start_end_lib", + defaultValue = "true", + category = "strategy", // but also adds edges to the action graph + help = "Use the --start-lib/--end-lib ld options if supported by the toolchain.") + public boolean useStartEndLib; + + @Option(name = "interface_shared_objects", + defaultValue = "true", + category = "strategy", // but also adds edges to the action graph + help = "Use interface shared objects if supported by the toolchain. " + + "All ELF toolchains currently support this setting.") + public boolean useInterfaceSharedObjects; + + @Option(name = "cc_include_scanning", + defaultValue = "true", + category = "strategy", + help = "Whether to perform include scanning. Without it, your build will most likely " + + "fail.") + public boolean scanIncludes; + + @Option(name = "extract_generated_inclusions", + defaultValue = "true", + category = "undocumented", + help = "Run grep-includes actions (used for include scanning) over " + + "generated headers and sources.") + public boolean extractInclusions; + + @Option(name = "fission", + defaultValue = "no", + converter = FissionOptionConverter.class, + category = "semantics", + help = "Specifies which compilation modes use fission for C++ compilations and links. " + + " May be any combination of {'fastbuild', 'dbg', 'opt'} or the special values 'yes' " + + " to enable all modes and 'no' to disable all modes.") + public List<CompilationMode> fissionModes; + + @Option(name = "dynamic_mode", + defaultValue = "default", + converter = DynamicModeConverter.class, + category = "semantics", + help = "Determines whether C++ binaries will be linked dynamically. 'default' means " + + "blaze will choose whether to link dynamically. 'fully' means all libraries " + + "will be linked dynamically. 'off' means that all libraries will be linked " + + "in mostly static mode.") + public DynamicModeFlag dynamicMode; + + @Option(name = "force_pic", + defaultValue = "false", + category = "semantics", + help = "If enabled, all C++ compilations produce position-independent code (\"-fPIC\")," + + " links prefer PIC pre-built libraries over non-PIC libraries, and links produce" + + " position-independent executables (\"-pie\").") + public boolean forcePic; + + @Option(name = "force_ignore_dash_static", + defaultValue = "false", + category = "semantics", + help = "If set, '-static' options in the linkopts of cc_* rules will be ignored.") + public boolean forceIgnoreDashStatic; + + @Option(name = "experimental_skip_static_outputs", + defaultValue = "false", + category = "semantics", + help = "This flag is experimental and may go away at any time. " + + "If true, linker output for mostly-static C++ executables is a tiny amount of " + + "dummy dependency information, and NOT a usable binary. Kludge, but can reduce " + + "network and disk I/O load (and thus, continuous build cycle times) by a lot. " + + "NOTE: use of this flag REQUIRES --distinct_host_configuration.") + public boolean skipStaticOutputs; + + @Option(name = "hdrs_check", + allowMultiple = false, + defaultValue = "loose", + converter = HdrsCheckConverter.class, + category = "semantics", + help = "Headers check mode for rules that don't specify it explicitly using a " + + "hdrs_check attribute. Allowed values: 'loose' allows undeclared headers, 'warn' " + + "warns about undeclared headers, and 'strict' disallows them.") + public HeadersCheckingMode headersCheckingMode; + + @Option(name = "copt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to gcc.") + public List<String> coptList; + + @Option(name = "cwarn", + converter = GccWarnConverter.class, + defaultValue = "", + category = "flags", + allowMultiple = true, + help = "Additional warnings to enable when compiling C or C++ source files.") + public List<String> cWarns; + + @Option(name = "cxxopt", + defaultValue = "", + category = "flags", + allowMultiple = true, + help = "Additional option to pass to gcc when compiling C++ source files.") + public List<String> cxxoptList; + + @Option(name = "conlyopt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional option to pass to gcc when compiling C source files.") + public List<String> conlyoptList; + + @Option(name = "linkopt", + defaultValue = "", + category = "flags", + allowMultiple = true, + help = "Additional option to pass to gcc when linking.") + public List<String> linkoptList; + + @Option(name = "stripopt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to strip when generating a '<name>.stripped' binary.") + public List<String> stripoptList; + + @Option(name = "custom_malloc", + defaultValue = "null", + category = "semantics", + help = "Specifies a custom malloc implementation. This setting overrides malloc " + + "attributes in build rules.", + converter = LabelConverter.class) + public Label customMalloc; + + @Option(name = "cpp_module_maps", + defaultValue = "true", + category = "flags", + help = "If true then C++ targets create a module map based on BUILD files, and " + + "pass them to the compiler.") + public boolean cppModuleMaps; + + @Option(name = "legacy_whole_archive", + defaultValue = "true", + category = "semantics", + help = "When on, use --whole-archive for cc_binary rules that have " + + "linkshared=1 and either linkstatic=1 or '-static' in linkopts. " + + "This is for backwards compatibility only. " + + "A better alternative is to use alwayslink=1 where required.") + public boolean legacyWholeArchive; + + @Option(name = "strip", + defaultValue = "sometimes", + category = "flags", + help = "Specifies whether to strip binaries and shared libraries " + + " (using \"-Wl,--strip-debug\"). The default value of 'sometimes'" + + " means strip iff --compilation_mode=fastbuild.", + converter = StripModeConverter.class) + public StripMode stripBinaries; + + @Option(name = "fdo_instrument", + defaultValue = "null", + converter = OptionsUtils.PathFragmentConverter.class, + category = "flags", + implicitRequirements = {"--copt=-Wno-error"}, + help = "Generate binaries with FDO instrumentation. Specify the relative " + + "directory name for the .gcda files at runtime.") + public PathFragment fdoInstrument; + + @Option(name = "fdo_optimize", + defaultValue = "null", + category = "flags", + help = "Use FDO profile information to optimize compilation. Specify the name " + + "of the zip file containing the .gcda file tree or an afdo file containing " + + "an auto profile. This flag also accepts files specified as labels, for " + + "example //foo/bar:file.afdo. Such labels must refer to input files; you may " + + "need to add an exports_files directive to the corresponding package to make " + + "the file visible to Blaze.") + public String fdoOptimize; + + @Option(name = "autofdo_lipo_data", + defaultValue = "false", + category = "flags", + help = "If true then the directory name for non-LIPO targets will have a " + + "'-lipodata' suffix in AutoFDO mode.") + public boolean autoFdoLipoData; + + @Option(name = "lipo", + defaultValue = "off", + converter = LipoModeConverter.class, + category = "flags", + help = "Enable LIPO optimization (lightweight inter-procedural optimization, The allowed " + + "values for this option are 'off' and 'binary', which enables LIPO. This option only " + + "has an effect when FDO is also enabled. Currently LIPO is only supported when " + + "building a single cc_binary rule.") + public LipoMode lipoMode; + + @Option(name = "lipo_context", + defaultValue = "null", + category = "flags", + converter = LabelConverter.class, + implicitRequirements = {"--linkopt=-Wl,--warn-unresolved-symbols"}, + help = "Specifies the binary from which the LIPO profile information comes.") + public Label lipoContext; + + @Option(name = "experimental_stl", + converter = LabelConverter.class, + defaultValue = "null", + category = "version", + help = "If set, use this label instead of the default STL implementation. " + + "This option is EXPERIMENTAL and may go away in a future release.") + public Label stl; + + @Option(name = "save_temps", + defaultValue = "false", + category = "what", + help = "If set, temporary outputs from gcc will be saved. " + + "These include .s files (assembler code), .i files (preprocessed C) and " + + ".ii files (preprocessed C++).") + public boolean saveTemps; + + @Option(name = "per_file_copt", + allowMultiple = true, + converter = PerLabelOptions.PerLabelOptionsConverter.class, + defaultValue = "", + category = "semantics", + help = "Additional options to selectively pass to gcc when compiling certain files. " + + "This option can be passed multiple times. " + + "Syntax: regex_filter@option_1,option_2,...,option_n. Where regex_filter stands " + + "for a list of include and exclude regular expression patterns (Also see " + + "--instrumentation_filter). option_1 to option_n stand for " + + "arbitrary command line options. If an option contains a comma it has to be " + + "quoted with a backslash. Options can contain @. Only the first @ is used to " + + "split the string. Example: " + + "--per_file_copt=//foo/.*\\.cc,-//foo/bar\\.cc@-O0 adds the -O0 " + + "command line option to the gcc command line of all cc files in //foo/ " + + "except bar.cc.") + public List<PerLabelOptions> perFileCopts; + + @Option(name = "host_crosstool_top", + defaultValue = "null", + converter = LabelConverter.class, + category = "semantics", + help = "By default, the --crosstool_top, --glibc, and --compiler options are also used " + + "for the host configuration. If this flag is provided, Blaze uses the default glibc " + + "and compiler for the given crosstool_top.") + public Label hostCrosstoolTop; + + @Option(name = "host_copt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to gcc for host tools.") + public List<String> hostCoptList; + + @Option(name = "define", + converter = Converters.AssignmentConverter.class, + defaultValue = "", + category = "semantics", + allowMultiple = true, + help = "Each --define option specifies an assignment for a build variable.") + public List<Map.Entry<String, String>> commandLineDefinedVariables; + + @Option(name = "grte_top", + defaultValue = "null", // The default value is chosen by the toolchain. + category = "version", + converter = LibcTopConverter.class, + help = "A label to a checked-in libc library. The default value is selected by the crosstool " + + "toolchain, and you almost never need to override it.") + public LibcTop libcTop; + + @Option(name = "host_grte_top", + defaultValue = "null", // The default value is chosen by the toolchain. + category = "version", + converter = LibcTopConverter.class, + help = "If specified, this setting overrides the libc top-level directory (--grte_top) " + + "for the host configuration.") + public LibcTop hostLibcTop; + + @Option(name = "output_symbol_counts", + defaultValue = "false", + category = "flags", + help = "If enabled, every C++ binary linked with gold will store the number of used " + + "symbols per object file in a .sc file.") + public boolean symbolCounts; + + @Option(name = "experimental_inmemory_dotd_files", + defaultValue = "false", + category = "experimental", + help = "If enabled, C++ .d files will be passed through in memory directly from the remote " + + "build nodes instead of being written to disk.") + public boolean inmemoryDotdFiles; + + @Option(name = "use_isystem_for_includes", + defaultValue = "true", + category = "undocumented", + help = "Instruct C and C++ compilations to treat 'includes' paths as system header " + + "paths, by translating it into -isystem instead of -I.") + public boolean useIsystemForIncludes; + + @Option(name = "experimental_omitfp", + defaultValue = "false", + category = "semantics", + help = "If true, use libunwind for stack unwinding, and compile with " + + "-fomit-frame-pointer and -fasynchronous-unwind-tables.") + public boolean experimentalOmitfp; + + @Option(name = "share_native_deps", + defaultValue = "true", + category = "strategy", + help = "If true, native libraries that contain identical functionality " + + "will be shared among different targets") + public boolean shareNativeDeps; + + @Override + public FragmentOptions getHost(boolean fallback) { + CppOptions host = (CppOptions) getDefault(); + + host.commandLineDefinedVariables = commandLineDefinedVariables; + + // The crosstool options are partially copied from the target configuration. + if (!fallback) { + if (hostCrosstoolTop == null) { + host.cppCompiler = cppCompiler; + host.crosstoolTop = crosstoolTop; + host.glibc = glibc; + } else { + host.crosstoolTop = hostCrosstoolTop; + } + } + + if (hostLibcTop != null) { + host.libcTop = hostLibcTop; + } else if (hostCrosstoolTop == null) { + // Track libc in the host configuration if no host crosstool is set. + host.libcTop = libcTop; + } + + // -g0 is the default, but allowMultiple options cannot have default values so we just pass + // -g0 first and let the user options override it. + host.coptList = ImmutableList.<String>builder().add("-g0").addAll(hostCoptList).build(); + + host.useThinArchives = useThinArchives; + host.useStartEndLib = useStartEndLib; + host.extractInclusions = extractInclusions; + host.stripBinaries = StripMode.ALWAYS; + host.fdoOptimize = null; + host.lipoMode = LipoMode.OFF; + host.scanIncludes = scanIncludes; + host.inmemoryDotdFiles = inmemoryDotdFiles; + host.cppModuleMaps = cppModuleMaps; + + return host; + } + + @Override + public void addAllLabels(Multimap<String, Label> labelMap) { + labelMap.put("crosstool", crosstoolTop); + if (hostCrosstoolTop != null) { + labelMap.put("crosstool", hostCrosstoolTop); + } + + if (libcTop != null) { + Label libcLabel = libcTop.getLabel(); + if (libcLabel != null) { + labelMap.put("crosstool", libcLabel); + } + } + addOptionalLabel(labelMap, "fdo", fdoOptimize); + + if (stl != null) { + labelMap.put("STL", stl); + } + + if (customMalloc != null) { + labelMap.put("custom_malloc", customMalloc); + } + + if (getLipoContextLabel() != null) { + labelMap.put("lipo", getLipoContextLabel()); + } + } + + @Override + public Map<String, Set<Label>> getDefaultsLabels(BuildConfiguration.Options commonOptions) { + Set<Label> crosstoolLabels = new LinkedHashSet<>(); + crosstoolLabels.add(crosstoolTop); + if (hostCrosstoolTop != null) { + crosstoolLabels.add(hostCrosstoolTop); + } + + if (libcTop != null) { + Label libcLabel = libcTop.getLabel(); + if (libcLabel != null) { + crosstoolLabels.add(libcLabel); + } + } + + return ImmutableMap.of( + "CROSSTOOL", crosstoolLabels, + "COVERAGE", ImmutableSet.<Label>of()); + } + + public boolean isFdo() { + return fdoOptimize != null || fdoInstrument != null; + } + + public boolean isLipoOptimization() { + return lipoMode == LipoMode.BINARY && fdoOptimize != null && lipoContext != null; + } + + public boolean isLipoOptimizationOrInstrumentation() { + return lipoMode == LipoMode.BINARY && + ((fdoOptimize != null && lipoContext != null) || fdoInstrument != null); + } + + public Label getLipoContextLabel() { + return (lipoMode == LipoMode.BINARY && fdoOptimize != null) + ? lipoContext : null; + } + + public LipoMode getLipoMode() { + return lipoMode; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java new file mode 100644 index 0000000..de7c95d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java
@@ -0,0 +1,104 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ALWAYS_LINK_LIBRARY; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ALWAYS_LINK_PIC_LIBRARY; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ARCHIVE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.CPP_HEADER; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.CPP_SOURCE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.C_SOURCE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.OBJECT_FILE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.PIC_ARCHIVE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.PIC_OBJECT_FILE; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.SHARED_LIBRARY; +import static com.google.devtools.build.lib.rules.cpp.CppFileTypes.VERSIONED_SHARED_LIBRARY; + +import com.google.devtools.build.lib.analysis.LanguageDependentFragment.LibraryLanguage; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec; +import com.google.devtools.build.lib.util.FileTypeSet; + +/** + * Rule class definitions for C++ rules. + */ +public class CppRuleClasses { + // Artifacts of these types are discarded from the 'hdrs' attribute in cc rules + static final FileTypeSet DISALLOWED_HDRS_FILES = FileTypeSet.of( + ARCHIVE, + PIC_ARCHIVE, + ALWAYS_LINK_LIBRARY, + ALWAYS_LINK_PIC_LIBRARY, + SHARED_LIBRARY, + VERSIONED_SHARED_LIBRARY, + OBJECT_FILE, + PIC_OBJECT_FILE); + + /** + * The set of instrumented source file types; keep this in sync with the list above. Note that + * extension-less header files cannot currently be declared, so we cannot collect coverage for + * those. + */ + static final InstrumentationSpec INSTRUMENTATION_SPEC = new InstrumentationSpec( + FileTypeSet.of(CPP_SOURCE, C_SOURCE, CPP_HEADER, ASSEMBLER_WITH_C_PREPROCESSOR), + "srcs", "deps", "data", "hdrs", "implements", "implementation"); + + public static final LibraryLanguage LANGUAGE = new LibraryLanguage("C++"); + + /** + * Implicit outputs for cc_binary rules. + */ + public static final SafeImplicitOutputsFunction CC_BINARY_STRIPPED = + fromTemplates("%{name}.stripped"); + + + // Used for requesting dwp "debug packages". + public static final SafeImplicitOutputsFunction CC_BINARY_DEBUG_PACKAGE = + fromTemplates("%{name}.dwp"); + + + /** + * Path of the build_interface_so script in the Blaze binary. + */ + public static final String BUILD_INTERFACE_SO = "build_interface_so"; + + /** + * A string constant for the layering_check feature. + */ + public static final String LAYERING_CHECK = "layering_check"; + + /** + * A string constant for the parse_headers feature. + */ + public static final String PARSE_HEADERS = "parse_headers"; + + /** + * A string constant for the preprocess_headers feature. + */ + public static final String PREPROCESS_HEADERS = "preprocess_headers"; + + /** + * A string constant for the header_modules feature. + */ + public static final String HEADER_MODULES = "header_modules"; + + /** + * A string constant for the module_map_home_cwd feature. + */ + public static final String MODULE_MAP_HOME_CWD = "module_map_home_cwd"; + +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRunfilesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRunfilesProvider.java new file mode 100644 index 0000000..f4aa38c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRunfilesProvider.java
@@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Runfiles provider for C++ targets. + * + * <p>Contains two {@link Runfiles} objects: one for the eventual statically linked binary and + * one for the one that uses shared libraries. Data dependencies are present in both. + */ +@Immutable +public final class CppRunfilesProvider implements TransitiveInfoProvider { + private final Runfiles staticRunfiles; + private final Runfiles sharedRunfiles; + + public CppRunfilesProvider(Runfiles staticRunfiles, Runfiles sharedRunfiles) { + this.staticRunfiles = staticRunfiles; + this.sharedRunfiles = sharedRunfiles; + } + + public Runfiles getStaticRunfiles() { + return staticRunfiles; + } + + public Runfiles getSharedRunfiles() { + return sharedRunfiles; + } + + /** + * Returns a function that gets the static C++ runfiles from a {@link TransitiveInfoCollection} + * or the empty runfiles instance if it does not contain that provider. + */ + public static final Function<TransitiveInfoCollection, Runfiles> STATIC_RUNFILES = + new Function<TransitiveInfoCollection, Runfiles>() { + @Override + public Runfiles apply(TransitiveInfoCollection input) { + CppRunfilesProvider provider = input.getProvider(CppRunfilesProvider.class); + return provider == null + ? Runfiles.EMPTY + : provider.getStaticRunfiles(); + } + }; + + /** + * Returns a function that gets the shared C++ runfiles from a {@link TransitiveInfoCollection} + * or the empty runfiles instance if it does not contain that provider. + */ + public static final Function<TransitiveInfoCollection, Runfiles> SHARED_RUNFILES = + new Function<TransitiveInfoCollection, Runfiles>() { + @Override + public Runfiles apply(TransitiveInfoCollection input) { + CppRunfilesProvider provider = input.getProvider(CppRunfilesProvider.class); + return provider == null + ? Runfiles.EMPTY + : provider.getSharedRunfiles(); + } + }; + + /** + * Returns a function that gets the C++ runfiles from a {@link TransitiveInfoCollection} or + * the empty runfiles instance if it does not contain that provider. + */ + public static final Function<TransitiveInfoCollection, Runfiles> runfilesFunction( + boolean linkingStatically) { + return linkingStatically ? STATIC_RUNFILES : SHARED_RUNFILES; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppSemantics.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppSemantics.java new file mode 100644 index 0000000..600b2fa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppSemantics.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Pluggable C++ compilation semantics. + */ +public interface CppSemantics { + /** + * Returns the "effective source path" of a source file. + * + * <p>It is used, among other things, for computing the output path. + */ + PathFragment getEffectiveSourcePath(Artifact source); + + /** + * Called before a C++ compile action is built. + * + * <p>Gives the semantics implementation the opportunity to change compile actions at the last + * minute. + */ + void finalizeCompileActionBuilder( + RuleContext ruleContext, CppCompileActionBuilder actionBuilder); + + /** + * Called before {@link CppCompilationContext}s are finalized. + * + * <p>Gives the semantics implementation the opportunity to change what the C++ rule propagates + * to dependent rules. + */ + void setupCompilationContext( + RuleContext ruleContext, CppCompilationContext.Builder contextBuilder); +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationIdentifier.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationIdentifier.java new file mode 100644 index 0000000..1111189 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationIdentifier.java
@@ -0,0 +1,132 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CToolchain; + +import java.util.Objects; + +/** + * Contains parameters which uniquely describe a crosstool configuration + * and methods for comparing two crosstools against each other. + * + * <p>Two crosstools which contain equivalent values of these parameters are + * considered equal. + */ +public final class CrosstoolConfigurationIdentifier implements CrosstoolConfigurationOptions { + + /** The CPU associated with this crosstool configuration. */ + private final String cpu; + + /** The compiler (e.g. gcc) associated with this crosstool configuration. */ + private final String compiler; + + /** The version of libc (e.g. glibc-2.11) associated with this crosstool configuration. */ + private final String libc; + + private CrosstoolConfigurationIdentifier(String cpu, String compiler, String libc) { + this.cpu = cpu; + this.compiler = compiler; + this.libc = libc; + } + + /** + * Creates a new crosstool configuration from the given crosstool release and + * configuration options. + */ + public static CrosstoolConfigurationIdentifier fromReleaseAndCrosstoolConfiguration( + CrosstoolConfig.CrosstoolRelease release, BuildOptions buildOptions) { + String cpu = buildOptions.get(BuildConfiguration.Options.class).getCpu(); + if (cpu == null) { + cpu = release.getDefaultTargetCpu(); + } + CppOptions cppOptions = buildOptions.get(CppOptions.class); + return new CrosstoolConfigurationIdentifier(cpu, cppOptions.cppCompiler, cppOptions.glibc); + } + + public static CrosstoolConfigurationIdentifier fromToolchain(CToolchain toolchain) { + return new CrosstoolConfigurationIdentifier( + toolchain.getTargetCpu(), toolchain.getCompiler(), toolchain.getTargetLibc()); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof CrosstoolConfigurationIdentifier)) { + return false; + } + CrosstoolConfigurationIdentifier otherCrosstool = (CrosstoolConfigurationIdentifier) other; + return Objects.equals(cpu, otherCrosstool.cpu) + && Objects.equals(compiler, otherCrosstool.compiler) + && Objects.equals(libc, otherCrosstool.libc); + } + + @Override + public int hashCode() { + return Objects.hash(cpu, compiler, libc); + } + + + /** + * Returns a series of command line flags which specify the configuration options. + * Any of these options may be null, in which case its flag is omitted. + * + * <p>The appended string will be along the lines of + * " --cpu='cpu' --compiler='compiler' --glibc='libc'". + */ + public String describeFlags() { + StringBuilder message = new StringBuilder(); + if (getCpu() != null) { + message.append(" --cpu='").append(getCpu()).append("'"); + } + if (getCompiler() != null) { + message.append(" --compiler='").append(getCompiler()).append("'"); + } + if (getLibc() != null) { + message.append(" --glibc='").append(getLibc()).append("'"); + } + return message.toString(); + } + + /** Returns true if the specified toolchain is a candidate for use with this crosstool. */ + public boolean isCandidateToolchain(CToolchain toolchain) { + return (toolchain.getTargetCpu().equals(getCpu()) + && (getLibc() == null || toolchain.getTargetLibc().equals(getLibc())) + && (getCompiler() == null || toolchain.getCompiler().equals( + getCompiler()))); + } + + @Override + public String toString() { + return describeFlags(); + } + + @Override + public String getCpu() { + return cpu; + } + + @Override + public String getCompiler() { + return compiler; + } + + @Override + public String getLibc() { + return libc; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java new file mode 100644 index 0000000..a113f5f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java
@@ -0,0 +1,327 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.analysis.RedirectChaser; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig; +import com.google.protobuf.TextFormat; +import com.google.protobuf.TextFormat.ParseException; +import com.google.protobuf.UninitializedMessageException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.ExecutionException; + +import javax.annotation.Nullable; + +/** + * A loader that reads Crosstool configuration files and creates CToolchain + * instances from them. + */ +public class CrosstoolConfigurationLoader { + private static final String CROSSTOOL_CONFIGURATION_FILENAME = "CROSSTOOL"; + + /** + * Cache for storing result of toReleaseConfiguration function based on path and md5 sum of + * input file. We can use md5 because result of this function depends only on the file content. + */ + private static final LoadingCache<Pair<Path, String>, CrosstoolConfig.CrosstoolRelease> + crosstoolReleaseCache = CacheBuilder.newBuilder().concurrencyLevel(4).maximumSize(100).build( + new CacheLoader<Pair<Path, String>, CrosstoolConfig.CrosstoolRelease>() { + @Override + public CrosstoolConfig.CrosstoolRelease load(Pair<Path, String> key) throws IOException { + char[] data = FileSystemUtils.readContentAsLatin1(key.first); + return toReleaseConfiguration(key.first.getPathString(), new String(data)); + } + }); + + /** + * A class that holds the results of reading a CROSSTOOL file. + */ + public static class CrosstoolFile { + private final Label crosstoolTop; + private Path crosstoolPath; + private CrosstoolConfig.CrosstoolRelease crosstool; + private String md5; + + CrosstoolFile(Label crosstoolTop) { + this.crosstoolTop = crosstoolTop; + } + + void setCrosstoolPath(Path crosstoolPath) { + this.crosstoolPath = crosstoolPath; + } + + void setCrosstool(CrosstoolConfig.CrosstoolRelease crosstool) { + this.crosstool = crosstool; + } + + void setMd5(String md5) { + this.md5 = md5; + } + + /** + * Returns the crosstool top as resolved. + */ + public Label getCrosstoolTop() { + return crosstoolTop; + } + + /** + * Returns the absolute path from which the CROSSTOOL file was read. + */ + public Path getCrosstoolPath() { + return crosstoolPath; + } + + /** + * Returns the parsed contents of the CROSSTOOL file. + */ + public CrosstoolConfig.CrosstoolRelease getProto() { + return crosstool; + } + + /** + * Returns an MD5 hash of the CROSSTOOL file contents. + */ + public String getMd5() { + return md5; + } + } + + private CrosstoolConfigurationLoader() { + } + + /** + * Reads the given <code>data</code> String, which must be in ascii format, + * into a protocol buffer. It uses the <code>name</code> parameter for error + * messages. + * + * @throws IOException if the parsing failed + */ + @VisibleForTesting + static CrosstoolConfig.CrosstoolRelease toReleaseConfiguration(String name, String data) + throws IOException { + CrosstoolConfig.CrosstoolRelease.Builder builder = + CrosstoolConfig.CrosstoolRelease.newBuilder(); + try { + TextFormat.merge(data, builder); + return builder.build(); + } catch (ParseException e) { + throw new IOException("Could not read the crosstool configuration file '" + name + "', " + + "because of a parser error (" + e.getMessage() + ")"); + } catch (UninitializedMessageException e) { + throw new IOException("Could not read the crosstool configuration file '" + name + "', " + + "because of an incomplete protocol buffer (" + e.getMessage() + ")"); + } + } + + private static boolean findCrosstoolConfiguration( + ConfigurationEnvironment env, + CrosstoolConfigurationLoader.CrosstoolFile file) + throws IOException, InvalidConfigurationException { + Label crosstoolTop = file.getCrosstoolTop(); + Path path = null; + try { + Package containingPackage = env.getTarget(crosstoolTop.getLocalTargetLabel("BUILD")) + .getPackage(); + if (containingPackage == null) { + return false; + } + path = env.getPath(containingPackage, CROSSTOOL_CONFIGURATION_FILENAME); + } catch (SyntaxException e) { + throw new InvalidConfigurationException(e); + } catch (NoSuchThingException e) { + // Handled later + } + + // If we can't find a file, fall back to the provided alternative. + if (path == null || !path.exists()) { + throw new InvalidConfigurationException("The crosstool_top you specified was resolved to '" + + crosstoolTop + "', which does not contain a CROSSTOOL file. " + + "You can use a crosstool from the depot by specifying its label."); + } else { + // Do this before we read the data, so if it changes, we get a different MD5 the next time. + // Alternatively, we could calculate the MD5 of the contents, which we also read, but this + // is faster if the file comes from a file system with md5 support. + file.setCrosstoolPath(path); + String md5 = BaseEncoding.base16().lowerCase().encode(path.getMD5Digest()); + CrosstoolConfig.CrosstoolRelease release; + try { + release = crosstoolReleaseCache.get(new Pair<Path, String>(path, md5)); + file.setCrosstool(release); + file.setMd5(md5); + } catch (ExecutionException e) { + throw new InvalidConfigurationException(e); + } + } + return true; + } + + /** + * Reads a crosstool file. + */ + @Nullable + public static CrosstoolConfigurationLoader.CrosstoolFile readCrosstool( + ConfigurationEnvironment env, Label crosstoolTop) throws InvalidConfigurationException { + crosstoolTop = RedirectChaser.followRedirects(env, crosstoolTop, "crosstool_top"); + if (crosstoolTop == null) { + return null; + } + CrosstoolConfigurationLoader.CrosstoolFile file = + new CrosstoolConfigurationLoader.CrosstoolFile(crosstoolTop); + try { + boolean allDependenciesPresent = findCrosstoolConfiguration(env, file); + return allDependenciesPresent ? file : null; + } catch (IOException e) { + throw new InvalidConfigurationException(e); + } + } + + /** + * Selects a crosstool toolchain corresponding to the given crosstool + * configuration options. If all of these options are null, it returns the default + * toolchain specified in the crosstool release. If only cpu is non-null, it + * returns the default toolchain for that cpu, as specified in the crosstool + * release. Otherwise, all values must be non-null, and this method + * returns the toolchain which matches all of the values. + * + * @throws NullPointerException if {@code release} is null + * @throws InvalidConfigurationException if no matching toolchain can be found, or + * if the input parameters do not obey the constraints described above + */ + public static CrosstoolConfig.CToolchain selectToolchain( + CrosstoolConfig.CrosstoolRelease release, BuildOptions options, + Function<String, String> cpuTransformer) + throws InvalidConfigurationException { + CrosstoolConfigurationIdentifier config = + CrosstoolConfigurationIdentifier.fromReleaseAndCrosstoolConfiguration(release, options); + if ((config.getCompiler() != null) || (config.getLibc() != null)) { + ArrayList<CrosstoolConfig.CToolchain> candidateToolchains = new ArrayList<>(); + for (CrosstoolConfig.CToolchain toolchain : release.getToolchainList()) { + if (config.isCandidateToolchain(toolchain)) { + candidateToolchains.add(toolchain); + } + } + switch (candidateToolchains.size()) { + case 0: { + StringBuilder message = new StringBuilder(); + message.append("No toolchain found for"); + message.append(config.describeFlags()); + message.append(". Valid toolchains are: "); + describeToolchainList(message, release.getToolchainList()); + throw new InvalidConfigurationException(message.toString()); + } + case 1: + return candidateToolchains.get(0); + default: { + StringBuilder message = new StringBuilder(); + message.append("Multiple toolchains found for"); + message.append(config.describeFlags()); + message.append(": "); + describeToolchainList(message, candidateToolchains); + throw new InvalidConfigurationException(message.toString()); + } + } + } + String selectedIdentifier = null; + // We use fake CPU values to allow cross-platform builds for other languages that use the + // C++ toolchain. Translate to the actual target architecture. + String desiredCpu = cpuTransformer.apply(config.getCpu()); + for (CrosstoolConfig.DefaultCpuToolchain selector : release.getDefaultToolchainList()) { + if (selector.getCpu().equals(desiredCpu)) { + selectedIdentifier = selector.getToolchainIdentifier(); + break; + } + } + checkToolChain(selectedIdentifier, desiredCpu); + for (CrosstoolConfig.CToolchain toolchain : release.getToolchainList()) { + if (toolchain.getToolchainIdentifier().equals(selectedIdentifier)) { + return toolchain; + } + } + throw new InvalidConfigurationException("Inconsistent crosstool configuration; no toolchain " + + "corresponding to '" + selectedIdentifier + "' found for cpu '" + config.getCpu() + "'"); + } + + private static String describeToolchainFlags(CrosstoolConfig.CToolchain toolchain) { + return CrosstoolConfigurationIdentifier.fromToolchain(toolchain).describeFlags(); + } + + /** + * Appends a series of toolchain descriptions (as the blaze command line flags + * that would specify that toolchain) to 'message'. + */ + private static void describeToolchainList(StringBuilder message, + Collection<CrosstoolConfig.CToolchain> toolchains) { + message.append("["); + for (CrosstoolConfig.CToolchain toolchain : toolchains) { + message.append(describeToolchainFlags(toolchain)); + message.append(","); + } + message.append("]"); + } + + /** + * Makes sure that {@code selectedIdentifier} is a valid identifier for a toolchain, + * i.e. it starts with a letter or an underscore and continues with only dots, dashes, + * spaces, letters, digits or underscores (i.e. matches the following regular expression: + * "[a-zA-Z_][\.\- \w]*"). + * + * @throws InvalidConfigurationException if selectedIdentifier is null or does not match the + * aforementioned regular expression. + */ + private static void checkToolChain(String selectedIdentifier, String cpu) + throws InvalidConfigurationException { + if (selectedIdentifier == null) { + throw new InvalidConfigurationException("No toolchain found for cpu '" + cpu + "'"); + } + // If you update this regex, please do so in the javadoc comment too, and also in the + // crosstool_config.proto file. + String rx = "[a-zA-Z_][\\.\\- \\w]*"; + if (!selectedIdentifier.matches(rx)) { + throw new InvalidConfigurationException("Toolchain identifier for cpu '" + cpu + "' " + + "is illegal (does not match '" + rx + "')"); + } + } + + public static CrosstoolConfig.CrosstoolRelease getCrosstoolReleaseProto( + ConfigurationEnvironment env, BuildOptions options, + Label crosstoolTop, Function<String, String> cpuTransformer) + throws InvalidConfigurationException { + CrosstoolConfigurationLoader.CrosstoolFile file = + readCrosstool(env, crosstoolTop); + // Make sure that we have the requested toolchain in the result. Throw an exception if not. + selectToolchain(file.getProto(), options, cpuTransformer); + return file.getProto(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationOptions.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationOptions.java new file mode 100644 index 0000000..e311ab6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationOptions.java
@@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +/** + * A container object which provides crosstool configuration options to the build. + */ +public interface CrosstoolConfigurationOptions { + /** Returns the CPU associated with this crosstool configuration. */ + public String getCpu(); + + /** Returns the compiler associated with this crosstool configuration. */ + public String getCompiler(); + + /** Returns the libc version associated with this crosstool configuration. */ + public String getLibc(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/DiscoveredSourceInputsHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/DiscoveredSourceInputsHelper.java new file mode 100644 index 0000000..a446125 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/DiscoveredSourceInputsHelper.java
@@ -0,0 +1,139 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Helper for actions that do include scanning. Currently only deals with source files, so is only + * appropriate for actions that do not discover generated files. Currently does not do .d file + * parsing, so the set of artifacts returned may be an overapproximation to the ones actually used + * during execution. + */ +public class DiscoveredSourceInputsHelper { + + private DiscoveredSourceInputsHelper() { + } + + /** + * Converts PathFragments into source Artifacts using an ArtifactResolver, ignoring any that are + * already in mandatoryInputs. Silently drops any PathFragments that cannot be resolved into + * Artifacts. + */ + public static ImmutableList<Artifact> getDiscoveredInputsFromPaths( + Iterable<Artifact> mandatoryInputs, ArtifactResolver artifactResolver, + Collection<PathFragment> inputPaths) { + Set<PathFragment> knownPathFragments = new HashSet<>(); + for (Artifact input : mandatoryInputs) { + knownPathFragments.add(input.getExecPath()); + } + ImmutableList.Builder<Artifact> foundInputs = ImmutableList.builder(); + for (PathFragment execPath : inputPaths) { + if (!knownPathFragments.add(execPath)) { + // Don't add any inputs that we already added, or original inputs, which we probably + // couldn't convert into artifacts anyway. + continue; + } + Artifact artifact = artifactResolver.resolveSourceArtifact(execPath); + // It is unlikely that this artifact is null, but tolerate the situation just in case. + // It is safe to ignore such paths because dependency checker would identify change in inputs + // (ignored path was used before) and will force action execution. + if (artifact != null) { + foundInputs.add(artifact); + } + } + return foundInputs.build(); + } + + /** + * Converts ActionInputs discovered as inputs during execution into source Artifacts, ignoring any + * that are already in mandatoryInputs or that live in builtInIncludeDirectories. If any + * ActionInputs cannot be resolved, an ActionExecutionException will be thrown. + * + * <p>This method duplicates the functionality of CppCompileAction#populateActionInputs, though it + * is simpler because it need not deal with derived artifacts and doesn't parse the .d file. + */ + public static ImmutableList<Artifact> getDiscoveredInputsFromActionInputs( + Iterable<Artifact> mandatoryInputs, + ArtifactResolver artifactResolver, + Iterable<? extends ActionInput> discoveredInputs, + Iterable<PathFragment> builtInIncludeDirectories, + Action action, + Artifact primaryInput) throws ActionExecutionException { + List<PathFragment> systemIncludePrefixes = new ArrayList<>(); + for (PathFragment includePath : builtInIncludeDirectories) { + if (includePath.isAbsolute()) { + systemIncludePrefixes.add(includePath); + } + } + + // Avoid duplicates by keeping track of the ones we've seen so far, even though duplicates are + // unlikely, since they would have to be inputs to this (non-CppCompile) action and also + // #included by a C++ source file. + Set<Artifact> knownInputs = new HashSet<>(); + Iterables.addAll(knownInputs, mandatoryInputs); + ImmutableList.Builder<Artifact> foundInputs = ImmutableList.builder(); + // Check inclusions. + IncludeProblems problems = new IncludeProblems(); + for (ActionInput input : discoveredInputs) { + if (input instanceof Artifact) { + Artifact artifact = (Artifact) input; + if (knownInputs.add(artifact)) { + foundInputs.add(artifact); + } + continue; + } + PathFragment execPath = new PathFragment(input.getExecPathString()); + if (execPath.isAbsolute()) { + // Absolute includes from system paths are ignored. + if (FileSystemUtils.startsWithAny(execPath, systemIncludePrefixes)) { + continue; + } + // Theoretically, the more sophisticated logic of CppCompileAction#populateActioInputs could + // be used here, to allow absolute includes that started with the execRoot. However, since + // we don't hit this codepath for local execution, that should be unnecessary. If and when + // we examine the results of local execution for scanned includes, that case may need to be + // dealt with. + problems.add(execPath.getPathString()); + } + Artifact artifact = artifactResolver.resolveSourceArtifact(execPath); + if (artifact != null) { + if (knownInputs.add(artifact)) { + foundInputs.add(artifact); + } + } else { + // Abort if we see files that we can't resolve, likely caused by + // undeclared includes or illegal include constructs. + problems.add(execPath.getPathString()); + } + } + problems.assertProblemFree(action, primaryInput); + return foundInputs.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/DwoArtifactsCollector.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/DwoArtifactsCollector.java new file mode 100644 index 0000000..142a67a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/DwoArtifactsCollector.java
@@ -0,0 +1,120 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; + +/** + * Provides generic functionality for collecting the .dwo artifacts produced by any target + * that compiles C++ files. Supports both transitive and "only direct outputs" collection. + * Provides accessors for both PIC and non-PIC compilation modes. + */ +public class DwoArtifactsCollector { + + /** + * The .dwo files collected by this target in non-PIC compilation mode (i.e. myobject.dwo). + */ + private final NestedSet<Artifact> dwoArtifacts; + + /** + * The .dwo files collected by this target in PIC compilation mode (i.e. myobject.pic.dwo). + */ + private final NestedSet<Artifact> picDwoArtifacts; + + /** + * Instantiates a "real" collector on meaningful data. + */ + private DwoArtifactsCollector(CcCompilationOutputs compilationOutputs, + Iterable<TransitiveInfoCollection> deps) { + + Preconditions.checkNotNull(compilationOutputs); + Preconditions.checkNotNull(deps); + + // Note: .dwo collection works fine with any order, but tests may assume a + // specific order for readability / simplicity purposes. See + // DebugInfoPackagingTest for details. + NestedSetBuilder<Artifact> dwoBuilder = NestedSetBuilder.compileOrder(); + NestedSetBuilder<Artifact> picDwoBuilder = NestedSetBuilder.compileOrder(); + + dwoBuilder.addAll(compilationOutputs.getDwoFiles()); + picDwoBuilder.addAll(compilationOutputs.getPicDwoFiles()); + + for (TransitiveInfoCollection info : deps) { + CppDebugFileProvider provider = info.getProvider(CppDebugFileProvider.class); + if (provider != null) { + dwoBuilder.addTransitive(provider.getTransitiveDwoFiles()); + picDwoBuilder.addTransitive(provider.getTransitivePicDwoFiles()); + } + } + + dwoArtifacts = dwoBuilder.build(); + picDwoArtifacts = picDwoBuilder.build(); + } + + /** + * Instantiates an empty collector. + */ + private DwoArtifactsCollector() { + dwoArtifacts = NestedSetBuilder.<Artifact>emptySet(Order.COMPILE_ORDER); + picDwoArtifacts = NestedSetBuilder.<Artifact>emptySet(Order.COMPILE_ORDER); + } + + /** + * Returns a new instance that collects direct outputs and transitive dependencies. + * + * @param compilationOutputs the output compilation context for the owning target + * @param deps which of the target's transitive info collections should be visited + */ + public static DwoArtifactsCollector transitiveCollector(CcCompilationOutputs compilationOutputs, + Iterable<TransitiveInfoCollection> deps) { + return new DwoArtifactsCollector(compilationOutputs, deps); + } + + /** + * Returns a new instance that collects direct outputs only. + * + * @param compilationOutputs the output compilation context for the owning target + */ + public static DwoArtifactsCollector directCollector(CcCompilationOutputs compilationOutputs) { + return new DwoArtifactsCollector( + compilationOutputs, ImmutableList.<TransitiveInfoCollection>of()); + } + + /** + * Returns a new instance that doesn't collect anything (its artifact sets are empty). + */ + public static DwoArtifactsCollector emptyCollector() { + return new DwoArtifactsCollector(); + } + + /** + * Returns the .dwo files applicable to non-PIC compilation mode (i.e. myobject.dwo). + */ + public NestedSet<Artifact> getDwoArtifacts() { + return dwoArtifacts; + } + + /** + * Returns the .dwo files applicable to PIC compilation mode (i.e. myobject.pic.dwo). + */ + public NestedSet<Artifact> getPicDwoArtifacts() { + return picDwoArtifacts; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/ExtractInclusionAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/ExtractInclusionAction.java new file mode 100644 index 0000000..15d7010 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/ExtractInclusionAction.java
@@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; + +import java.io.IOException; + +/** + * An action which greps for includes over a given .cc or .h file. + * This is a part of the work required for C++ include scanning. + * + * <p>Note that this may run grep-includes over-optimistically, where we previously + * had not. For example, consider a cc_library of generated headers. If another + * library depends on it, and only references one of the headers, the other + * grep-includes will have been wasted. + */ +final class ExtractInclusionAction extends AbstractAction { + + private static final String GUID = "45b43e5a-4734-43bb-a05e-012313808142"; + + /** + * Constructs a new action. + */ + public ExtractInclusionAction(ActionOwner owner, Artifact input, Artifact output) { + super(owner, ImmutableList.of(input), ImmutableList.of(output)); + } + + @Override + protected String computeKey() { + return GUID; + } + + @Override + public String describeStrategy(Executor executor) { + return executor.getContext(CppCompileActionContext.class).strategyLocality(); + } + + @Override + public String getMnemonic() { + return "GrepIncludes"; + } + + @Override + protected String getRawProgressMessage() { + return "Extracting include lines from " + getPrimaryInput().prettyPrint(); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + IncludeScanningContext context = executor.getContext(IncludeScanningContext.class); + try { + context.extractIncludes(actionExecutionContext, this, getPrimaryInput(), + getPrimaryOutput()); + } catch (IOException e) { + throw new ActionExecutionException(e, this, false); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FakeCppCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FakeCppCompileAction.java new file mode 100644 index 0000000..bd15455 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FakeCppCompileAction.java
@@ -0,0 +1,212 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.UUID; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +/** + * Action that represents a fake C++ compilation step. + */ +@ThreadCompatible +public class FakeCppCompileAction extends CppCompileAction { + + private static final Logger LOG = Logger.getLogger(FakeCppCompileAction.class.getName()); + + public static final UUID GUID = UUID.fromString("b2d95c91-1434-47ae-a786-816017de8494"); + + private final PathFragment tempOutputFile; + + FakeCppCompileAction(ActionOwner owner, + ImmutableList<String> features, + FeatureConfiguration featureConfiguration, + Artifact sourceFile, + Label sourceLabel, + NestedSet<Artifact> mandatoryInputs, + Artifact outputFile, + PathFragment tempOutputFile, + DotdFile dotdFile, + BuildConfiguration configuration, + CppConfiguration cppConfiguration, + CppCompilationContext context, + ImmutableList<String> copts, + ImmutableList<String> pluginOpts, + Predicate<String> nocopts, + ImmutableList<PathFragment> extraSystemIncludePrefixes, + boolean enableLayeringCheck, + @Nullable String fdoBuildStamp) { + super(owner, features, featureConfiguration, sourceFile, sourceLabel, mandatoryInputs, + outputFile, dotdFile, null, null, null, + configuration, cppConfiguration, + // We only allow inclusion of header files explicitly declared in + // "srcs", so we only use declaredIncludeSrcs, not declaredIncludeDirs. + // (Disallowing use of undeclared headers for cc_fake_binary is needed + // because the header files get included in the runfiles for the + // cc_fake_binary and for the negative compilation tests that depend on + // the cc_fake_binary, and the runfiles must be determined at analysis + // time, so they can't depend on the contents of the ".d" file.) + CppCompilationContext.disallowUndeclaredHeaders(context), null, copts, pluginOpts, nocopts, + extraSystemIncludePrefixes, enableLayeringCheck, fdoBuildStamp, VOID_INCLUDE_RESOLVER, + ImmutableList.<IncludeScannable>of(), + GUID, /*compileHeaderModules=*/false); + this.tempOutputFile = Preconditions.checkNotNull(tempOutputFile); + } + + @Override + @ThreadCompatible + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + + // First, do an normal compilation, to generate the ".d" file. The generated + // object file is built to a temporary location (tempOutputFile) and ignored + // afterwards. + LOG.info("Generating " + getDotdFile()); + CppCompileActionContext context = executor.getContext(CppCompileActionContext.class); + CppCompileActionContext.Reply reply = null; + try { + // We delegate stdout/stderr to nowhere, i.e. same as redirecting to /dev/null. + reply = context.execWithReply( + this, actionExecutionContext.withFileOutErr(new FileOutErr())); + } catch (ExecException e) { + // We ignore failures here (other than capturing the Distributor reply). + // The compilation may well fail (that's the whole point of negative compilation tests). + // We execute it here just for the side effect of generating the ".d" file. + reply = context.getReplyFromException(e, this); + if (reply == null) { + // This can only happen if the ExecException does not come from remote execution. + throw e.toActionExecutionException("", executor.getVerboseFailures(), this); + } + } + IncludeScanningContext scanningContext = executor.getContext(IncludeScanningContext.class); + updateActionInputs(executor.getExecRoot(), scanningContext.getArtifactResolver(), reply); + + // Even cc_fake_binary rules need to properly declare their dependencies... + // In fact, they need to declare their dependencies even more than cc_binary rules do. + // CcCommonConfiguredTarget passes in an empty set of declaredIncludeDirs, + // so this check below will only allow inclusion of header files that are explicitly + // listed in the "srcs" of the cc_fake_binary or in the "srcs" of a cc_library that it + // depends on. + try { + validateInclusions(actionExecutionContext.getMiddlemanExpander(), executor.getEventHandler()); + } catch (ActionExecutionException e) { + // TODO(bazel-team): (2009) make this into an error, once most of the current warnings + // are fixed. + executor.getEventHandler().handle(Event.warn( + getOwner().getLocation(), + e.getMessage() + ";\n this warning may eventually become an error")); + } + + // Generate a fake ".o" file containing the command line needed to generate + // the real object file. + LOG.info("Generating " + outputFile); + + // A cc_fake_binary rule generates fake .o files and a fake target file, + // which merely contain instructions on building the real target. We need to + // be careful to use a new set of output file names in the instructions, as + // to not overwrite the fake output files when someone tries to follow the + // instructions. As the real compilation is executed by the test from its + // runfiles directory (where writing is forbidden), we patch the command + // line to write to $TEST_TMPDIR instead. + final String outputPrefix = "$TEST_TMPDIR/"; + String argv = Joiner.on(' ').join( + Iterables.transform(getArgv(outputFile.getExecPath()), new Function<String, String>() { + @Override + public String apply(String input) { + String result = ShellEscaper.escapeString(input); + if (input.equals(outputFile.getExecPathString()) + || input.equals(getDotdFile().getSafeExecPath().getPathString())) { + result = outputPrefix + result; + } + return result; + } + })); + + // Write the command needed to build the real .o file to the fake .o file. + // Generate a command to ensure that the output directory exists; otherwise + // the compilation would fail. + try { + // Ensure that the .d file and .o file are siblings, so that the "mkdir" below works for + // both. + Preconditions.checkState(outputFile.getExecPath().getParentDirectory().equals( + getDotdFile().getSafeExecPath().getParentDirectory())); + FileSystemUtils.writeContent(outputFile.getPath(), ISO_8859_1, + outputFile.getPath().getBaseName() + ": " + + "mkdir -p " + outputPrefix + "$(dirname " + outputFile.getExecPath() + ")" + + " && " + argv + "\n"); + } catch (IOException e) { + throw new ActionExecutionException("failed to create fake compile command for rule '" + + getOwner().getLabel() + ": " + e.getMessage(), + this, false); + } + } + + @Override + protected PathFragment getInternalOutputFile() { + return tempOutputFile; + } + + @Override + public String getMnemonic() { return "FakeCppCompile"; } + + @Override + public String describeStrategy(Executor executor) { + return "fake"; + } + + @Override + public ResourceSet estimateResourceConsumptionLocal() { + return new ResourceSet(/*memoryMb=*/1, /*cpuUsage=*/0.1, /*ioUsage=*/0.0); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return executor.getContext(CppCompileActionContext.class).estimateResourceConsumption(this); + } + + @Override + protected boolean needsIncludeScanning(Executor executor) { + return false; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoStubAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoStubAction.java new file mode 100644 index 0000000..f50a1ae --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoStubAction.java
@@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.vfs.Path; + +/** + * Stub action to be used as the generating action for FDO files that are extracted from the + * FDO zip. + * + * <p>This is needed because the extraction is currently not a bona fide action, therefore, Blaze + * would complain that these files have no generating action if we did not set it to an instance of + * this class. + */ +public class FdoStubAction extends AbstractAction { + public FdoStubAction(ActionOwner owner, Artifact output) { + // TODO(bazel-team): Make extracting the zip file a honest-to-God action so that we can do away + // with this ugliness. + super(owner, ImmutableList.<Artifact>of(), ImmutableList.of(output)); + } + + @Override + public String describeStrategy(Executor executor) { + return ""; + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) { + } + + @Override + public String getMnemonic() { + return "FdoStubAction"; + } + + @Override + protected String computeKey() { + return "fdoStubAction"; + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + + @Override + public void prepare(Path execRoot) { + // The superclass would delete the output files here. We can't let that happen, since this + // action does not in fact create those files; it is only a placeholder and the actual files + // are created *before* the execution phase in FdoSupport.extractFdoZip() + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java new file mode 100644 index 0000000..911d888 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java
@@ -0,0 +1,679 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.PackageRootResolver; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.skyframe.FileValue; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.lib.vfs.ZipFileSystem; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode; +import com.google.devtools.build.skyframe.SkyFunction; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.regex.Pattern; +import java.util.zip.ZipException; + +/** + * Support class for FDO (feedback directed optimization) and LIPO (lightweight inter-procedural + * optimization). + * + * <p>There is a 1:1 relationship between {@link CppConfiguration} objects and {@code FdoSupport} + * objects. The FDO support of a build configuration can be retrieved using {@link + * CppConfiguration#getFdoSupport()}. + * + * <p>With respect to thread-safety, the {@link #prepareToBuild} method is not thread-safe, and must + * not be called concurrently with other methods on this class. + * + * <p>Here follows a quick run-down of how FDO/LIPO builds work (for non-FDO/LIPO builds, none + * of this applies): + * + * <p>{@link CppConfiguration#prepareHook} is called before the analysis phase, which calls + * {@link #prepareToBuild}, which extracts the FDO .zip (in case we work with an explicitly + * generated FDO profile file) or analyzes the .afdo.imports file next to the .afdo file (if + * AutoFDO is in effect). + * + * <p>.afdo.imports files contain one import a line. A line is two paths separated by a colon, + * with functions in the second path being referenced by functions in the first path. These are + * then put into the imports map. If we do AutoFDO, we don't handle individual .gcda files, so + * gcdaFiles will be empty. + * + * <p>Regular .fdo zip files contain .gcda files (which are added to gcdaFiles) and + * .gcda.imports files. There is one .gcda.imports file for every source file and it contains one + * path in every line, which can either be a path to a source file that contains a function + * referenced by the original source file or the .gcda file for such a referenced file. They + * both are added to the imports map. + * + * <p>If we do LIPO, we create an extra configuration that is called the "LIPO context collector", + * whose job it is to collect information that every configured target compiled with LIPO needs. + * The top-level target of this configuration is the LIPO context (always a cc_binary) and is an + * implicit dependency of every cc_* rule through their :lipo_context_collector attribute. The + * collected information is encapsulated in {@link LipoContextProvider}. + * + * <p>For each C++ compile action in the target configuration, {@link #configureCompilation} is + * called, which adds command line options and input files required for the build. There are + * three cases: + * + * <ul> + * <li>If we do AutoFDO, the .afdo file and the source files containing the functions imported + * by the original source file (as determined from the inputs map) are added. + * <li>If we do FDO, the .gcda file corresponding to the source file is added. + * <li>If we do LIPO, in addition to the .gcda file corresponding to the source file + * (like for FDO) the source files that contain the functions referenced by the source file and + * their .gcda files are added, too. + * </ul> + * + * <p>If we do LIPO, the actual C++ compilation context for LIPO compilation actions is pieced + * together from the CppCompileContext in LipoContextProvider and that of the rule being compiled. + * (see {@link CppCompilationContext#mergeForLipo}) This is so that the include files for the + * extra LIPO sources are found and is, strictly speaking, incorrect, since it also changes the + * declared include directories of the main source file, which in theory can result in the + * compilation passing even though it should fail with undeclared inclusion errors. + * + * <p>During the actual execution of the C++ compile action, the extra sources also need to be + * include scanned, which is the reason why they are {@link IncludeScannable} objects and not + * simple artifacts. We currently create these {@link IncludeScannable} objects by creating actual + * C++ compile actions in the LIPO context collector configuration which are then never executed. + * In fact, these C++ compile actions are never even registered with Skyframe. For this we + * propagate a bit from {@code BuildConfiguration.isActionsEnabled} to + * {@code CachingAnalysisEnvironment.allowRegisteringActions}, which causes actions to be silently + * discarded after configured targets are created. + */ +public class FdoSupport implements Serializable { + + /** + * Path within profile data .zip files that is considered the root of the + * profile information directory tree. + */ + private static final PathFragment ZIP_ROOT = new PathFragment("/"); + + /** + * Returns true if the give fdoFile represents an AutoFdo profile. + */ + public static final boolean isAutoFdo(String fdoFile) { + return CppFileTypes.GCC_AUTO_PROFILE.matches(fdoFile); + } + + /** + * Coverage information output directory passed to {@code --fdo_instrument}, + * or {@code null} if FDO instrumentation is disabled. + */ + private final PathFragment fdoInstrument; + + /** + * Path of the profile file passed to {@code --fdo_optimize}, or + * {@code null} if FDO optimization is disabled. The profile file + * can be a coverage ZIP or an AutoFDO feedback file. + */ + private final Path fdoProfile; + + /** + * Temporary directory to which the coverage ZIP file is extracted to + * (relative to the exec root), or {@code null} if FDO optimization is + * disabled. This is used to create artifacts for the extracted files. + * + * <p>Note that this root is intentionally not registered with the artifact + * factory. + */ + private final Root fdoRoot; + + /** + * The relative path of the FDO root to the exec root. + */ + private final PathFragment fdoRootExecPath; + + /** + * Path of FDO files under the FDO root. + */ + private final PathFragment fdoPath; + + /** + * LIPO mode passed to {@code --lipo}. This is only used if + * {@code fdoProfile != null}. + */ + private final LipoMode lipoMode; + + /** + * Flag indicating whether to use AutoFDO (as opposed to + * instrumentation-based FDO). + */ + private final boolean useAutoFdo; + + /** + * The {@code .gcda} files that have been extracted from the ZIP file, + * relative to the root of the ZIP file. + * + * <p>Set only in {@link #prepareToBuild}. + */ + private ImmutableSet<PathFragment> gcdaFiles = ImmutableSet.of(); + + /** + * Multimap from .gcda file base names to auxiliary input files. + * + * <p>The keys of the multimap are the exec root relative paths of .gcda files + * with the extension removed. The values are the lines from the accompanying + * .gcda.imports file. + * + * <p>The contents of the multimap are copied verbatim from the .gcda.imports + * files and not yet checked for validity. + * + * <p>Set only in {@link #prepareToBuild}. + */ + private ImmutableMultimap<PathFragment, Artifact> imports; + + /** + * Creates an FDO support object. + * + * @param fdoInstrument value of the --fdo_instrument option + * @param fdoProfile path to the profile file passed to --fdo_optimize option + * @param lipoMode value of the --lipo_mode option + */ + public FdoSupport(PathFragment fdoInstrument, Path fdoProfile, LipoMode lipoMode, Path execRoot) { + this.fdoInstrument = fdoInstrument; + this.fdoProfile = fdoProfile; + this.fdoRoot = (fdoProfile == null) + ? null + : Root.asDerivedRoot(execRoot, execRoot.getRelative("blaze-fdo")); + this.fdoRootExecPath = fdoProfile == null + ? null + : fdoRoot.getExecPath().getRelative(new PathFragment("_fdo").getChild( + FileSystemUtils.removeExtension(fdoProfile.getBaseName()))); + this.fdoPath = fdoProfile == null + ? null + : new PathFragment("_fdo").getChild( + FileSystemUtils.removeExtension(fdoProfile.getBaseName())); + this.lipoMode = lipoMode; + this.useAutoFdo = fdoProfile != null && isAutoFdo(fdoProfile.getBaseName()); + } + + public Root getFdoRoot() { + return fdoRoot; + } + + public void declareSkyframeDependencies(SkyFunction.Environment env, Path execRoot) { + if (fdoProfile != null) { + if (isLipoEnabled()) { + // Incrementality is not supported for LIPO builds, see FdoSupport#scannables. + // Ensure that the Skyframe value containing the configuration will not be reused to avoid + // incrementality issues. + PrecomputedValue.dependOnBuildId(env); + return; + } + + // IMPORTANT: Keep the following in sync with #prepareToBuild. + Path path; + if (useAutoFdo) { + path = fdoProfile.getParentDirectory().getRelative( + fdoProfile.getBaseName() + ".imports"); + } else { + path = fdoProfile; + } + env.getValue(FileValue.key(RootedPath.toRootedPathMaybeUnderRoot(path, + ImmutableList.of(execRoot)))); + } + } + + /** + * Prepares the FDO support for building. + * + * <p>When an {@code --fdo_optimize} compile is requested, unpacks the given + * FDO gcda zip file into a clean working directory under execRoot. + * + * @throws FdoException if the FDO ZIP contains a file of unknown type + */ + @ThreadHostile // must be called before starting the build + public void prepareToBuild(Path execRoot, PathFragment genfilesPath, + ArtifactFactory artifactDeserializer, PackageRootResolver resolver) + throws IOException, FdoException { + // The execRoot != null case is only there for testing. We cannot provide a real ZIP file in + // tests because ZipFileSystem does not work with a ZIP on an in-memory file system. + // IMPORTANT: Keep in sync with #declareSkyframeDependencies to avoid incrementality issues. + if (fdoProfile != null && execRoot != null) { + Path fdoDirPath = execRoot.getRelative(fdoRootExecPath); + + FileSystemUtils.deleteTreesBelow(fdoDirPath); + FileSystemUtils.createDirectoryAndParents(fdoDirPath); + + if (useAutoFdo) { + Path fdoImports = fdoProfile.getParentDirectory().getRelative( + fdoProfile.getBaseName() + ".imports"); + if (isLipoEnabled()) { + imports = readAutoFdoImports(artifactDeserializer, fdoImports, genfilesPath, resolver); + } + FileSystemUtils.ensureSymbolicLink( + execRoot.getRelative(getAutoProfilePath()), fdoProfile); + } else { + Path zipFilePath = new ZipFileSystem(fdoProfile).getRootDirectory(); + if (!zipFilePath.getRelative("blaze-out").isDirectory()) { + throw new ZipException("FDO zip files must be zipped directly above 'blaze-out' " + + "for the compiler to find the profile"); + } + ImmutableSet.Builder<PathFragment> gcdaFilesBuilder = ImmutableSet.builder(); + ImmutableMultimap.Builder<PathFragment, Artifact> importsBuilder = + ImmutableMultimap.builder(); + extractFdoZip(artifactDeserializer, zipFilePath, fdoDirPath, + gcdaFilesBuilder, importsBuilder, resolver); + gcdaFiles = gcdaFilesBuilder.build(); + imports = importsBuilder.build(); + } + } + } + + /** + * Recursively extracts a directory from the GCDA ZIP file into a target + * directory. + * + * <p>Imports files are not written to disk. Their content is directly added + * to an internal data structure. + * + * <p>The files are written at $EXECROOT/blaze-fdo/_fdo/(base name of profile zip), and the + * {@code _fdo} directory there is symlinked to from the exec root, so that the file are also + * available at $EXECROOT/_fdo/..., which is their exec path. We need to jump through these + * hoops because the FDO root 1. needs to be a source root, thus the exec path of its root is + * ".", 2. it must not be equal to the exec root so that the artifact factory does not get + * confused, 3. the files under it must be reachable by their exec path from the exec root. + * + * @throws IOException if any of the I/O operations failed + * @throws FdoException if the FDO ZIP contains a file of unknown type + */ + private void extractFdoZip(ArtifactFactory artifactFactory, Path sourceDir, + Path targetDir, ImmutableSet.Builder<PathFragment> gcdaFilesBuilder, + ImmutableMultimap.Builder<PathFragment, Artifact> importsBuilder, + PackageRootResolver resolver) throws IOException, FdoException { + for (Path sourceFile : sourceDir.getDirectoryEntries()) { + Path targetFile = targetDir.getRelative(sourceFile.getBaseName()); + if (sourceFile.isDirectory()) { + targetFile.createDirectory(); + extractFdoZip(artifactFactory, sourceFile, targetFile, gcdaFilesBuilder, importsBuilder, + resolver); + } else { + if (CppFileTypes.COVERAGE_DATA.matches(sourceFile)) { + FileSystemUtils.copyFile(sourceFile, targetFile); + gcdaFilesBuilder.add( + sourceFile.relativeTo(sourceFile.getFileSystem().getRootDirectory())); + } else if (CppFileTypes.COVERAGE_DATA_IMPORTS.matches(sourceFile)) { + readCoverageImports(artifactFactory, sourceFile, importsBuilder, resolver); + } else { + throw new FdoException("FDO ZIP file contained a file of unknown type: " + + sourceFile); + } + } + } + } + + /** + * Reads a .gcda.imports file and stores the imports information. + * + * @throws FdoException if an auxiliary LIPO input was not found + */ + private void readCoverageImports(ArtifactFactory artifactFactory, Path importsFile, + ImmutableMultimap.Builder<PathFragment, Artifact> importsBuilder, + PackageRootResolver resolver) throws IOException, FdoException { + PathFragment key = importsFile.asFragment().relativeTo(ZIP_ROOT); + String baseName = key.getBaseName(); + String ext = Iterables.getOnlyElement(CppFileTypes.COVERAGE_DATA_IMPORTS.getExtensions()); + key = key.replaceName(baseName.substring(0, baseName.length() - ext.length())); + + for (String line : FileSystemUtils.iterateLinesAsLatin1(importsFile)) { + if (!line.isEmpty()) { + // We can't yet fully check the validity of a line. this is done later + // when we actually parse the contained paths. + PathFragment execPath = new PathFragment(line); + if (execPath.isAbsolute()) { + throw new FdoException("Absolute paths not allowed in gcda imports file " + importsFile + + ": " + execPath); + } + Artifact artifact = artifactFactory.deserializeArtifact(new PathFragment(line), resolver); + if (artifact == null) { + throw new FdoException("Auxiliary LIPO input not found: " + line); + } + + importsBuilder.put(key, artifact); + } + } + } + + /** + * Reads a .afdo.imports file and stores the imports information. + */ + private ImmutableMultimap<PathFragment, Artifact> readAutoFdoImports( + ArtifactFactory artifactFactory, Path importsFile, PathFragment genFilePath, + PackageRootResolver resolver) + throws IOException, FdoException { + ImmutableMultimap.Builder<PathFragment, Artifact> importBuilder = ImmutableMultimap.builder(); + for (String line : FileSystemUtils.iterateLinesAsLatin1(importsFile)) { + if (!line.isEmpty()) { + PathFragment key = new PathFragment(line.substring(0, line.indexOf(':'))); + if (key.startsWith(genFilePath)) { + key = key.relativeTo(genFilePath); + } + if (key.isAbsolute()) { + throw new FdoException("Absolute paths not allowed in afdo imports file " + importsFile + + ": " + key); + } + key = FileSystemUtils.replaceSegments(key, "PROTECTED", "_protected", true); + for (String auxFile : line.substring(line.indexOf(':') + 1).split(" ")) { + if (auxFile.length() == 0) { + continue; + } + Artifact artifact = artifactFactory.deserializeArtifact(new PathFragment(auxFile), + resolver); + if (artifact == null) { + throw new FdoException("Auxiliary LIPO input not found: " + auxFile); + } + importBuilder.put(key, artifact); + } + } + } + return importBuilder.build(); + } + + /** + * Returns the imports from the .afdo.imports file of a source file. + * + * @param sourceName the source file + */ + private Collection<Artifact> getAutoFdoImports(PathFragment sourceName) { + Preconditions.checkState(isLipoEnabled()); + ImmutableCollection<Artifact> afdoImports = imports.get(sourceName); + Preconditions.checkState(afdoImports != null, + "AutoFDO import data missing for %s", sourceName); + return afdoImports; + } + + /** + * Returns the imports from the .gcda.imports file of an object file. + * + * @param objDirectory the object directory of the object file's target + * @param objectName the object file + */ + private Iterable<Artifact> getImports(PathFragment objDirectory, PathFragment objectName) { + Preconditions.checkState(isLipoEnabled()); + Preconditions.checkState(imports != null, + "Tried to look up imports of uninitialized FDOSupport"); + PathFragment key = objDirectory.getRelative(FileSystemUtils.removeExtension(objectName)); + ImmutableCollection<Artifact> importsForObject = imports.get(key); + Preconditions.checkState(importsForObject != null, "Import data missing for %s", key); + return importsForObject; + } + + /** + * Configures a compile action builder by adding command line options and + * auxiliary inputs according to the FDO configuration. This method does + * nothing If FDO is disabled. + */ + @ThreadSafe + public void configureCompilation(CppCompileActionBuilder builder, RuleContext ruleContext, + AnalysisEnvironment env, Label lipoLabel, PathFragment sourceName, final Pattern nocopts, + boolean usePic, LipoContextProvider lipoInputProvider) { + // It is a bug if this method is called with useLipo if lipo is disabled. However, it is legal + // if is is called with !useLipo, even though lipo is enabled. + Preconditions.checkArgument(lipoInputProvider == null || isLipoEnabled()); + + // FDO is disabled -> do nothing. + if ((fdoInstrument == null) && (fdoRoot == null)) { + return; + } + + List<String> fdoCopts = new ArrayList<>(); + // Instrumentation phase + if (fdoInstrument != null) { + fdoCopts.add("-fprofile-generate=" + fdoInstrument.getPathString()); + if (lipoMode != LipoMode.OFF) { + fdoCopts.add("-fripa"); + } + } + + // Optimization phase + if (fdoRoot != null) { + // Declare dependency on contents of zip file. + if (env.getSkyframeEnv().valuesMissing()) { + return; + } + Iterable<Artifact> auxiliaryInputs = getAuxiliaryInputs( + ruleContext, env, lipoLabel, sourceName, usePic, lipoInputProvider); + builder.addMandatoryInputs(auxiliaryInputs); + if (!Iterables.isEmpty(auxiliaryInputs)) { + if (useAutoFdo) { + fdoCopts.add("-fauto-profile=" + getAutoProfilePath().getPathString()); + } else { + fdoCopts.add("-fprofile-use=" + fdoRootExecPath); + } + fdoCopts.add("-fprofile-correction"); + if (lipoInputProvider != null) { + fdoCopts.add("-fripa"); + } + } + } + Iterable<String> filteredCopts = fdoCopts; + if (nocopts != null) { + // Filter fdoCopts with nocopts if they exist. + filteredCopts = Iterables.filter(fdoCopts, new Predicate<String>() { + @Override + public boolean apply(String copt) { + return !nocopts.matcher(copt).matches(); + } + }); + } + builder.addCopts(0, filteredCopts); + } + + /** + * Returns the auxiliary files that need to be added to the {@link CppCompileAction}. + */ + private Iterable<Artifact> getAuxiliaryInputs( + RuleContext ruleContext, AnalysisEnvironment env, Label lipoLabel, PathFragment sourceName, + boolean usePic, LipoContextProvider lipoContextProvider) { + // If --fdo_optimize was not specified, we don't have any additional inputs. + if (fdoProfile == null) { + return ImmutableSet.of(); + } else if (useAutoFdo) { + ImmutableSet.Builder<Artifact> auxiliaryInputs = ImmutableSet.builder(); + + Artifact artifact = env.getDerivedArtifact( + fdoPath.getRelative(getAutoProfileRootRelativePath()), fdoRoot); + env.registerAction(new FdoStubAction(ruleContext.getActionOwner(), artifact)); + auxiliaryInputs.add(artifact); + if (lipoContextProvider != null) { + auxiliaryInputs.addAll(getAutoFdoImports(sourceName)); + } + return auxiliaryInputs.build(); + } else { + ImmutableSet.Builder<Artifact> auxiliaryInputs = ImmutableSet.builder(); + + PathFragment objectName = + FileSystemUtils.replaceExtension(sourceName, usePic ? ".pic.o" : ".o"); + + auxiliaryInputs.addAll( + getGcdaArtifactsForObjectFileName(ruleContext, env, objectName, lipoLabel)); + + if (lipoContextProvider != null) { + for (Artifact importedFile : getImports( + getNonLipoObjDir(ruleContext, lipoLabel), objectName)) { + if (CppFileTypes.COVERAGE_DATA.matches(importedFile.getFilename())) { + Artifact gcdaArtifact = getGcdaArtifactsForGcdaPath( + ruleContext, env, importedFile.getExecPath()); + if (gcdaArtifact == null) { + ruleContext.ruleError(String.format( + ".gcda file %s is not in the FDO zip (referenced by source file %s)", + importedFile.getExecPath(), sourceName)); + } else { + auxiliaryInputs.add(gcdaArtifact); + } + } else { + auxiliaryInputs.add(importedFile); + } + } + } + + return auxiliaryInputs.build(); + } + } + + /** + * Returns the .gcda file artifacts for a .gcda path from the .gcda.imports file or null if the + * referenced .gcda file is not in the FDO zip. + */ + private Artifact getGcdaArtifactsForGcdaPath(RuleContext ruleContext, + AnalysisEnvironment env, PathFragment gcdaPath) { + if (!gcdaFiles.contains(gcdaPath)) { + return null; + } + + Artifact artifact = env.getDerivedArtifact(fdoPath.getRelative(gcdaPath), fdoRoot); + env.registerAction(new FdoStubAction(ruleContext.getActionOwner(), artifact)); + return artifact; + } + + private PathFragment getNonLipoObjDir(RuleContext ruleContext, Label label) { + return ruleContext.getConfiguration().getBinFragment() + .getRelative(CppHelper.getObjDirectory(label)); + } + + /** + * Returns a list of .gcda file artifacts for an object file path. + * + * <p>The resulting set is either empty (because no .gcda file exists for the + * given object file) or contains one or two artifacts (the file itself and a + * symlink to it). + */ + private ImmutableList<Artifact> getGcdaArtifactsForObjectFileName(RuleContext ruleContext, + AnalysisEnvironment env, PathFragment objectFileName, Label lipoLabel) { + // We put the .gcda files relative to the location of the .o file in the instrumentation run. + String gcdaExt = Iterables.getOnlyElement(CppFileTypes.COVERAGE_DATA.getExtensions()); + PathFragment baseName = FileSystemUtils.replaceExtension(objectFileName, gcdaExt); + PathFragment gcdaFile = getNonLipoObjDir(ruleContext, lipoLabel).getRelative(baseName); + + if (!gcdaFiles.contains(gcdaFile)) { + // If the object is a .pic.o file and .pic.gcda is not found, we should try finding .gcda too + String picoExt = Iterables.getOnlyElement(CppFileTypes.PIC_OBJECT_FILE.getExtensions()); + baseName = FileSystemUtils.replaceExtension(objectFileName, gcdaExt, picoExt); + if (baseName == null) { + // Object file is not .pic.o + return ImmutableList.of(); + } + gcdaFile = getNonLipoObjDir(ruleContext, lipoLabel).getRelative(baseName); + if (!gcdaFiles.contains(gcdaFile)) { + // .gcda file not found + return ImmutableList.of(); + } + } + + final Artifact artifact = env.getDerivedArtifact(fdoPath.getRelative(gcdaFile), fdoRoot); + env.registerAction(new FdoStubAction(ruleContext.getActionOwner(), artifact)); + + return ImmutableList.of(artifact); + } + + + private PathFragment getAutoProfilePath() { + return fdoRootExecPath.getRelative(getAutoProfileRootRelativePath()); + } + + private PathFragment getAutoProfileRootRelativePath() { + return new PathFragment(fdoProfile.getBaseName()); + } + + /** + * Returns whether LIPO is enabled. + */ + @ThreadSafe + public boolean isLipoEnabled() { + return fdoProfile != null && lipoMode != LipoMode.OFF; + } + + /** + * Returns whether AutoFDO is enabled. + */ + @ThreadSafe + public boolean isAutoFdoEnabled() { + return useAutoFdo; + } + + /** + * Returns an immutable list of command line arguments to add to the linker + * command line. If FDO is disabled, and empty list is returned. + */ + @ThreadSafe + public ImmutableList<String> getLinkOptions() { + return fdoInstrument != null + ? ImmutableList.of("-fprofile-generate=" + fdoInstrument.getPathString()) + : ImmutableList.<String>of(); + } + + /** + * Returns the path of the FDO output tree (relative to the execution root) + * containing the .gcda profile files, or null if FDO is not enabled. + */ + @VisibleForTesting + public PathFragment getFdoOptimizeDir() { + return fdoRootExecPath; + } + + /** + * Returns the path of the FDO zip containing the .gcda profile files, or null + * if FDO is not enabled. + */ + @VisibleForTesting + public Path getFdoOptimizeProfile() { + return fdoProfile; + } + + /** + * Returns the path fragment of the instrumentation output dir for gcc when + * FDO is enabled, or null if FDO is not enabled. + */ + @ThreadSafe + public PathFragment getFdoInstrument() { + return fdoInstrument; + } + + @VisibleForTesting + public void setGcdaFilesForTesting(ImmutableSet<PathFragment> gcdaFiles) { + this.gcdaFiles = gcdaFiles; + } + + /** + * An exception indicating an issue with FDO coverage files. + */ + public static final class FdoException extends Exception { + FdoException(String message) { + super(message); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/HeaderTargetModuleMapProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/HeaderTargetModuleMapProvider.java new file mode 100644 index 0000000..17e2e5c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/HeaderTargetModuleMapProvider.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.List; + +/** + * A provider for cc_public_library rules to be able to convey the information about the + * header target's module map references to the public library target. + */ +@Immutable +public final class HeaderTargetModuleMapProvider implements TransitiveInfoProvider { + + private final ImmutableList<CppModuleMap> cppModuleMaps; + + public HeaderTargetModuleMapProvider(Iterable<CppModuleMap> cppModuleMaps) { + this.cppModuleMaps = ImmutableList.copyOf(cppModuleMaps); + } + + /** + * Returns the module maps referenced by cc_public_library's headers target. + */ + public List<CppModuleMap> getCppModuleMaps() { + return cppModuleMaps; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/ImplementedCcPublicLibrariesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/ImplementedCcPublicLibrariesProvider.java new file mode 100644 index 0000000..4f2a585 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/ImplementedCcPublicLibrariesProvider.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; + +/** + * A provider for cc_library rules to be able to convey the information about which + * cc_public_library rules they implement to dependent targets. + */ +@Immutable +public final class ImplementedCcPublicLibrariesProvider implements TransitiveInfoProvider { + + private final ImmutableList<Label> implementedCcPublicLibraries; + + public ImplementedCcPublicLibrariesProvider(ImmutableList<Label> implementedCcPublicLibraries) { + this.implementedCcPublicLibraries = implementedCcPublicLibraries; + } + + /** + * Returns the labels for the "$headers" target that are implemented by the target which + * implements this interface. + */ + public ImmutableList<Label> getImplementedCcPublicLibraries() { + return implementedCcPublicLibraries; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeParser.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeParser.java new file mode 100644 index 0000000..0b60b45 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeParser.java
@@ -0,0 +1,711 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import com.google.common.io.CharStreams; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.rules.cpp.IncludeParser.Inclusion.Kind; +import com.google.devtools.build.lib.rules.cpp.RemoteIncludeExtractor.RemoteParseData; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import javax.annotation.Nullable; + +/** + * Scans a source file and extracts the literal inclusions it specifies. Does not store results -- + * repeated requests to the same file will result in repeated scans. Clients should implement a + * caching layer in order to avoid unnecessary disk access when requesting an already scanned file. + */ +public class IncludeParser implements SkyValue { + private static final Logger LOG = Logger.getLogger(IncludeParser.class.getName()); + private static final boolean LOG_FINE = LOG.isLoggable(Level.FINE); + private static final boolean LOG_FINER = LOG.isLoggable(Level.FINER); + + /** + * Immutable object representation of the four columns making up a single Rule + * in a Hints set. See {@link Hints} for more details. + */ + private static class Rule { + private enum Type { PATH, FILE, INCLUDE_QUOTE, INCLUDE_ANGLE; } + final Type type; + final Pattern pattern; + final String findRoot; + final Pattern findFilter; + + private Rule(String type, String pattern, String findRoot, Pattern findFilter) { + this.type = Type.valueOf(type.trim().toUpperCase()); + this.pattern = Pattern.compile("^" + pattern + "$"); + this.findRoot = findRoot; + this.findFilter = findFilter; + } + + /** + * @throws PatternSyntaxException, IllegalArgumentException if bad values + * are provided + */ + public Rule(String type, String pattern, String findRoot, String findFilter) { + this(type, pattern, findRoot.replace("\\", "$"), Pattern.compile(findFilter)); + Preconditions.checkArgument((this.type == Type.PATH) || (this.type == Type.FILE)); + } + + public Rule(String type, String pattern, String findRoot) { + this(type, pattern, findRoot, (Pattern) null); + Preconditions.checkArgument((this.type == Type.INCLUDE_QUOTE) + || (this.type == Type.INCLUDE_ANGLE)); + } + + @Override public String toString() { + return "" + type + " " + pattern + " " + findRoot + " " + findFilter; + } + } + + /** + * This class is a representation of the INCLUDE_HINTS file maintained and + * delivered with the remote client. The hints file contains regexp-based rules + * to help this simple include scanner cope with computed includes, which + * would otherwise require a full preprocessor with symbol support. Instead of + * actually processing symbols to evaluate the computed includes, we instead + * apply rules to gather inclusions for matching paths. + * <p> + * The hints file is read, line by line, into a list of rules each of which + * encapsulates a line of four columns. Each non-blank, non-comment line has + * the format: + * + * <pre> + * "file"|"path" match-pattern find-root find-filter + * </pre> + * + * <p> + * The first column specifies whether the line is a rule based on matching + * source <em>files</em> (passed directly to gcc as inputs, or transitively + * #included by other inputs) or include <em>paths</em> (passed to gcc as + * -I, -iquote, or -isystem flags). + * <p> + * The second column is a regexp for files or paths. Whenever a compiler + * argument of the specified type matches that regexp, the rule is taken. (All + * matching rules for every path and file on a compiler command line are + * followed, and the results are combined.) + * <p> + * The third column is a point in the local filesystem from which to extract a + * recursive listing. (This follows symlinks) Backrefs may be used to refer to + * the regexp or its capturing groups. (This is mostly necessary because + * --package_path can cause input paths to carry arbitrary prefixes.) + * <p> + * The fourth column is a regexp applied to each file found by the recursive + * listing. All matching files are treated as dependencies. + */ + public static class Hints implements SkyValue { + + private static final Pattern WS_PAT = Pattern.compile("\\s+"); + + private final Path workingDir; + private final List<Rule> rules = new ArrayList<>(); + private final ArtifactFactory artifactFactory; + + private final LoadingCache<Artifact, Collection<Artifact>> fileLevelHintsCache = + CacheBuilder.newBuilder().build( + new CacheLoader<Artifact, Collection<Artifact>>() { + @Override + public Collection<Artifact> load(Artifact path) { + return getHintedInclusions(Rule.Type.FILE, path.getPath(), path.getRoot()); + } + }); + + private final LoadingCache<Path, Collection<Artifact>> pathLevelHintsCache = + CacheBuilder.newBuilder().build( + new CacheLoader<Path, Collection<Artifact>>() { + @Override + public Collection<Artifact> load(Path path) { + return getHintedInclusions(Rule.Type.PATH, path, null); + } + }); + + /** + * Constructs a hint set for a given working/exec directory and INCLUDE_HINTS file to read. + * + * @param workingDir the working/exec directory that processed paths are relative to + * @param hintsFile the hints file to read + * @throws IOException if the hints file can't be read or parsed + */ + public Hints(Path workingDir, Path hintsFile, ArtifactFactory artifactFactory) + throws IOException { + this.workingDir = workingDir; + this.artifactFactory = artifactFactory; + try (InputStream is = hintsFile.getInputStream()) { + for (String line : CharStreams.readLines(new InputStreamReader(is, "UTF-8"))) { + line = line.trim(); + if (line.length() == 0 || line.startsWith("#")) { + continue; + } + String[] tokens = WS_PAT.split(line); + try { + if (tokens.length == 3) { + rules.add(new Rule(tokens[0], tokens[1], tokens[2])); + } else if (tokens.length == 4) { + rules.add(new Rule(tokens[0], tokens[1], tokens[2], tokens[3])); + } else { + throw new IOException("Malformed hint line: " + line); + } + } catch (PatternSyntaxException e) { + throw new IOException("Malformed hint regex on: " + line + "\n " + e.getMessage()); + } catch (IllegalArgumentException e) { + throw new IOException("Invalid type on: " + line + "\n " + e.getMessage()); + } + } + } + } + + /** + * Returns the "file" type hinted inclusions for a given path, caching results by path. + */ + public Collection<Artifact> getFileLevelHintedInclusions(Artifact path) { + return fileLevelHintsCache.getUnchecked(path); + } + + public Collection<Artifact> getPathLevelHintedInclusions(Path path) { + return pathLevelHintsCache.getUnchecked(path); + } + + /** + * Performs the work of matching a given file/path of a specified file/path type against the + * hints and returns the expanded paths. + */ + private Collection<Artifact> getHintedInclusions(Rule.Type type, Path path, + @Nullable Root sourceRoot) { + String pathString = path.getPathString(); + // Delay creation until we know we need one. Use a TreeSet to make sure that the results are + // sorted with a stable order and unique. + Set<Path> hints = null; + for (final Rule rule : rules) { + if (type != rule.type) { + continue; + } + Matcher m = rule.pattern.matcher(pathString); + if (!m.matches()) { + continue; + } + if (hints == null) { hints = Sets.newTreeSet(); } + Path root = workingDir.getRelative(m.replaceFirst(rule.findRoot)); + if (LOG_FINE) { + LOG.fine("hint for " + rule.type + " " + pathString + " root: " + root); + } + try { + // The assumption is made here that all files specified by this hint are under the same + // package path as the original file -- this filesystem tree traversal is completely + // ignorant of package paths. This could be violated if there were a hint that resolved to + // foo/**/*.h, there was a package foo/bar, and the packages foo and foo/bar were in + // different package paths. In that case, this traversal would fail to pick up + // foo/bar/**/*.h. No examples of this currently exist in the INCLUDE_HINTS + // file. + FileSystemUtils.traverseTree(hints, root, new Predicate<Path>() { + @Override + public boolean apply(Path p) { + boolean take = p.isFile() && rule.findFilter.matcher(p.getPathString()).matches(); + if (LOG_FINER && take) { + LOG.finer("hinted include: " + p); + } + return take; + } + }); + } catch (IOException e) { + LOG.warning("Error in hint expansion: " + e); + } + } + if (hints != null && !hints.isEmpty()) { + // Transform paths into source artifacts (all hints must be to source artifacts). + List<Artifact> result = new ArrayList<>(hints.size()); + for (Path hint : hints) { + if (hint.startsWith(workingDir)) { + // Paths that are under the execRoot can be resolved as source artifacts as usual. All + // include directories are specified relative to the execRoot, and so fall here. + result.add(Preconditions.checkNotNull( + artifactFactory.resolveSourceArtifact(hint.relativeTo(workingDir)), hint)); + } else { + // The file passed in might not have been under the execRoot, for instance + // <workspace>/foo/foo.cc. + Preconditions.checkNotNull(sourceRoot, "%s %s", path, hint); + Path sourcePath = sourceRoot.getPath(); + Preconditions.checkState(hint.startsWith(sourcePath), + "%s %s %s", hint, path, sourceRoot); + result.add(Preconditions.checkNotNull( + artifactFactory.getSourceArtifact(hint.relativeTo(sourcePath), sourceRoot))); + } + } + return result; + } else { + return ImmutableList.of(); + } + } + + private Collection<Inclusion> getHintedInclusions(Artifact path) { + String pathString = path.getPath().getPathString(); + // Delay creation until we know we need one. Use a LinkedHashSet to make sure that the results + // are sorted with a stable order and unique. + Set<Inclusion> hints = null; + for (final Rule rule : rules) { + if ((rule.type != Rule.Type.INCLUDE_ANGLE) && (rule.type != Rule.Type.INCLUDE_QUOTE)) { + continue; + } + Matcher m = rule.pattern.matcher(pathString); + if (!m.matches()) { + continue; + } + if (hints == null) { hints = Sets.newLinkedHashSet(); } + Inclusion inclusion = new Inclusion(rule.findRoot, rule.type == Rule.Type.INCLUDE_QUOTE + ? Kind.QUOTE : Kind.ANGLE); + hints.add(inclusion); + if (LOG_FINE) { + LOG.fine("hint for " + rule.type + " " + pathString + " root: " + inclusion); + } + } + if (hints != null && !hints.isEmpty()) { + return ImmutableList.copyOf(hints); + } else { + return ImmutableList.of(); + } + } + } + + public Hints getHints() { + return hints; + } + + /** + * An immutable inclusion tuple. This models an {@code #include} or {@code + * #include_next} line in a file without the context how this file got + * included. + */ + public static class Inclusion { + /** The format of the #include in the source file -- quoted, angle bracket, etc. */ + public enum Kind { + /** Quote includes: {@code #include "name"}. */ + QUOTE, + + /** Angle bracket includes: {@code #include <name>}. */ + ANGLE, + + /** Quote next includes: {@code #include_next "name"}. */ + NEXT_QUOTE, + + /** Angle next includes: {@code #include_next <name>}. */ + NEXT_ANGLE, + + /** Computed or other unhandlable includes: {@code #include HEADER}. */ + OTHER; + + /** + * Returns true if this is an {@code #include_next} inclusion, + */ + public boolean isNext() { + return this == NEXT_ANGLE || this == NEXT_QUOTE; + } + } + + /** The kind of inclusion. */ + public final Kind kind; + /** The relative path of the inclusion. */ + public final PathFragment pathFragment; + + public Inclusion(String includeTarget, Kind kind) { + this.kind = kind; + this.pathFragment = new PathFragment(includeTarget); + } + + public Inclusion(PathFragment pathFragment, Kind kind) { + this.kind = kind; + this.pathFragment = Preconditions.checkNotNull(pathFragment); + } + + public String getPathString() { + return pathFragment.getPathString(); + } + + @Override + public String toString() { + return kind.toString() + ":" + pathFragment.getPathString(); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof Inclusion)) { + return false; + } + Inclusion that = (Inclusion) o; + return kind == that.kind && pathFragment.equals(that.pathFragment); + } + + @Override + public int hashCode() { + return pathFragment.hashCode() * 37 + kind.hashCode(); + } + } + + /** + * The externally-scoped immutable hints helper that is shared by all scanners. + */ + private final Hints hints; + + /** + * A scanner that extracts includes from an individual files remotely, used when scanning files + * generated remotely. + */ + private final Supplier<? extends RemoteIncludeExtractor> remoteExtractor; + + /** + * Constructs a new FileParser. + * @param remoteExtractor a processor that extracts includes from an individual file remotely. + * @param hints regexps for converting computed includes into simple strings + */ + public IncludeParser(@Nullable RemoteIncludeExtractor remoteExtractor, Hints hints) { + this.hints = hints; + this.remoteExtractor = Suppliers.ofInstance(remoteExtractor); + } + + /** + * Constructs a new FileParser. + * @param remoteExtractorSupplier a supplier of a processor that extracts includes from an + * individual file remotely. + * @param hints regexps for converting computed includes into simple strings + */ + public IncludeParser(Supplier<? extends RemoteIncludeExtractor> remoteExtractorSupplier, + Hints hints) { + this.hints = hints; + this.remoteExtractor = remoteExtractorSupplier; + } + + /** + * Skips whitespace, \+NL pairs, and block-style / * * / comments. Assumes + * line comments are handled outside. Does not handle digraphs, trigraphs or + * decahexagraphs. + * + * @param chars characters to scan + * @param pos the starting position + * @return the resulting position after skipping whitespace and comments. + */ + protected static int skipWhitespace(char[] chars, int pos, int end) { + while (pos < end) { + if (Character.isWhitespace(chars[pos])) { + pos++; + } else if (chars[pos] == '\\' && pos + 1 < end && chars[pos + 1] == '\n') { + pos++; + } else if (chars[pos] == '/' && pos + 1 < end && chars[pos + 1] == '*') { + pos += 2; + while (pos < end - 1) { + if (chars[pos++] == '*') { + if (chars[pos] == '/') { + pos++; + break; // proper comment end + } + } + } + } else { // not whitespace + return pos; + } + } + return pos; // pos == len, meaning we fell off the end. + } + + /** + * Checks for and skips a given token. + * + * @param chars characters to scan + * @param pos the starting position + * @param expected the expected token + * @return the resulting position if found, otherwise -1 + */ + protected static int expect(char[] chars, int pos, int end, String expected) { + int si = 0; + int expectedLen = expected.length(); + while (pos < end) { + if (si == expectedLen) { + return pos; + } + if (chars[pos++] != expected.charAt(si++)) { + return -1; + } + } + return -1; + } + + /** + * Finds the index of a given character token from a starting pos. + * + * @param chars characters to scan + * @param pos the starting position + * @param echar the character to find + * @return the resulting position of echar if found, otherwise -1 + */ + private static int indexOf(char[] chars, int pos, int end, char echar) { + while (pos < end) { + if (chars[pos] == echar) { + return pos; + } + pos++; + } + return -1; + } + + private static final Pattern BS_NL_PAT = Pattern.compile("\\\\" + "\n"); + + // Keep this in sync with the auxiliary binary's scanning output format. + private static final ImmutableMap<Character, Kind> KIND_MAP = ImmutableMap.of( + '"', Kind.QUOTE, + '<', Kind.ANGLE, + 'q', Kind.NEXT_QUOTE, + 'a', Kind.NEXT_ANGLE); + + /** + * Processes the output generated by an auxiliary include-scanning binary. Closes the stream upon + * completion. + * + * <p>If a source file has the following include statements: + * <pre> + * #include <string> + * #include "directory/header.h" + * </pre> + * + * <p>Then the output file has the following contents: + * <pre> + * "directory/header.h + * <string + * </pre> + * <p>Each line of the output is translated into an Inclusion object. + */ + public static List<Inclusion> processIncludes(Object streamName, InputStream is) + throws IOException { + List<Inclusion> inclusions = new ArrayList<>(); + InputStreamReader reader = new InputStreamReader(is, ISO_8859_1); + try { + for (String line : CharStreams.readLines(reader)) { + char qchar = line.charAt(0); + String name = line.substring(1); + Inclusion.Kind kind = KIND_MAP.get(qchar); + if (kind == null) { + throw new IOException("Illegal inclusion kind '" + qchar + "'"); + } + inclusions.add(new Inclusion(name, kind)); + } + } catch (IOException e) { + throw new IOException("Error reading include file " + streamName + ": " + e.getMessage()); + } finally { + reader.close(); + } + return inclusions; + } + + @VisibleForTesting + Inclusion extractInclusion(String line) { + return extractInclusion(line.toCharArray(), 0, line.length()); + } + + /** + * Extracts a new, unresolved an Inclusion from a line of source. + * + * @param chars the char array containing the line chars to parse + * @param lineBegin the position of the first character in the line + * @param lineEnd the position of the character after the last + * @return the inclusion object if possible, null if none + */ + private Inclusion extractInclusion(char[] chars, int lineBegin, int lineEnd) { + // expect WS#WS(include|include_next)WS("name"|<name>|junk) + int pos = expectIncludeKeyword(chars, lineBegin, lineEnd); + if (pos == -1 || pos == lineEnd) { + return null; + } + boolean isNext = false; + int npos = expect(chars, pos, lineEnd, "_next"); + if (npos >= 0) { + isNext = true; + pos = npos; + } + if ((pos = skipWhitespace(chars, pos, lineEnd)) == lineEnd) { + return null; + } + if (chars[pos] == '"' || chars[pos] == '<') { + char qchar = chars[pos++]; + int spos = pos; + pos = indexOf(chars, pos + 1, lineEnd, qchar == '<' ? '>' : '"'); + if (pos < 0) { + return null; + } + if (chars[spos] == '/') { + return null; // disallow absolute paths + } + String name = new String(chars, spos, pos - spos); + if (name.contains("\n")) { // strip any \+NL pairs within name + name = BS_NL_PAT.matcher(name).replaceAll(""); + } + if (isNext) { + return new Inclusion(name, qchar == '"' ? Kind.NEXT_QUOTE : Kind.NEXT_ANGLE); + } else { + return new Inclusion(name, qchar == '"' ? Kind.QUOTE : Kind.ANGLE); + } + } else { + return createOtherInclusion(new String(chars, pos, lineEnd - pos)); + } + } + + /** + * Extracts all inclusions from characters of a file. + * + * @param chars the file contents to parse & extract inclusions from + * @return a new set of inclusions, normalized to the cache + */ + @VisibleForTesting + List<Inclusion> extractInclusions(char[] chars) { + List<Inclusion> inclusions = new ArrayList<>(); + int lineBegin = 0; // the first char of each line + int end = chars.length; // the file end + while (lineBegin < end) { + int lineEnd = lineBegin; // the char after the last non-\n in each line + // skip to the next \n or after end of buffer, ignoring continuations + while (lineEnd < end) { + if (chars[lineEnd] == '\n') { + break; + } else if (chars[lineEnd] == '\\') { + lineEnd++; + if (chars[lineEnd] == '\n') { + lineEnd++; + } + } else { + lineEnd++; + } + } + + // TODO(bazel-team) handle multiline block comments /* */ for the cases: + // /* blah blah blah + // lalala */ #include "foo.h" + // and: + // /* blah + // #include "foo.h" + // */ + + // extract the inclusion, and save only the kind we care about. + Inclusion inclusion = extractInclusion(chars, lineBegin, lineEnd); + if (inclusion != null) { + if (isValidInclusionKind(inclusion.kind)) { + inclusions.add(inclusion); + } else { + //System.err.println("Funky include " + inclusion + " in " + file); + } + } + lineBegin = lineEnd + 1; // next line starts after the previous line + } + return inclusions; + } + + /** + * Extracts all inclusions from a given source file. + * + * @param file the file to parse & extract inclusions from + * @param greppedFile if non-null, this file has the already-grepped include lines of file. + * @param actionExecutionContext Services in the scope of the action, like the stream to which + * scanning messages are printed + * @return a new set of inclusions, normalized to the cache + */ + public Collection<Inclusion> extractInclusions(Artifact file, @Nullable Path greppedFile, + ActionExecutionContext actionExecutionContext) + throws IOException, InterruptedException { + Collection<Inclusion> inclusions; + if (greppedFile != null) { + inclusions = processIncludes(greppedFile, greppedFile.getInputStream()); + } else { + RemoteParseData remoteParseData = remoteExtractor.get() == null + ? null + : remoteExtractor.get().shouldParseRemotely(file.getPath()); + if (remoteParseData != null && remoteParseData.shouldParseRemotely()) { + inclusions = + remoteExtractor.get().extractInclusions(file, actionExecutionContext, + remoteParseData); + } else { + inclusions = extractInclusions(FileSystemUtils.readContentAsLatin1(file.getPath())); + } + } + if (hints != null) { + inclusions.addAll(hints.getHintedInclusions(file)); + } + return ImmutableList.copyOf(inclusions); + } + + /** + * Parses include keyword in the provided char array and returns position + * immediately after include keyword or -1 if keyword was not found. Can be + * overridden by subclasses. + */ + protected int expectIncludeKeyword(char[] chars, int position, int end) { + int pos = expect(chars, skipWhitespace(chars, position, end), end, "#"); + if (pos > 0) { + int npos = skipWhitespace(chars, pos, end); + if ((pos = expect(chars, npos, end, "include")) > 0) { + return pos; + } else if ((pos = expect(chars, npos, end, "import")) > 0) { + if (expect(chars, pos, end, "_") == -1) { // Needed to avoid #import_next. + return pos; + } + } + } + return -1; + } + + /** + * Returns true if we interested in the given inclusion kind. Can be + * overridden by the subclass. + */ + protected boolean isValidInclusionKind(Kind kind) { + return kind != Kind.OTHER; + } + + /** + * Returns inclusion object for non-standard inclusion cases or null if + * inclusion should be ignored. + */ + protected Inclusion createOtherInclusion(String inclusionContent) { + return new Inclusion(inclusionContent, Kind.OTHER); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeProblems.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeProblems.java new file mode 100644 index 0000000..f6be877 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeProblems.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.Artifact; + +/** + * Accumulator for problems encountered while reading or validating inclusion + * results. + */ +class IncludeProblems { + + private StringBuilder message; // null when no problems + + void add(String included) { + if (message == null) { message = new StringBuilder(); } + message.append("\n '" + included + "'"); + } + + boolean hasProblems() { return message != null; } + + String getMessage(Action action, Artifact sourceFile) { + if (message != null) { + return "undeclared inclusion(s) in rule '" + action.getOwner().getLabel() + "':\n" + + "this rule is missing dependency declarations for the following files " + + "included by '" + sourceFile.prettyPrint() + "':" + + message; + } + return null; + } + + void assertProblemFree(Action action, Artifact sourceFile) throws ActionExecutionException { + if (hasProblems()) { + throw new ActionExecutionException(getMessage(action, sourceFile), action, false); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScannable.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScannable.java new file mode 100644 index 0000000..9c70090 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScannable.java
@@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * To be implemented by actions (such as C++ compilation steps) whose inputs + * can be scanned to discover other implicit inputs (such as C++ header files). + * + * <p>This is useful for remote execution strategies to be able to compute the + * complete set of files that must be distributed in order to execute such an action. + */ +public interface IncludeScannable { + + /** + * Returns the built-in list of system include paths for the toolchain compiler. All paths in this + * list should be relative to the exec directory. They may be absolute if they are also installed + * on the remote build nodes or for local compilation. + */ + List<PathFragment> getBuiltInIncludeDirectories(); + + /** + * Returns an immutable list of "-iquote" include paths that should be used by + * the IncludeScanner for this action. GCC searches these paths first, but + * only for {@code #include "foo"}, not for {@code #include <foo>}. + */ + List<PathFragment> getQuoteIncludeDirs(); + + /** + * Returns an immutable list of "-I" include paths that should be used by the + * IncludeScanner for this action. GCC searches these paths ahead of the + * system include paths, but after "-iquote" include paths. + */ + List<PathFragment> getIncludeDirs(); + + /** + * Returns an immutable list of "-isystem" include paths that should be used + * by the IncludeScanner for this action. GCC searches these paths ahead of + * the built-in system include paths, but after all other paths. "-isystem" + * paths are treated the same as normal system directories. + */ + List<PathFragment> getSystemIncludeDirs(); + + /** + * Returns an immutable list of "-include" inclusions specified explicitly on + * the command line of this action. GCC will imagine that these files have + * been quote-included at the beginning of each source file. + */ + List<String> getCmdlineIncludes(); + + /** + * Returns an immutable list of sources that the IncludeScanner should scan + * for this action. + */ + Collection<Artifact> getIncludeScannerSources(); + + /** + * Returns additional scannables that need also be scanned when scanning this + * scannable. May be empty but not null. This is not evaluated recursively. + */ + Iterable<IncludeScannable> getAuxiliaryScannables(); + + /** + * Returns a map of generated files:files grepped for headers which may be reached during include + * scanning. Generated files which are reached, but not in the key set, must be ignored. + * + * <p>If grepping of output files is not enabled via --extract_generated_inclusions, keys + * should just map to null. + */ + Map<Artifact, Path> getLegalGeneratedScannerFileMap(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanner.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanner.java new file mode 100644 index 0000000..9c00efd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanner.java
@@ -0,0 +1,177 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.UserExecException; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Scans source files to determine the bounding set of transitively referenced include files. + * + * <p>Note that include scanning is performance-critical code. + */ +public interface IncludeScanner { + /** + * Processes a source file and a list of includes extracted from command line + * flags. Adds all found files to the provided set {@code includes}. This + * method takes into account the path- and file-level hints that are part of + * this include scanner. + */ + public void process(Artifact source, Map<Artifact, Path> legalOutputPaths, + List<String> cmdlineIncludes, Set<Artifact> includes, + ActionExecutionContext actionExecutionContext) + throws IOException, ExecException, InterruptedException; + + /** Supplies IncludeScanners upon request. */ + interface IncludeScannerSupplier { + /** Returns the possibly shared scanner to be used for a given pair of include paths. */ + IncludeScanner scannerFor(List<Path> quoteIncludePaths, List<Path> includePaths); + } + + /** + * Helper class that exists just to provide a static method that prepares the arguments with which + * to call an IncludeScanner. + */ + class IncludeScanningPreparer { + private IncludeScanningPreparer() {} + + /** + * Returns the files transitively included by the source files of the given IncludeScannable. + * + * @param action IncludeScannable whose sources' transitive includes will be returned. + * @param includeScannerSupplier supplies IncludeScanners to actually do the transitive + * scanning (and caching results) for a given source file. + * @param actionExecutionContext the context for {@code action}. + * @param profilerTaskName what the {@link Profiler} should record this call for. + */ + public static Collection<Artifact> scanForIncludedInputs(IncludeScannable action, + IncludeScannerSupplier includeScannerSupplier, + ActionExecutionContext actionExecutionContext, + String profilerTaskName) + throws ExecException, InterruptedException, ActionExecutionException { + + Set<Artifact> includes = Sets.newConcurrentHashSet(); + + Executor executor = actionExecutionContext.getExecutor(); + Path execRoot = executor.getExecRoot(); + + final List<Path> absoluteBuiltInIncludeDirs = new ArrayList<>(); + + Profiler profiler = Profiler.instance(); + try { + profiler.startTask(ProfilerTask.SCANNER, profilerTaskName); + + // We need to scan the action itself, but also the auxiliary scannables + // (for LIPO). There is no need to call getAuxiliaryScannables + // recursively. + for (IncludeScannable scannable : + Iterables.concat(ImmutableList.of(action), action.getAuxiliaryScannables())) { + + Map<Artifact, Path> legalOutputPaths = scannable.getLegalGeneratedScannerFileMap(); + List<PathFragment> includeDirs = new ArrayList<>(scannable.getIncludeDirs()); + List<PathFragment> quoteIncludeDirs = scannable.getQuoteIncludeDirs(); + List<String> cmdlineIncludes = scannable.getCmdlineIncludes(); + + for (PathFragment pathFragment : scannable.getSystemIncludeDirs()) { + includeDirs.add(pathFragment); + } + + // Add the system include paths to the list of include paths. + for (PathFragment pathFragment : action.getBuiltInIncludeDirectories()) { + if (pathFragment.isAbsolute()) { + absoluteBuiltInIncludeDirs.add(execRoot.getRelative(pathFragment)); + } + includeDirs.add(pathFragment); + } + + IncludeScanner scanner = includeScannerSupplier.scannerFor( + relativeTo(execRoot, quoteIncludeDirs), + relativeTo(execRoot, includeDirs)); + + for (Artifact source : scannable.getIncludeScannerSources()) { + // Add all include scanning entry points to the inputs; this is necessary + // when we have more than one source to scan from, for example when building + // C++ modules. + // In that case we have one of two cases: + // 1. We compile a header module - there, the .cppmap file is the main source file + // (which we do not include-scan, as that would require an extra parser), and + // thus already in the input; all headers in the .cppmap file are our entry points + // for include scanning, but are not yet in the inputs - they get added here. + // 2. We compile an object file that uses a header module; currently using a header + // module requires all headers it can reference to be available for the compilation. + // The header module can reference headers that are not in the transitive include + // closure of the current translation unit. Therefore, {@code CppCompileAction} + // adds all headers specified transitively for compiled header modules as include + // scanning entry points, and we need to add the entry points to the inputs here. + includes.add(source); + scanner.process(source, legalOutputPaths, cmdlineIncludes, includes, + actionExecutionContext); + } + } + } catch (IOException e) { + throw new EnvironmentalExecException(e.getMessage()); + } finally { + profiler.completeTask(ProfilerTask.SCANNER); + } + + // Collect inputs and output + List<Artifact> inputs = new ArrayList<>(); + IncludeProblems includeProblems = new IncludeProblems(); + for (Artifact included : includes) { + if (FileSystemUtils.startsWithAny(included.getPath(), absoluteBuiltInIncludeDirs)) { + // Skip include files found in absolute include directories. This currently only applies + // to grte. + continue; + } + if (included.getRoot().getPath().getParentDirectory() == null) { + throw new UserExecException( + "illegal absolute path to include file: " + included.getPath()); + } + inputs.add(included); + } + return inputs; + } + + private static List<Path> relativeTo( + Path path, Collection<PathFragment> fragments) { + List<Path> result = Lists.newArrayListWithCapacity(fragments.size()); + for (PathFragment fragment : fragments) { + result.add(path.getRelative(fragment)); + } + return result; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanningContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanningContext.java new file mode 100644 index 0000000..69cd26b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanningContext.java
@@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionMetadata; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +import java.io.IOException; + +/** + * Context for actions that do include scanning. + */ +public interface IncludeScanningContext extends ActionContext { + /** + * Extracts the set of include files from a source file. + * + * @param actionExecutionContext the execution context + * @param resourceOwner the resource owner + * @param primaryInput the source file to be include scanned + * @param primaryOutput the output file where the results should be put + */ + void extractIncludes(ActionExecutionContext actionExecutionContext, + ActionMetadata resourceOwner, Artifact primaryInput, Artifact primaryOutput) + throws IOException, InterruptedException; + + /** + * Returns the artifact resolver. + */ + ArtifactResolver getArtifactResolver(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/Link.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/Link.java new file mode 100644 index 0000000..26175eb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/Link.java
@@ -0,0 +1,274 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.util.FileTypeSet; + +import java.util.Iterator; + +/** + * Utility types and methods for generating command lines for the linker, given + * a CppLinkAction or LinkConfiguration. + * + * <p>The linker commands, e.g. "ar", may not be functional, i.e. + * they may mutate the output file rather than overwriting it. + * To avoid this, we need to delete the output file before invoking the + * command. But that is not done by this class; deleting the output + * file is the responsibility of the classes derived from LinkStrategy. + */ +public abstract class Link { + + private Link() {} // uninstantiable + + /** The set of valid linker input files. */ + public static final FileTypeSet VALID_LINKER_INPUTS = FileTypeSet.of( + CppFileTypes.ARCHIVE, CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_LIBRARY, CppFileTypes.ALWAYS_LINK_PIC_LIBRARY, + CppFileTypes.OBJECT_FILE, CppFileTypes.PIC_OBJECT_FILE, + CppFileTypes.SHARED_LIBRARY, CppFileTypes.VERSIONED_SHARED_LIBRARY, + CppFileTypes.INTERFACE_SHARED_LIBRARY); + + /** + * These file are supposed to be added using {@code addLibrary()} calls to {@link CppLinkAction} + * but will never be expanded to their constituent {@code .o} files. {@link CppLinkAction} checks + * that these files are never added as non-libraries. + */ + public static final FileTypeSet SHARED_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY, + CppFileTypes.INTERFACE_SHARED_LIBRARY); + + /** + * These need special handling when --thin_archive is true. {@link CppLinkAction} checks that + * these files are never added as non-libraries. + */ + public static final FileTypeSet ARCHIVE_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.ARCHIVE, + CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_LIBRARY, + CppFileTypes.ALWAYS_LINK_PIC_LIBRARY); + + public static final FileTypeSet ARCHIVE_FILETYPES = FileTypeSet.of( + CppFileTypes.ARCHIVE, + CppFileTypes.PIC_ARCHIVE); + + public static final FileTypeSet LINK_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.ALWAYS_LINK_LIBRARY, + CppFileTypes.ALWAYS_LINK_PIC_LIBRARY); + + + /** The set of object files */ + public static final FileTypeSet OBJECT_FILETYPES = FileTypeSet.of( + CppFileTypes.OBJECT_FILE, + CppFileTypes.PIC_OBJECT_FILE); + + /** + * Prefix that is prepended to command line entries that refer to the output + * of cc_fake_binary compile actions. This is a bad hack to signal to the code + * in {@code CppLinkAction#executeFake(Executor, FileOutErr)} that it needs + * special handling. + */ + public static final String FAKE_OBJECT_PREFIX = "fake:"; + + /** + * Types of ELF files that can be created by the linker (.a, .so, .lo, + * executable). + */ + public enum LinkTargetType { + /** A normal static archive. */ + STATIC_LIBRARY(".a", true), + + /** A static archive with .pic.o object files (compiled with -fPIC). */ + PIC_STATIC_LIBRARY(".pic.a", true), + + /** An interface dynamic library. */ + INTERFACE_DYNAMIC_LIBRARY(".ifso", false), + + /** A dynamic library. */ + DYNAMIC_LIBRARY(".so", false), + + /** A static archive without removal of unused object files. */ + ALWAYS_LINK_STATIC_LIBRARY(".lo", true), + + /** A PIC static archive without removal of unused object files. */ + ALWAYS_LINK_PIC_STATIC_LIBRARY(".pic.lo", true), + + /** An executable binary. */ + EXECUTABLE("", false); + + private final String extension; + private final boolean staticLibraryLink; + + private LinkTargetType(String extension, boolean staticLibraryLink) { + this.extension = extension; + this.staticLibraryLink = staticLibraryLink; + } + + public String getExtension() { + return extension; + } + + public boolean isStaticLibraryLink() { + return staticLibraryLink; + } + } + + /** + * The degree of "staticness" of symbol resolution during linking. + */ + public enum LinkStaticness { + FULLY_STATIC, // Static binding of all symbols. + MOSTLY_STATIC, // Use dynamic binding only for symbols from glibc. + DYNAMIC, // Use dynamic binding wherever possible. + } + + /** + * Types of archive. + */ + public enum ArchiveType { + FAT, // Regular archive that includes its members. + THIN, // Thin archive that just points to its members. + START_END_LIB // A --start-lib ... --end-lib group in the command line. + } + + static boolean useStartEndLib(LinkerInput linkerInput, ArchiveType archiveType) { + // TODO(bazel-team): Figure out if PicArchives are actually used. For it to be used, both + // linkingStatically and linkShared must me true, we must be in opt mode and cpu has to be k8. + return archiveType == ArchiveType.START_END_LIB + && ARCHIVE_FILETYPES.matches(linkerInput.getArtifact().getFilename()) + && linkerInput.containsObjectFiles(); + } + + /** + * Replace always used archives with its members. This is used to build the linker cmd line. + */ + public static Iterable<LinkerInput> mergeInputsCmdLine(NestedSet<LibraryToLink> inputs, + boolean globalNeedWholeArchive, ArchiveType archiveType) { + return new FilterMembersForLinkIterable(inputs, globalNeedWholeArchive, archiveType, false); + } + + /** + * Add in any object files which are implicitly named as inputs by the linker. + */ + public static Iterable<LinkerInput> mergeInputsDependencies(NestedSet<LibraryToLink> inputs, + boolean globalNeedWholeArchive, ArchiveType archiveType) { + return new FilterMembersForLinkIterable(inputs, globalNeedWholeArchive, archiveType, true); + } + + /** + * On the fly implementation to filter the members. + */ + private static final class FilterMembersForLinkIterable implements Iterable<LinkerInput> { + private final boolean globalNeedWholeArchive; + private final ArchiveType archiveType; + private final boolean deps; + + private final Iterable<LibraryToLink> inputs; + + private FilterMembersForLinkIterable(Iterable<LibraryToLink> inputs, + boolean globalNeedWholeArchive, ArchiveType archiveType, boolean deps) { + this.globalNeedWholeArchive = globalNeedWholeArchive; + this.archiveType = archiveType; + this.deps = deps; + this.inputs = CollectionUtils.makeImmutable(inputs); + } + + @Override + public Iterator<LinkerInput> iterator() { + return new FilterMembersForLinkIterator(inputs.iterator(), globalNeedWholeArchive, + archiveType, deps); + } + } + + /** + * On the fly implementation to filter the members. + */ + private static final class FilterMembersForLinkIterator extends AbstractIterator<LinkerInput> { + private final boolean globalNeedWholeArchive; + private final ArchiveType archiveType; + private final boolean deps; + + private final Iterator<LibraryToLink> inputs; + private Iterator<LinkerInput> delayList = ImmutableList.<LinkerInput>of().iterator(); + + private FilterMembersForLinkIterator(Iterator<LibraryToLink> inputs, + boolean globalNeedWholeArchive, ArchiveType archiveType, boolean deps) { + this.globalNeedWholeArchive = globalNeedWholeArchive; + this.archiveType = archiveType; + this.deps = deps; + this.inputs = inputs; + } + + @Override + protected LinkerInput computeNext() { + if (delayList.hasNext()) { + return delayList.next(); + } + + while (inputs.hasNext()) { + LibraryToLink inputLibrary = inputs.next(); + Artifact input = inputLibrary.getArtifact(); + String name = input.getFilename(); + + // True if the linker might use the members of this file, i.e., if the file is a thin or + // start_end_lib archive (aka static library). Also check if the library contains object + // files - otherwise getObjectFiles returns null, which would lead to an NPE in + // simpleLinkerInputs. + boolean needMembersForLink = archiveType != ArchiveType.FAT + && ARCHIVE_LIBRARY_FILETYPES.matches(name) && inputLibrary.containsObjectFiles(); + + // True if we will pass the members instead of the original archive. + boolean passMembersToLinkCmd = needMembersForLink + && (globalNeedWholeArchive || LINK_LIBRARY_FILETYPES.matches(name)); + + // If deps is false (when computing the inputs to be passed on the command line), then it's + // an if-then-else, i.e., the passMembersToLinkCmd flag decides whether to pass the object + // files or the archive itself. This flag in turn is based on whether the archives are fat + // or not (thin archives or start_end_lib) - we never expand fat archives, but we do expand + // non-fat archives if we need whole-archives for the entire link, or for the specific + // library (i.e., if alwayslink=1). + // + // If deps is true (when computing the inputs to be passed to the action as inputs), then it + // becomes more complicated. We always need to pass the members for thin and start_end_lib + // archives (needMembersForLink). And we _also_ need to pass the archive file itself unless + // it's a start_end_lib archive (unless it's an alwayslink library). + + // A note about ordering: the order in which the object files and the library are returned + // does not currently matter - this code results in the library returned first, and the + // object files returned after, but only if both are returned, which can only happen if + // deps is true, in which case this code only computes the list of inputs for the link + // action (so the order isn't critical). + if (passMembersToLinkCmd || (deps && needMembersForLink)) { + delayList = LinkerInputs.simpleLinkerInputs(inputLibrary.getObjectFiles()).iterator(); + } + + if (!(passMembersToLinkCmd || (deps && useStartEndLib(inputLibrary, archiveType)))) { + return inputLibrary; + } + + if (delayList.hasNext()) { + return delayList.next(); + } + } + return endOfData(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLine.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLine.java new file mode 100644 index 0000000..1dccafa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLine.java
@@ -0,0 +1,1121 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +/** + * Represents the command line of a linker invocation. It supports executables and dynamic + * libraries as well as static libraries. + */ +@Immutable +public final class LinkCommandLine extends CommandLine { + private final BuildConfiguration configuration; + private final CppConfiguration cppConfiguration; + private final ActionOwner owner; + private final Artifact output; + @Nullable private final Artifact interfaceOutput; + @Nullable private final Artifact symbolCountsOutput; + private final ImmutableList<Artifact> buildInfoHeaderArtifacts; + private final Iterable<? extends LinkerInput> linkerInputs; + private final Iterable<? extends LinkerInput> runtimeInputs; + private final LinkTargetType linkTargetType; + private final LinkStaticness linkStaticness; + private final ImmutableList<String> linkopts; + private final ImmutableSet<String> features; + private final ImmutableMap<Artifact, Artifact> linkstamps; + private final ImmutableList<String> linkstampCompileOptions; + @Nullable private final PathFragment runtimeSolibDir; + private final boolean nativeDeps; + private final boolean useTestOnlyFlags; + private final boolean needWholeArchive; + private final boolean supportsParamFiles; + @Nullable private final Artifact interfaceSoBuilder; + + private LinkCommandLine( + BuildConfiguration configuration, + ActionOwner owner, + Artifact output, + @Nullable Artifact interfaceOutput, + @Nullable Artifact symbolCountsOutput, + ImmutableList<Artifact> buildInfoHeaderArtifacts, + Iterable<? extends LinkerInput> linkerInputs, + Iterable<? extends LinkerInput> runtimeInputs, + LinkTargetType linkTargetType, + LinkStaticness linkStaticness, + ImmutableList<String> linkopts, + ImmutableSet<String> features, + ImmutableMap<Artifact, Artifact> linkstamps, + ImmutableList<String> linkstampCompileOptions, + @Nullable PathFragment runtimeSolibDir, + boolean nativeDeps, + boolean useTestOnlyFlags, + boolean needWholeArchive, + boolean supportsParamFiles, + Artifact interfaceSoBuilder) { + Preconditions.checkArgument(linkTargetType != LinkTargetType.INTERFACE_DYNAMIC_LIBRARY, + "you can't link an interface dynamic library directly"); + if (linkTargetType != LinkTargetType.DYNAMIC_LIBRARY) { + Preconditions.checkArgument(interfaceOutput == null, + "interface output may only be non-null for dynamic library links"); + } + if (linkTargetType.isStaticLibraryLink()) { + Preconditions.checkArgument(linkstamps.isEmpty(), + "linkstamps may only be present on dynamic library or executable links"); + Preconditions.checkArgument(linkStaticness == LinkStaticness.FULLY_STATIC, + "static library link must be static"); + Preconditions.checkArgument(buildInfoHeaderArtifacts.isEmpty(), + "build info headers may only be present on dynamic library or executable links"); + Preconditions.checkArgument(symbolCountsOutput == null, + "the symbol counts output must be null for static links"); + Preconditions.checkArgument(runtimeSolibDir == null, + "the runtime solib directory must be null for static links"); + Preconditions.checkArgument(!nativeDeps, + "the native deps flag must be false for static links"); + Preconditions.checkArgument(!needWholeArchive, + "the need whole archive flag must be false for static links"); + } + + this.configuration = Preconditions.checkNotNull(configuration); + this.cppConfiguration = configuration.getFragment(CppConfiguration.class); + this.owner = Preconditions.checkNotNull(owner); + this.output = Preconditions.checkNotNull(output); + this.interfaceOutput = interfaceOutput; + this.symbolCountsOutput = symbolCountsOutput; + this.buildInfoHeaderArtifacts = Preconditions.checkNotNull(buildInfoHeaderArtifacts); + this.linkerInputs = Preconditions.checkNotNull(linkerInputs); + this.runtimeInputs = Preconditions.checkNotNull(runtimeInputs); + this.linkTargetType = Preconditions.checkNotNull(linkTargetType); + this.linkStaticness = Preconditions.checkNotNull(linkStaticness); + // For now, silently ignore linkopts if this is a static library link. + this.linkopts = linkTargetType.isStaticLibraryLink() + ? ImmutableList.<String>of() + : Preconditions.checkNotNull(linkopts); + this.features = Preconditions.checkNotNull(features); + this.linkstamps = Preconditions.checkNotNull(linkstamps); + this.linkstampCompileOptions = linkstampCompileOptions; + this.runtimeSolibDir = runtimeSolibDir; + this.nativeDeps = nativeDeps; + this.useTestOnlyFlags = useTestOnlyFlags; + this.needWholeArchive = needWholeArchive; + this.supportsParamFiles = supportsParamFiles; + // For now, silently ignore interfaceSoBuilder if we don't build an interface dynamic library. + this.interfaceSoBuilder = + ((linkTargetType == LinkTargetType.DYNAMIC_LIBRARY) && (interfaceOutput != null)) + ? Preconditions.checkNotNull(interfaceSoBuilder, + "cannot build interface dynamic library without builder") + : null; + } + + /** + * Returns an interface shared object output artifact produced during linking. This only returns + * non-null if {@link #getLinkTargetType} is {@code DYNAMIC_LIBRARY} and an interface shared + * object was requested. + */ + @Nullable public Artifact getInterfaceOutput() { + return interfaceOutput; + } + + /** + * Returns an artifact containing the number of symbols used per object file passed to the linker. + * This is currently a gold only feature, and is only produced for executables. If another target + * is being linked, or if symbol counts output is disabled, this will be null. + */ + @Nullable public Artifact getSymbolCountsOutput() { + return symbolCountsOutput; + } + + /** + * Returns the (ordered, immutable) list of header files that contain build info. + */ + public ImmutableList<Artifact> getBuildInfoHeaderArtifacts() { + return buildInfoHeaderArtifacts; + } + + /** + * Returns the (ordered, immutable) list of paths to the linker's input files. + */ + public Iterable<? extends LinkerInput> getLinkerInputs() { + return linkerInputs; + } + + /** + * Returns the runtime inputs to the linker. + */ + public Iterable<? extends LinkerInput> getRuntimeInputs() { + return runtimeInputs; + } + + /** + * Returns the current type of link target set. + */ + public LinkTargetType getLinkTargetType() { + return linkTargetType; + } + + /** + * Returns the "staticness" of the link. + */ + public LinkStaticness getLinkStaticness() { + return linkStaticness; + } + + /** + * Returns the additional linker options for this link. + */ + public ImmutableList<String> getLinkopts() { + return linkopts; + } + + /** + * Returns a (possibly empty) mapping of (C++ source file, .o output file) pairs for source files + * that need to be compiled at link time. + * + * <p>This is used to embed various values from the build system into binaries to identify their + * provenance. + */ + public ImmutableMap<Artifact, Artifact> getLinkstamps() { + return linkstamps; + } + + /** + * Returns the location of the C++ runtime solib symlinks. If null, the C++ dynamic runtime + * libraries either do not exist (because they do not come from the depot) or they are in the + * regular solib directory. + */ + @Nullable public PathFragment getRuntimeSolibDir() { + return runtimeSolibDir; + } + + /** + * Returns true for libraries linked as native dependencies for other languages. + */ + public boolean isNativeDeps() { + return nativeDeps; + } + + /** + * Returns true if this link should use test-specific flags (e.g. $EXEC_ORIGIN as the root for + * finding shared libraries or lazy binding); false by default. See bug "Please use + * $EXEC_ORIGIN instead of $ORIGIN when linking cc_tests" for further context. + */ + public boolean useTestOnlyFlags() { + return useTestOnlyFlags; + } + + /** + * Splits the link command-line into a part to be written to a parameter file, and the remaining + * actual command line to be executed (which references the parameter file). Call {@link + * #canBeSplit} first to check if the command-line can be split. + * + * @throws IllegalStateException if the command-line cannot be split + */ + @VisibleForTesting + final Pair<List<String>, List<String>> splitCommandline(PathFragment paramExecPath) { + Preconditions.checkState(canBeSplit()); + List<String> args = getRawLinkArgv(); + if (linkTargetType.isStaticLibraryLink()) { + // Ar link commands can also generate huge command lines. + List<String> paramFileArgs = args.subList(1, args.size()); + List<String> commandlineArgs = new ArrayList<>(); + commandlineArgs.add(args.get(0)); + + commandlineArgs.add("@" + paramExecPath.getPathString()); + return Pair.of(commandlineArgs, paramFileArgs); + } else { + // Gcc link commands tend to generate humongous commandlines for some targets, which may + // not fit on some remote execution machines. To work around this we will employ the help of + // a parameter file and pass any linker options through it. + List<String> paramFileArgs = new ArrayList<>(); + List<String> commandlineArgs = new ArrayList<>(); + extractArgumentsForParamFile(args, commandlineArgs, paramFileArgs); + + commandlineArgs.add("-Wl,@" + paramExecPath.getPathString()); + return Pair.of(commandlineArgs, paramFileArgs); + } + } + + boolean canBeSplit() { + if (!supportsParamFiles) { + return false; + } + switch (linkTargetType) { + // We currently can't split dynamic library links if they have interface outputs. That was + // probably an unintended side effect of the change that introduced interface outputs. + case DYNAMIC_LIBRARY: + return interfaceOutput == null; + case EXECUTABLE: + case STATIC_LIBRARY: + case PIC_STATIC_LIBRARY: + case ALWAYS_LINK_STATIC_LIBRARY: + case ALWAYS_LINK_PIC_STATIC_LIBRARY: + return true; + default: + return false; + } + } + + private static void extractArgumentsForParamFile(List<String> args, List<String> commandlineArgs, + List<String> paramFileArgs) { + // Note, that it is not important that all linker arguments are extracted so that + // they can be moved into a parameter file, but the vast majority should. + commandlineArgs.add(args.get(0)); // gcc command, must not be moved! + int argsSize = args.size(); + for (int i = 1; i < argsSize; i++) { + String arg = args.get(i); + if (arg.equals("-Wl,-no-whole-archive")) { + paramFileArgs.add("-no-whole-archive"); + } else if (arg.equals("-Wl,-whole-archive")) { + paramFileArgs.add("-whole-archive"); + } else if (arg.equals("-Wl,--start-group")) { + paramFileArgs.add("--start-group"); + } else if (arg.equals("-Wl,--end-group")) { + paramFileArgs.add("--end-group"); + } else if (arg.equals("-Wl,--start-lib")) { + paramFileArgs.add("--start-lib"); + } else if (arg.equals("-Wl,--end-lib")) { + paramFileArgs.add("--end-lib"); + } else if (arg.equals("--incremental-unchanged")) { + paramFileArgs.add(arg); + } else if (arg.equals("--incremental-changed")) { + paramFileArgs.add(arg); + } else if (arg.charAt(0) == '-') { + if (arg.startsWith("-l")) { + paramFileArgs.add(arg); + } else { + // Anything else starting with a '-' can stay on the commandline. + commandlineArgs.add(arg); + if (arg.equals("-o")) { + // Special case for '-o': add the following argument as well - it is the output file! + commandlineArgs.add(args.get(++i)); + } + } + } else if (arg.endsWith(".a") || arg.endsWith(".lo") || arg.endsWith(".so") + || arg.endsWith(".ifso") || arg.endsWith(".o") + || CppFileTypes.VERSIONED_SHARED_LIBRARY.matches(arg)) { + // All objects of any kind go into the linker parameters. + paramFileArgs.add(arg); + } else { + // Everything that's left stays conservatively on the commandline. + commandlineArgs.add(arg); + } + } + } + + /** + * Returns a raw link command for the given link invocation, including both command and + * arguments (argv). After any further usage-specific processing, this can be passed to + * {@link #finalizeWithLinkstampCommands} to give the final command line. + * + * @return raw link command line. + */ + public List<String> getRawLinkArgv() { + List<String> argv = new ArrayList<>(); + switch (linkTargetType) { + case EXECUTABLE: + addCppArgv(argv); + break; + + case DYNAMIC_LIBRARY: + if (interfaceOutput != null) { + argv.add(configuration.getShExecutable().getPathString()); + argv.add("-c"); + argv.add("build_iface_so=\"$0\"; impl=\"$1\"; iface=\"$2\"; cmd=\"$3\"; shift 3; " + + "\"$cmd\" \"$@\" && \"$build_iface_so\" \"$impl\" \"$iface\""); + argv.add(interfaceSoBuilder.getExecPathString()); + argv.add(output.getExecPathString()); + argv.add(interfaceOutput.getExecPathString()); + } + addCppArgv(argv); + // -pie is not compatible with -shared and should be + // removed when the latter is part of the link command. Should we need to further + // distinguish between shared libraries and executables, we could add additional + // command line / CROSSTOOL flags that distinguish them. But as long as this is + // the only relevant use case we're just special-casing it here. + Iterables.removeIf(argv, Predicates.equalTo("-pie")); + break; + + case STATIC_LIBRARY: + case PIC_STATIC_LIBRARY: + case ALWAYS_LINK_STATIC_LIBRARY: + case ALWAYS_LINK_PIC_STATIC_LIBRARY: + // The static library link command follows this template: + // ar <cmd> <output_archive> <input_files...> + argv.add(cppConfiguration.getArExecutable().getPathString()); + argv.addAll( + cppConfiguration.getArFlags(cppConfiguration.archiveType() == Link.ArchiveType.THIN)); + argv.add(output.getExecPathString()); + addInputFileLinkOptions(argv, /*needWholeArchive=*/false, + /*includeLinkopts=*/false); + break; + + default: + throw new IllegalArgumentException(); + } + + // Fission mode: debug info is in .dwo files instead of .o files. Inform the linker of this. + if (!linkTargetType.isStaticLibraryLink() && cppConfiguration.useFission()) { + argv.add("-Wl,--gdb-index"); + } + + return argv; + } + + @Override + public List<String> arguments() { + return finalizeWithLinkstampCommands(getRawLinkArgv()); + } + + /** + * Takes a raw link command line and gives the final link command that will + * also first compile any linkstamps necessary. Elements of rawLinkArgv are + * shell-escaped. + * + * @param rawLinkArgv raw link command line + * + * @return final link command line suitable for execution + */ + public List<String> finalizeWithLinkstampCommands(List<String> rawLinkArgv) { + return addLinkstampingToCommand(getLinkstampCompileCommands(""), rawLinkArgv, true); + } + + /** + * Takes a raw link command line and gives the final link command that will also first compile any + * linkstamps necessary. Elements of rawLinkArgv are not shell-escaped. + * + * @param rawLinkArgv raw link command line + * @param outputPrefix prefix to add before the linkstamp outputs' exec paths + * + * @return final link command line suitable for execution + */ + public List<String> finalizeAlreadyEscapedWithLinkstampCommands( + List<String> rawLinkArgv, String outputPrefix) { + return addLinkstampingToCommand(getLinkstampCompileCommands(outputPrefix), rawLinkArgv, false); + } + + /** + * Adds linkstamp compilation to the (otherwise) fully specified link + * command if {@link #getLinkstamps} is non-empty. + * + * <p>Linkstamps were historically compiled implicitly as part of the link + * command, but implicit compilation doesn't guarantee consistent outputs. + * For example, the command "gcc input.o input.o foo/linkstamp.cc -o myapp" + * causes gcc to implicitly run "gcc foo/linkstamp.cc -o /tmp/ccEtJHDB.o", + * for some internally decided output path /tmp/ccEtJHDB.o, then add that path + * to the linker's command line options. The name of this path can change + * even between equivalently specified gcc invocations. + * + * <p>So now we explicitly compile these files in their own command + * invocations before running the link command, thus giving us direct + * control over the naming of their outputs. This method adds those extra + * steps as necessary. + * @param linkstampCommands individual linkstamp compilation commands + * @param linkCommand the complete list of link command arguments (after + * .params file compacting) for an invocation + * @param escapeArgs if true, linkCommand arguments are shell escaped. if + * false, arguments are returned as-is + * + * @return The original argument list if no linkstamps compilation commands + * are given, otherwise an expanded list that adds the linkstamp + * compilation commands and funnels their outputs into the link step. + * Note that these outputs only need to persist for the duration of + * the link step. + */ + private static List<String> addLinkstampingToCommand( + List<String> linkstampCommands, + List<String> linkCommand, + boolean escapeArgs) { + if (linkstampCommands.isEmpty()) { + return linkCommand; + } else { + List<String> batchCommand = Lists.newArrayListWithCapacity(3); + batchCommand.add("/bin/bash"); + batchCommand.add("-c"); + batchCommand.add( + Joiner.on(" && ").join(linkstampCommands) + " && " + + (escapeArgs + ? ShellEscaper.escapeJoinAll(linkCommand) + : Joiner.on(" ").join(linkCommand))); + return ImmutableList.copyOf(batchCommand); + } + } + + /** + * Computes, for each C++ source file in + * {@link #getLinkstamps}, the command necessary to compile + * that file such that the output is correctly fed into the link command. + * + * <p>As these options (as well as all others) are taken into account when + * computing the action key, they do not directly contain volatile build + * information to avoid unnecessary relinking. Instead this information is + * passed as an additional header generated by + * {@link com.google.devtools.build.lib.rules.cpp.WriteBuildInfoHeaderAction}. + * + * @param outputPrefix prefix to add before the linkstamp outputs' exec paths + * @return a list of shell-escaped compiler commmands, one for each entry + * in {@link #getLinkstamps} + */ + public List<String> getLinkstampCompileCommands(String outputPrefix) { + if (linkstamps.isEmpty()) { + return ImmutableList.of(); + } + + String compilerCommand = cppConfiguration.getCppExecutable().getPathString(); + List<String> commands = Lists.newArrayListWithCapacity(linkstamps.size()); + + for (Map.Entry<Artifact, Artifact> linkstamp : linkstamps.entrySet()) { + List<String> optionList = new ArrayList<>(); + + // Defines related to the build info are read from generated headers. + for (Artifact header : buildInfoHeaderArtifacts) { + optionList.add("-include"); + optionList.add(header.getExecPathString()); + } + + String labelReplacement = Matcher.quoteReplacement( + isSharedNativeLibrary() ? output.getExecPathString() : Label.print(owner.getLabel())); + String outputPathReplacement = Matcher.quoteReplacement( + output.getExecPathString()); + for (String option : linkstampCompileOptions) { + optionList.add(option + .replaceAll(Pattern.quote("${LABEL}"), labelReplacement) + .replaceAll(Pattern.quote("${OUTPUT_PATH}"), outputPathReplacement)); + } + + optionList.add("-DGPLATFORM=\"" + cppConfiguration + "\""); + + // Needed to find headers included from linkstamps. + optionList.add("-I."); + + // Add sysroot. + PathFragment sysroot = cppConfiguration.getSysroot(); + if (sysroot != null) { + optionList.add("--sysroot=" + sysroot.getPathString()); + } + + // Add toolchain compiler options. + optionList.addAll(cppConfiguration.getCompilerOptions(features)); + optionList.addAll(cppConfiguration.getCOptions()); + optionList.addAll(cppConfiguration.getUnfilteredCompilerOptions(features)); + + // For dynamic libraries, produce position independent code. + if (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY + && cppConfiguration.toolchainNeedsPic()) { + optionList.add("-fPIC"); + } + + // Stamp FDO builds with FDO subtype string + String fdoBuildStamp = CppHelper.getFdoBuildStamp(cppConfiguration); + if (fdoBuildStamp != null) { + optionList.add("-D" + CppConfiguration.FDO_STAMP_MACRO + "=\"" + fdoBuildStamp + "\""); + } + + // Add the compilation target. + optionList.add("-c"); + optionList.add(linkstamp.getKey().getExecPathString()); + + // Assemble the final command, exempting outputPrefix from shell escaping. + commands.add(compilerCommand + " " + + ShellEscaper.escapeJoinAll(optionList) + + " -o " + + outputPrefix + + ShellEscaper.escapeString(linkstamp.getValue().getExecPathString())); + } + + return commands; + } + + /** + * Determine the arguments to pass to the C++ compiler when linking. + * Add them to the {@code argv} parameter. + */ + private void addCppArgv(List<String> argv) { + argv.add(cppConfiguration.getCppExecutable().getPathString()); + + // When using gold to link an executable, output the number of used and unused symbols. + if (symbolCountsOutput != null) { + argv.add("-Wl,--print-symbol-counts=" + symbolCountsOutput.getExecPathString()); + } + + if (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY) { + argv.add("-shared"); + } + + // Add the outputs of any associated linkstamp compilations. + for (Artifact linkstampOutput : linkstamps.values()) { + argv.add(linkstampOutput.getExecPathString()); + } + + boolean fullyStatic = (linkStaticness == LinkStaticness.FULLY_STATIC); + boolean mostlyStatic = (linkStaticness == LinkStaticness.MOSTLY_STATIC); + boolean sharedLinkopts = + linkTargetType == LinkTargetType.DYNAMIC_LIBRARY + || linkopts.contains("-shared") + || cppConfiguration.getLinkOptions().contains("-shared"); + + if (output != null) { + argv.add("-o"); + String execpath = output.getExecPathString(); + if (mostlyStatic + && linkTargetType == LinkTargetType.EXECUTABLE + && cppConfiguration.skipStaticOutputs()) { + // Linked binary goes to /dev/null; bogus dependency info in its place. + Collections.addAll(argv, "/dev/null", "-MMD", "-MF", execpath); // thanks Ambrose + } else { + argv.add(execpath); + } + } + + addInputFileLinkOptions(argv, needWholeArchive, /*includeLinkopts=*/true); + + // Extra toolchain link options based on the output's link staticness. + if (fullyStatic) { + argv.addAll(cppConfiguration.getFullyStaticLinkOptions(features, sharedLinkopts)); + } else if (mostlyStatic) { + argv.addAll(cppConfiguration.getMostlyStaticLinkOptions(features, sharedLinkopts)); + } else { + argv.addAll(cppConfiguration.getDynamicLinkOptions(features, sharedLinkopts)); + } + + // Extra test-specific link options. + if (useTestOnlyFlags) { + argv.addAll(cppConfiguration.getTestOnlyLinkOptions()); + } + + if (configuration.isCodeCoverageEnabled()) { + argv.add("-lgcov"); + } + + if (linkTargetType == LinkTargetType.EXECUTABLE && cppConfiguration.forcePic()) { + argv.add("-pie"); + } + + argv.addAll(cppConfiguration.getLinkOptions()); + argv.addAll(cppConfiguration.getFdoSupport().getLinkOptions()); + } + + private static boolean isDynamicLibrary(LinkerInput linkInput) { + Artifact libraryArtifact = linkInput.getArtifact(); + String name = libraryArtifact.getFilename(); + return Link.SHARED_LIBRARY_FILETYPES.matches(name) && name.startsWith("lib"); + } + + private boolean isSharedNativeLibrary() { + return nativeDeps && cppConfiguration.shareNativeDeps(); + } + + /** + * When linking a shared library fully or mostly static then we need to link in + * *all* dependent files, not just what the shared library needs for its own + * code. This is done by wrapping all objects/libraries with + * -Wl,-whole-archive and -Wl,-no-whole-archive. For this case the + * globalNeedWholeArchive parameter must be set to true. Otherwise only + * library objects (.lo) need to be wrapped with -Wl,-whole-archive and + * -Wl,-no-whole-archive. + */ + private void addInputFileLinkOptions(List<String> argv, boolean globalNeedWholeArchive, + boolean includeLinkopts) { + // The Apple ld doesn't support -whole-archive/-no-whole-archive. It + // does have -all_load/-noall_load, but -all_load is a global setting + // that affects all subsequent files, and -noall_load is simply ignored. + // TODO(bazel-team): Not sure what the implications of this are, other than + // bloated binaries. + boolean macosx = cppConfiguration.getTargetLibc().equals("macosx"); + if (globalNeedWholeArchive) { + argv.add(macosx ? "-Wl,-all_load" : "-Wl,-whole-archive"); + } + + // Used to collect -L and -Wl,-rpath options, ensuring that each used only once. + Set<String> libOpts = new LinkedHashSet<>(); + + // List of command line parameters to link input files (either directly or using -l). + List<String> linkerInputs = new ArrayList<>(); + + // List of command line parameters that need to be placed *outside* of + // --whole-archive ... --no-whole-archive. + List<String> noWholeArchiveInputs = new ArrayList<>(); + + PathFragment solibDir = configuration.getBinDirectory().getExecPath() + .getRelative(cppConfiguration.getSolibDirectory()); + String runtimeSolibName = runtimeSolibDir != null ? runtimeSolibDir.getBaseName() : null; + boolean runtimeRpath = runtimeSolibDir != null + && (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY + || (linkTargetType == LinkTargetType.EXECUTABLE + && linkStaticness == LinkStaticness.DYNAMIC)); + + String rpathRoot = null; + List<String> runtimeRpathEntries = new ArrayList<>(); + + if (output != null) { + String origin = + useTestOnlyFlags && cppConfiguration.supportsExecOrigin() ? "$EXEC_ORIGIN/" : "$ORIGIN/"; + if (runtimeRpath) { + runtimeRpathEntries.add("-Wl,-rpath," + origin + runtimeSolibName + "/"); + } + + // Calculate the correct relative value for the "-rpath" link option (which sets + // the search path for finding shared libraries). + if (isSharedNativeLibrary()) { + // For shared native libraries, special symlinking is applied to ensure C++ + // runtimes are available under $ORIGIN/_solib_[arch]. So we set the RPATH to find + // them. + // + // Note that we have to do this because $ORIGIN points to different paths for + // different targets. In other words, blaze-bin/d1/d2/d3/a_shareddeps.so and + // blaze-bin/d4/b_shareddeps.so have different path depths. The first could + // reference a standard blaze-bin/_solib_[arch] via $ORIGIN/../../../_solib[arch], + // and the second could use $ORIGIN/../_solib_[arch]. But since this is a shared + // artifact, both are symlinks to the same place, so + // there's no *one* RPATH setting that fits all targets involved in the sharing. + rpathRoot = "-Wl,-rpath," + origin + ":" + + origin + cppConfiguration.getSolibDirectory() + "/"; + if (runtimeRpath) { + runtimeRpathEntries.add("-Wl,-rpath," + origin + "../" + runtimeSolibName + "/"); + } + } else { + // For all other links, calculate the relative path from the output file to _solib_[arch] + // (the directory where all shared libraries are stored, which resides under the blaze-bin + // directory. In other words, given blaze-bin/my/package/binary, rpathRoot would be + // "../../_solib_[arch]". + if (runtimeRpath) { + runtimeRpathEntries.add("-Wl,-rpath," + origin + + Strings.repeat("../", output.getRootRelativePath().segmentCount() - 1) + + runtimeSolibName + "/"); + } + + rpathRoot = "-Wl,-rpath," + + origin + Strings.repeat("../", output.getRootRelativePath().segmentCount() - 1) + + cppConfiguration.getSolibDirectory() + "/"; + + if (nativeDeps) { + // We also retain the $ORIGIN/ path to solibs that are in _solib_<arch>, as opposed to + // the package directory) + if (runtimeRpath) { + runtimeRpathEntries.add("-Wl,-rpath," + origin + "../" + runtimeSolibName + "/"); + } + rpathRoot += ":" + origin; + } + } + } + + boolean includeSolibDir = false; + + for (LinkerInput input : getLinkerInputs()) { + if (isDynamicLibrary(input)) { + PathFragment libDir = input.getArtifact().getExecPath().getParentDirectory(); + Preconditions.checkState( + libDir.startsWith(solibDir), + "Artifact '%s' is not under directory '%s'.", input.getArtifact(), solibDir); + if (libDir.equals(solibDir)) { + includeSolibDir = true; + } + addDynamicInputLinkOptions(input, linkerInputs, libOpts, solibDir, rpathRoot); + } else { + addStaticInputLinkOptions(input, linkerInputs); + } + } + + boolean includeRuntimeSolibDir = false; + + for (LinkerInput input : runtimeInputs) { + List<String> optionsList = globalNeedWholeArchive + ? noWholeArchiveInputs + : linkerInputs; + + if (isDynamicLibrary(input)) { + PathFragment libDir = input.getArtifact().getExecPath().getParentDirectory(); + Preconditions.checkState(runtimeSolibDir != null && libDir.equals(runtimeSolibDir), + "Artifact '%s' is not under directory '%s'.", input.getArtifact(), solibDir); + includeRuntimeSolibDir = true; + addDynamicInputLinkOptions(input, optionsList, libOpts, solibDir, rpathRoot); + } else { + addStaticInputLinkOptions(input, optionsList); + } + } + + // rpath ordering matters for performance; first add the one where most libraries are found. + if (includeSolibDir && rpathRoot != null) { + argv.add(rpathRoot); + } + if (includeRuntimeSolibDir) { + argv.addAll(runtimeRpathEntries); + } + argv.addAll(libOpts); + + // Need to wrap static libraries with whole-archive option + for (String option : linkerInputs) { + if (!globalNeedWholeArchive && Link.LINK_LIBRARY_FILETYPES.matches(option)) { + argv.add(macosx ? "-Wl,-all_load" : "-Wl,-whole-archive"); + argv.add(option); + argv.add(macosx ? "-Wl,-noall_load" : "-Wl,-no-whole-archive"); + } else { + argv.add(option); + } + } + + if (globalNeedWholeArchive) { + argv.add(macosx ? "-Wl,-noall_load" : "-Wl,-no-whole-archive"); + argv.addAll(noWholeArchiveInputs); + } + + if (includeLinkopts) { + /* + * For backwards compatibility, linkopts come _after_ inputFiles. + * This is needed to allow linkopts to contain libraries and + * positional library-related options such as + * -Wl,--begin-group -lfoo -lbar -Wl,--end-group + * or + * -Wl,--as-needed -lfoo -Wl,--no-as-needed + * + * As for the relative order of the three different flavours of linkopts + * (global defaults, per-target linkopts, and command-line linkopts), + * we have no idea what the right order should be, or if anyone cares. + */ + argv.addAll(linkopts); + } + } + + /** + * Adds command-line options for a dynamic library input file into + * options and libOpts. + */ + private void addDynamicInputLinkOptions(LinkerInput input, List<String> options, + Set<String> libOpts, PathFragment solibDir, String rpathRoot) { + Preconditions.checkState(isDynamicLibrary(input)); + Preconditions.checkState( + !Link.useStartEndLib(input, cppConfiguration.archiveType())); + + Artifact inputArtifact = input.getArtifact(); + PathFragment libDir = inputArtifact.getExecPath().getParentDirectory(); + if (rpathRoot != null + && !libDir.equals(solibDir) + && (runtimeSolibDir == null || !runtimeSolibDir.equals(libDir))) { + String dotdots = ""; + PathFragment commonParent = solibDir; + while (!libDir.startsWith(commonParent)) { + dotdots += "../"; + commonParent = commonParent.getParentDirectory(); + } + + libOpts.add(rpathRoot + dotdots + libDir.relativeTo(commonParent).getPathString()); + } + + libOpts.add("-L" + inputArtifact.getExecPath().getParentDirectory().getPathString()); + + String name = inputArtifact.getFilename(); + if (CppFileTypes.SHARED_LIBRARY.matches(name)) { + String libName = name.replaceAll("(^lib|\\.so$)", ""); + options.add("-l" + libName); + } else { + // Interface shared objects have a non-standard extension + // that the linker won't be able to find. So use the + // filename directly rather than a -l option. Since the + // library has an SONAME attribute, this will work fine. + options.add(inputArtifact.getExecPathString()); + } + } + + /** + * Adds command-line options for a static library or non-library input + * into options. + */ + private void addStaticInputLinkOptions(LinkerInput input, List<String> options) { + Preconditions.checkState(!isDynamicLibrary(input)); + + // start-lib/end-lib library: adds its input object files. + if (Link.useStartEndLib(input, cppConfiguration.archiveType())) { + Iterable<Artifact> archiveMembers = input.getObjectFiles(); + if (!Iterables.isEmpty(archiveMembers)) { + options.add("-Wl,--start-lib"); + for (Artifact member : archiveMembers) { + options.add(member.getExecPathString()); + } + options.add("-Wl,--end-lib"); + } + // For anything else, add the input directly. + } else { + Artifact inputArtifact = input.getArtifact(); + if (input.isFake()) { + options.add(Link.FAKE_OBJECT_PREFIX + inputArtifact.getExecPathString()); + } else { + options.add(inputArtifact.getExecPathString()); + } + } + } + + /** + * A builder for a {@link LinkCommandLine}. + */ + public static final class Builder { + // TODO(bazel-team): Pass this in instead of having it here. Maybe move to cc_toolchain. + private static final ImmutableList<String> DEFAULT_LINKSTAMP_OPTIONS = ImmutableList.of( + // G3_VERSION_INFO and G3_TARGET_NAME are C string literals that normally + // contain the label of the target being linked. However, they are set + // differently when using shared native deps. In that case, a single .so file + // is shared by multiple targets, and its contents cannot depend on which + // target(s) were specified on the command line. So in that case we have + // to use the (obscure) name of the .so file instead, or more precisely + // the path of the .so file relative to the workspace root. + "-DG3_VERSION_INFO=\"${LABEL}\"", + "-DG3_TARGET_NAME=\"${LABEL}\"", + + // G3_BUILD_TARGET is a C string literal containing the output of this + // link. (An undocumented and untested invariant is that G3_BUILD_TARGET is the location of + // the executable, either absolutely, or relative to the directory part of BUILD_INFO.) + "-DG3_BUILD_TARGET=\"${OUTPUT_PATH}\""); + + private final BuildConfiguration configuration; + private final ActionOwner owner; + + @Nullable private Artifact output; + @Nullable private Artifact interfaceOutput; + @Nullable private Artifact symbolCountsOutput; + private ImmutableList<Artifact> buildInfoHeaderArtifacts = ImmutableList.of(); + private Iterable<? extends LinkerInput> linkerInputs = ImmutableList.of(); + private Iterable<? extends LinkerInput> runtimeInputs = ImmutableList.of(); + @Nullable private LinkTargetType linkTargetType; + private LinkStaticness linkStaticness = LinkStaticness.FULLY_STATIC; + private ImmutableList<String> linkopts = ImmutableList.of(); + private ImmutableSet<String> features = ImmutableSet.of(); + private ImmutableMap<Artifact, Artifact> linkstamps = ImmutableMap.of(); + private List<String> linkstampCompileOptions = new ArrayList<>(); + @Nullable private PathFragment runtimeSolibDir; + private boolean nativeDeps; + private boolean useTestOnlyFlags; + private boolean needWholeArchive; + private boolean supportsParamFiles; + @Nullable private Artifact interfaceSoBuilder; + + public Builder(BuildConfiguration configuration, ActionOwner owner) { + this.configuration = configuration; + this.owner = owner; + } + + public Builder(RuleContext ruleContext) { + this(ruleContext.getConfiguration(), ruleContext.getActionOwner()); + } + + public LinkCommandLine build() { + ImmutableList<String> actualLinkstampCompileOptions; + if (linkstampCompileOptions.isEmpty()) { + actualLinkstampCompileOptions = DEFAULT_LINKSTAMP_OPTIONS; + } else { + actualLinkstampCompileOptions = ImmutableList.copyOf( + Iterables.concat(DEFAULT_LINKSTAMP_OPTIONS, linkstampCompileOptions)); + } + return new LinkCommandLine(configuration, owner, output, interfaceOutput, + symbolCountsOutput, buildInfoHeaderArtifacts, linkerInputs, runtimeInputs, linkTargetType, + linkStaticness, linkopts, features, linkstamps, actualLinkstampCompileOptions, + runtimeSolibDir, nativeDeps, useTestOnlyFlags, needWholeArchive, supportsParamFiles, + interfaceSoBuilder); + } + + /** + * Sets the type of the link. It is an error to try to set this to {@link + * LinkTargetType#INTERFACE_DYNAMIC_LIBRARY}. Note that all the static target types (see {@link + * LinkTargetType#isStaticLibraryLink}) are equivalent, and there is no check that the output + * artifact matches the target type extension. + */ + public Builder setLinkTargetType(LinkTargetType linkTargetType) { + Preconditions.checkArgument(linkTargetType != LinkTargetType.INTERFACE_DYNAMIC_LIBRARY); + this.linkTargetType = linkTargetType; + return this; + } + + /** + * Sets the primary output artifact. This must be called before calling {@link #build}. + */ + public Builder setOutput(Artifact output) { + this.output = output; + return this; + } + + /** + * Sets a list of linker inputs. These get turned into linker options depending on the + * staticness and the target type. This call makes an immutable copy of the inputs, if the + * provided Iterable isn't already immutable (see {@link CollectionUtils#makeImmutable}). + */ + public Builder setLinkerInputs(Iterable<LinkerInput> linkerInputs) { + this.linkerInputs = CollectionUtils.makeImmutable(linkerInputs); + return this; + } + + public Builder setRuntimeInputs(ImmutableList<LinkerInput> runtimeInputs) { + this.runtimeInputs = runtimeInputs; + return this; + } + + /** + * Sets the additional interface output artifact, which is only used for dynamic libraries. The + * {@link #build} method throws an exception if the target type is not {@link + * LinkTargetType#DYNAMIC_LIBRARY}. + */ + public Builder setInterfaceOutput(Artifact interfaceOutput) { + this.interfaceOutput = interfaceOutput; + return this; + } + + /** + * Sets an additional output artifact that contains symbol counts. The {@link #build} method + * throws an exception if this is non-null for a static link (see + * {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setSymbolCountsOutput(Artifact symbolCountsOutput) { + this.symbolCountsOutput = symbolCountsOutput; + return this; + } + + /** + * Sets the linker options. These are passed to the linker in addition to the other linker + * options like linker inputs, symbol count options, etc. The {@link #build} method + * throws an exception if the linker options are non-empty for a static link (see {@link + * LinkTargetType#isStaticLibraryLink}). + */ + public Builder setLinkopts(ImmutableList<String> linkopts) { + this.linkopts = linkopts; + return this; + } + + /** + * Sets how static the link is supposed to be. For static target types (see {@link + * LinkTargetType#isStaticLibraryLink}), the {@link #build} method throws an exception if this + * is not {@link LinkStaticness#FULLY_STATIC}. The default setting is {@link + * LinkStaticness#FULLY_STATIC}. + */ + public Builder setLinkStaticness(LinkStaticness linkStaticness) { + this.linkStaticness = linkStaticness; + return this; + } + + /** + * Sets the binary that should be used to create the interface output for a dynamic library. + * This is ignored unless the target type is {@link LinkTargetType#DYNAMIC_LIBRARY} and an + * interface output artifact is specified. + */ + public Builder setInterfaceSoBuilder(Artifact interfaceSoBuilder) { + this.interfaceSoBuilder = interfaceSoBuilder; + return this; + } + + /** + * Sets the linkstamps. Linkstamps are additional C++ source files that are compiled as part of + * the link command. The {@link #build} method throws an exception if the linkstamps are + * non-empty for a static link (see {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setLinkstamps(ImmutableMap<Artifact, Artifact> linkstamps) { + this.linkstamps = linkstamps; + return this; + } + + /** + * Adds the given C++ compiler options to the list of options passed to the linkstamp + * compilation. + */ + public Builder addLinkstampCompileOptions(List<String> linkstampCompileOptions) { + this.linkstampCompileOptions.addAll(linkstampCompileOptions); + return this; + } + + /** + * The build info header artifacts are generated header files that are used for link stamping. + * The {@link #build} method throws an exception if the build info header artifacts are + * non-empty for a static link (see {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setBuildInfoHeaderArtifacts(ImmutableList<Artifact> buildInfoHeaderArtifacts) { + this.buildInfoHeaderArtifacts = buildInfoHeaderArtifacts; + return this; + } + + /** + * Sets the features enabled for the rule. + */ + public Builder setFeatures(ImmutableSet<String> features) { + this.features = features; + return this; + } + + /** + * Sets the directory of the dynamic runtime libraries, which is added to the rpath. The {@link + * #build} method throws an exception if the runtime dir is non-null for a static link (see + * {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setRuntimeSolibDir(PathFragment runtimeSolibDir) { + this.runtimeSolibDir = runtimeSolibDir; + return this; + } + + /** + * Whether the resulting library is intended to be used as a native library from another + * programming language. This influences the rpath. The {@link #build} method throws an + * exception if this is true for a static link (see {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setNativeDeps(boolean nativeDeps) { + this.nativeDeps = nativeDeps; + return this; + } + + /** + * Sets whether to use test-specific linker flags, e.g. {@code $EXEC_ORIGIN} instead of + * {@code $ORIGIN} in the rpath or lazy binding. + */ + public Builder setUseTestOnlyFlags(boolean useTestOnlyFlags) { + this.useTestOnlyFlags = useTestOnlyFlags; + return this; + } + + public Builder setNeedWholeArchive(boolean needWholeArchive) { + this.needWholeArchive = needWholeArchive; + return this; + } + + public Builder setSupportsParamFiles(boolean supportsParamFiles) { + this.supportsParamFiles = supportsParamFiles; + return this; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkStrategy.java new file mode 100644 index 0000000..4f7673e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkStrategy.java
@@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +/** + * A strategy for executing {@link CppLinkAction}s. + * + * <p>The linker commands, e.g. "ar", are not necessary functional, i.e. + * they may mutate the output file rather than overwriting it. + * To avoid this, we need to delete the output file before invoking the + * command. That must be done by the classes that extend this class. + */ +public abstract class LinkStrategy implements CppLinkActionContext { + public LinkStrategy() { + } + + /** The strategy name, preferably suitable for passing to --link_strategy. */ + public abstract String linkStrategyName(); + + @Override + public String strategyLocality(CppLinkAction execOwner) { + return linkStrategyName(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInput.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInput.java new file mode 100644 index 0000000..15a8b90 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInput.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; + +/** + * Something that appears on the command line of the linker. Since we sometimes expand archive + * files to their constituent object files, we need to keep information whether a certain file + * contains embedded objects and if so, the list of the object files themselves. + */ +public interface LinkerInput { + /** + * Returns the artifact that is the input of the linker. + */ + Artifact getArtifact(); + + /** + * Returns the original library to link. If this library is a solib symlink, returns the + * artifact the symlink points to, otherwise, the library itself. + */ + Artifact getOriginalLibraryArtifact(); + + /** + * Whether the input artifact contains object files or is opaque. + */ + boolean containsObjectFiles(); + + /** + * Returns whether the input artifact is a fake object file or not. + */ + boolean isFake(); + + /** + * Return the list of object files included in the input artifact, if there are any. It is + * legal to call this only when {@link #containsObjectFiles()} returns true. + */ + Iterable<Artifact> getObjectFiles(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInputs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInputs.java new file mode 100644 index 0000000..24120ce --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInputs.java
@@ -0,0 +1,353 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.concurrent.ThreadSafety; + +/** + * Factory for creating new {@link LinkerInput} objects. + */ +public abstract class LinkerInputs { + /** + * An opaque linker input that is not a library, for example a linker script or an individual + * object file. + */ + @ThreadSafety.Immutable + public static class SimpleLinkerInput implements LinkerInput { + private final Artifact artifact; + + public SimpleLinkerInput(Artifact artifact) { + this.artifact = Preconditions.checkNotNull(artifact); + } + + @Override + public Artifact getArtifact() { + return artifact; + } + + @Override + public Artifact getOriginalLibraryArtifact() { + return artifact; + } + + @Override + public boolean containsObjectFiles() { + return false; + } + + @Override + public boolean isFake() { + return false; + } + + @Override + public Iterable<Artifact> getObjectFiles() { + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof SimpleLinkerInput)) { + return false; + } + + SimpleLinkerInput other = (SimpleLinkerInput) that; + return artifact.equals(other.artifact) && isFake() == other.isFake(); + } + + @Override + public int hashCode() { + return artifact.hashCode(); + } + + @Override + public String toString() { + return "SimpleLinkerInput(" + artifact.toString() + ")"; + } + } + + /** + * A linker input that is a fake object file generated by cc_fake_binary. The contained + * artifact must be an object file. + */ + @ThreadSafety.Immutable + private static class FakeLinkerInput extends SimpleLinkerInput { + private FakeLinkerInput(Artifact artifact) { + super(artifact); + Preconditions.checkState(Link.OBJECT_FILETYPES.matches(artifact.getFilename())); + } + + @Override + public boolean isFake() { + return true; + } + } + + /** + * A library the user can link to. This is different from a simple linker input in that it also + * has a library identifier. + */ + public interface LibraryToLink extends LinkerInput { + /** + * Returns whether the library is a solib symlink. + */ + boolean isSolibSymlink(); + } + + /** + * This class represents a solib library symlink. Its library identifier is inherited from + * the library that it links to. + */ + @ThreadSafety.Immutable + public static class SolibLibraryToLink implements LibraryToLink { + private final Artifact solibSymlinkArtifact; + private final Artifact libraryArtifact; + + private SolibLibraryToLink(Artifact solibSymlinkArtifact, Artifact libraryArtifact) { + this.solibSymlinkArtifact = Preconditions.checkNotNull(solibSymlinkArtifact); + this.libraryArtifact = libraryArtifact; + } + + @Override + public String toString() { + return String.format("SolibLibraryToLink(%s -> %s", + solibSymlinkArtifact.toString(), libraryArtifact.toString()); + } + + @Override + public Artifact getArtifact() { + return solibSymlinkArtifact; + } + + @Override + public boolean containsObjectFiles() { + return false; + } + + @Override + public boolean isFake() { + return false; + } + + @Override + public Iterable<Artifact> getObjectFiles() { + throw new IllegalStateException(); + } + + @Override + public Artifact getOriginalLibraryArtifact() { + return libraryArtifact; + } + + @Override + public boolean isSolibSymlink() { + return true; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof SolibLibraryToLink)) { + return false; + } + + SolibLibraryToLink thatSolib = (SolibLibraryToLink) that; + return + solibSymlinkArtifact.equals(thatSolib.solibSymlinkArtifact) && + libraryArtifact.equals(thatSolib.libraryArtifact); + } + + @Override + public int hashCode() { + return solibSymlinkArtifact.hashCode(); + } + } + + /** + * This class represents a library that may contain object files. + */ + @ThreadSafety.Immutable + private static class CompoundLibraryToLink implements LibraryToLink { + private final Artifact libraryArtifact; + private final Iterable<Artifact> objectFiles; + + private CompoundLibraryToLink(Artifact libraryArtifact, Iterable<Artifact> objectFiles) { + this.libraryArtifact = Preconditions.checkNotNull(libraryArtifact); + this.objectFiles = objectFiles == null ? null : CollectionUtils.makeImmutable(objectFiles); + } + + @Override + public String toString() { + return String.format("CompoundLibraryToLink(%s)", libraryArtifact.toString()); + } + + @Override + public Artifact getArtifact() { + return libraryArtifact; + } + + @Override + public Artifact getOriginalLibraryArtifact() { + return libraryArtifact; + } + + @Override + public boolean containsObjectFiles() { + return objectFiles != null; + } + + @Override + public boolean isFake() { + return false; + } + + @Override + public Iterable<Artifact> getObjectFiles() { + Preconditions.checkNotNull(objectFiles); + return objectFiles; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof CompoundLibraryToLink)) { + return false; + } + + return libraryArtifact.equals(((CompoundLibraryToLink) that).libraryArtifact); + } + + @Override + public int hashCode() { + return libraryArtifact.hashCode(); + } + + @Override + public boolean isSolibSymlink() { + return false; + } + } + + ////////////////////////////////////////////////////////////////////////////////////// + // Public factory constructors: + ////////////////////////////////////////////////////////////////////////////////////// + + /** + * Creates linker input objects for non-library files. + */ + public static Iterable<LinkerInput> simpleLinkerInputs(Iterable<Artifact> input) { + return Iterables.transform(input, new Function<Artifact, LinkerInput>() { + @Override + public LinkerInput apply(Artifact artifact) { + return simpleLinkerInput(artifact); + } + }); + } + + /** + * Creates a linker input for which we do not know what objects files it consists of. + */ + public static LinkerInput simpleLinkerInput(Artifact artifact) { + // This precondition check was in place and *most* of the tests passed with them; the only + // exception is when you mention a generated .a file in the srcs of a cc_* rule. + // Preconditions.checkArgument(!ARCHIVE_LIBRARY_FILETYPES.contains(artifact.getFileType())); + return new SimpleLinkerInput(artifact); + } + + /** + * Creates a fake linker input. The artifact must be an object file. + */ + public static LinkerInput fakeLinkerInput(Artifact artifact) { + return new FakeLinkerInput(artifact); + } + + /** + * Creates input libraries for which we do not know what objects files it consists of. + */ + public static Iterable<LibraryToLink> opaqueLibrariesToLink(Iterable<Artifact> input) { + return Iterables.transform(input, new Function<Artifact, LibraryToLink>() { + @Override + public LibraryToLink apply(Artifact artifact) { + return opaqueLibraryToLink(artifact); + } + }); + } + + /** + * Creates a solib library symlink from the given artifact. + */ + public static LibraryToLink solibLibraryToLink(Artifact solibSymlink, Artifact original) { + return new SolibLibraryToLink(solibSymlink, original); + } + + /** + * Creates an input library for which we do not know what objects files it consists of. + */ + public static LibraryToLink opaqueLibraryToLink(Artifact artifact) { + // This precondition check was in place and *most* of the tests passed with them; the only + // exception is when you mention a generated .a file in the srcs of a cc_* rule. + // It was very useful for proving that this actually works, though. + // Preconditions.checkArgument( + // !(artifact.getGeneratingAction() instanceof CppLinkAction) || + // !Link.ARCHIVE_LIBRARY_FILETYPES.contains(artifact.getFileType())); + return new CompoundLibraryToLink(artifact, null); + } + + /** + * Creates a library to link with the specified object files. + */ + public static LibraryToLink newInputLibrary(Artifact library, Iterable<Artifact> objectFiles) { + return new CompoundLibraryToLink(library, objectFiles); + } + + private static final Function<LibraryToLink, Artifact> LIBRARY_TO_NON_SOLIB = + new Function<LibraryToLink, Artifact>() { + @Override + public Artifact apply(LibraryToLink input) { + return input.getOriginalLibraryArtifact(); + } + }; + + public static Iterable<Artifact> toNonSolibArtifacts(Iterable<LibraryToLink> libraries) { + return Iterables.transform(libraries, LIBRARY_TO_NON_SOLIB); + } + + /** + * Returns the linker input artifacts from a collection of {@link LinkerInput} objects. + */ + public static Iterable<Artifact> toLibraryArtifacts(Iterable<? extends LinkerInput> artifacts) { + return Iterables.transform(artifacts, new Function<LinkerInput, Artifact>() { + @Override + public Artifact apply(LinkerInput input) { + return input.getArtifact(); + } + }); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkingMode.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkingMode.java new file mode 100644 index 0000000..8018108 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkingMode.java
@@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +/** + * This class represents the different linking modes. + */ +public enum LinkingMode { + + /** + * Everything is linked statically; e.g. {@code gcc -static x.o libfoo.a + * libbar.a -lm}. Specified by {@code -static} in linkopts. + */ + FULLY_STATIC, + + /** + * Link binaries statically except for system libraries + * e.g. {@code gcc x.o libfoo.a libbar.a -lm}. Specified by {@code linkstatic=1}. + * + * <p>This mode applies to executables. + */ + MOSTLY_STATIC, + + /** + * Same as MOSTLY_STATIC, but for shared libraries. + */ + MOSTLY_STATIC_LIBRARIES, + + /** + * All libraries are linked dynamically (if a dynamic version is available), + * e.g. {@code gcc x.o libfoo.so libbar.so -lm}. Specified by {@code + * linkstatic=0}. + */ + DYNAMIC; +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LipoContextProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LipoContextProvider.java new file mode 100644 index 0000000..a9ffea8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LipoContextProvider.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.Map; + +/** + * Provides LIPO context information to the LIPO-enabled target configuration. + * + * <p>This is a rollup of the data collected in the LIPO context collector configuration. + * Each target in the LIPO context collector configuration has a {@link TransitiveLipoInfoProvider} + * which is used to transitively collect the data, then the {@code cc_binary} that is referred to + * in {@code --lipo_context} puts the collected data into {@link LipoContextProvider}, of which + * there is only one in any given build. + */ +@Immutable +public final class LipoContextProvider implements TransitiveInfoProvider { + + private final CppCompilationContext cppCompilationContext; + + private final ImmutableMap<Artifact, IncludeScannable> includeScannables; + public LipoContextProvider(CppCompilationContext cppCompilationContext, + Map<Artifact, IncludeScannable> scannables) { + this.cppCompilationContext = cppCompilationContext; + this.includeScannables = ImmutableMap.copyOf(scannables); + } + + /** + * Returns merged compilation context for the whole LIPO subtree. + */ + public CppCompilationContext getLipoContext() { + return cppCompilationContext; + } + + /** + * Returns the map from source artifact to the include scannable object representing + * the corresponding FDO source input file. + */ + public ImmutableMap<Artifact, IncludeScannable> getIncludeScannables() { + return includeScannables; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalGccStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalGccStrategy.java new file mode 100644 index 0000000..80ee23d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalGccStrategy.java
@@ -0,0 +1,96 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.common.options.OptionsClassProvider; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * Run gcc locally by delegating to spawn. + */ +@ExecutionStrategy(name = { "local" }, + contextType = CppCompileActionContext.class) +public class LocalGccStrategy implements CppCompileActionContext { + private static final Reply CANNED_REPLY = new Reply() { + @Override + public byte[] getContents() { + throw new IllegalStateException("Remotely computed data requested for local action"); + } + }; + + public LocalGccStrategy(OptionsClassProvider options) { + } + + @Override + public String strategyLocality() { + return "local"; + } + + public static void updateEnv(CppCompileAction action, Map<String, String> env) { + // We cannot locally execute an action that does not expect to output a .d file, since we would + // have no way to tell what files that it included were used during compilation. + env.put("INTERCEPT_LOCALLY_EXECUTABLE", action.getDotdFile().artifact() == null ? "0" : "1"); + } + + @Override + public boolean needsIncludeScanning() { + return false; + } + + @Override + public Collection<? extends ActionInput> findAdditionalInputs(CppCompileAction action, + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException { + return ImmutableList.of(); + } + + @Override + public CppCompileActionContext.Reply execWithReply( + CppCompileAction action, ActionExecutionContext actionExecutionContext) + throws ExecException, InterruptedException { + Map<String, String> env = new HashMap<>(); + env.putAll(action.getEnvironment()); + updateEnv(action, env); + actionExecutionContext.getExecutor().getSpawnActionContext(action.getMnemonic()) + .exec(new BaseSpawn.Local(action.getArgv(), env, action), + actionExecutionContext); + return null; + } + + @Override + public ResourceSet estimateResourceConsumption(CppCompileAction action) { + return action.estimateResourceConsumptionLocal(); + } + + @Override + public Collection<Artifact> getScannedIncludeFiles( + CppCompileAction action, ActionExecutionContext actionExecutionContext) { + return ImmutableList.of(); + } + + @Override + public Reply getReplyFromException(ExecException e, CppCompileAction action) { + return CANNED_REPLY; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalLinkStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalLinkStrategy.java new file mode 100644 index 0000000..3e7c863 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalLinkStrategy.java
@@ -0,0 +1,62 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; + +import java.util.List; + +/** + * A link strategy that runs the linking step on the local host. + * + * <p>The set of input files necessary to successfully complete the link is the middleman-expanded + * set of the action's dependency inputs (which includes crosstool and libc dependencies, as + * defined by {@link com.google.devtools.build.lib.rules.cpp.CppHelper#getCrosstoolInputsForLink + * CppHelper.getCrosstoolInputsForLink}). + */ +@ExecutionStrategy(contextType = CppLinkActionContext.class, name = { "local" }) +public final class LocalLinkStrategy extends LinkStrategy { + + public LocalLinkStrategy() { + } + + @Override + public void exec(CppLinkAction action, ActionExecutionContext actionExecutionContext) + throws ExecException, ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + List<String> argv = + action.prepareCommandLine(executor.getExecRoot(), null); + executor.getSpawnActionContext(action.getMnemonic()).exec( + new BaseSpawn.Local(argv, ImmutableMap.<String, String>of(), action), + actionExecutionContext); + } + + @Override + public String linkStrategyName() { + return "local"; + } + + @Override + public ResourceSet estimateResourceConsumption(CppLinkAction action) { + return action.estimateResourceConsumptionLocal(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/RemoteIncludeExtractor.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/RemoteIncludeExtractor.java new file mode 100644 index 0000000..87a0712 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/RemoteIncludeExtractor.java
@@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.rules.cpp.IncludeParser.Inclusion; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; +import java.util.Collection; + +/** Parses a single file for its (direct) includes, possibly using a remote service. */ +public interface RemoteIncludeExtractor extends ActionContext { + /** Result of checking if this object should be used to parse a given file. */ + interface RemoteParseData { + boolean shouldParseRemotely(); + } + + /** + * Returns whether to use this object to parse the given file for includes. The returned data + * should be passed to {@link #extractInclusions} to direct its behavior. + */ + RemoteParseData shouldParseRemotely(Path file); + + /** + * Extracts all inclusions from a given source file, possibly using a remote service. + * + * @param file the file from which to parse and extract inclusions. + * @param actionExecutionContext services in the scope of the action. Like the Err/Out stream + * outputs. + * @param remoteParseData the returned value of {@link #shouldParseRemotely}. + * @return a collection of inclusions, normalized to the cache + */ + public Collection<Inclusion> extractInclusions(Artifact file, + ActionExecutionContext actionExecutionContext, RemoteParseData remoteParseData) + throws IOException, InterruptedException; + +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/SolibSymlinkAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/SolibSymlinkAction.java new file mode 100644 index 0000000..120ba86 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/SolibSymlinkAction.java
@@ -0,0 +1,234 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Actions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; + +/** + * Creates mangled symlinks in the solib directory for all shared libraries. + * Libraries that have a potential to contain SONAME field rely on the mangled + * symlink to the parent directory instead. + * + * Such symlinks are used by the linker to ensure that all rpath entries can be + * specified relative to the $ORIGIN. + */ +public final class SolibSymlinkAction extends AbstractAction { + + private final Artifact library; + private final Path target; + private final Artifact symlink; + + private SolibSymlinkAction(ActionOwner owner, Artifact library, Artifact symlink) { + super(owner, ImmutableList.of(library), ImmutableList.of(symlink)); + + Preconditions.checkArgument(Link.SHARED_LIBRARY_FILETYPES.matches(library.getFilename())); + this.library = Preconditions.checkNotNull(library); + this.symlink = Preconditions.checkNotNull(symlink); + this.target = library.getPath(); + } + + @Override + protected void deleteOutputs(Path execRoot) throws IOException { + // Do not delete outputs if action does not intend to do anything. + if (target != null) { + super.deleteOutputs(execRoot); + } + } + + @Override + public void execute( + ActionExecutionContext actionExecutionContext) throws ActionExecutionException { + Path mangledPath = symlink.getPath(); + try { + FileSystemUtils.createDirectoryAndParents(mangledPath.getParentDirectory()); + mangledPath.createSymbolicLink(target); + } catch (IOException e) { + throw new ActionExecutionException("failed to create _solib symbolic link '" + + symlink.prettyPrint() + "' to target '" + target + "'", e, this, false); + } + } + + @Override + public Artifact getPrimaryInput() { + return library; + } + + @Override + public Artifact getPrimaryOutput() { + return symlink; + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return new ResourceSet(/*memoryMb=*/0, /*cpuUsage=*/0, /*ioUsage=*/0.0); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addPath(symlink.getPath()); + if (target != null) { + f.addPath(target); + } + return f.hexDigestAndReset(); + } + + @Override + public String getMnemonic() { return "SolibSymlink"; } + + @Override + public String describeStrategy(Executor executor) { + return "local"; + } + + @Override + protected String getRawProgressMessage() { return null; } + + /** + * Replaces shared library artifact with mangled symlink and creates related + * symlink action. For artifacts that should retain filename (e.g. libraries + * with SONAME tag), link is created to the parent directory instead. + * + * This action is performed to minimize number of -rpath entries used during + * linking process (by essentially "collecting" as many shared libraries as + * possible in the single directory), since we will be paying quadratic price + * for each additional entry on the -rpath. + * + * @param ruleContext rule context, that requested symlink. + * @param library Shared library artifact that needs to be mangled. + * @param preserveName whether to preserve the name of the library + * @param prefixConsumer whether to prefix the output artifact name with the label of the + * consumer + * @return mangled symlink artifact. + */ + public static LibraryToLink getDynamicLibrarySymlink(final RuleContext ruleContext, + final Artifact library, + boolean preserveName, + boolean prefixConsumer, + BuildConfiguration configuration) { + PathFragment mangledName = getMangledName( + ruleContext, library.getRootRelativePath(), preserveName, prefixConsumer, + configuration.getFragment(CppConfiguration.class)); + return getDynamicLibrarySymlinkInternal(ruleContext, library, mangledName, configuration); + } + + /** + * Version of {@link #getDynamicLibrarySymlink} for the special case of C++ runtime libraries. + * These are handled differently than other libraries: neither their names nor directories are + * mangled, i.e. libstdc++.so.6 is symlinked from _solib_[arch]/libstdc++.so.6 + */ + public static LibraryToLink getCppRuntimeSymlink(RuleContext ruleContext, Artifact library, + String solibDirOverride, BuildConfiguration configuration) { + PathFragment solibDir = new PathFragment(solibDirOverride != null + ? solibDirOverride + : configuration.getFragment(CppConfiguration.class).getSolibDirectory()); + PathFragment symlinkName = solibDir.getRelative(library.getRootRelativePath().getBaseName()); + return getDynamicLibrarySymlinkInternal(ruleContext, library, symlinkName, configuration); + } + + /** + * Internal implementation that takes a pre-determined symlink name; supports both the + * generic {@link #getDynamicLibrarySymlink} and the specialized {@link #getCppRuntimeSymlink}. + */ + private static LibraryToLink getDynamicLibrarySymlinkInternal(RuleContext ruleContext, + Artifact library, PathFragment symlinkName, BuildConfiguration configuration) { + Preconditions.checkArgument(Link.SHARED_LIBRARY_FILETYPES.matches(library.getFilename())); + Preconditions.checkArgument(!library.getRootRelativePath().getSegment(0).startsWith("_solib_")); + + // Ignore libraries that are already represented by the symlinks. + Root root = configuration.getBinDirectory(); + Artifact symlink = ruleContext.getAnalysisEnvironment().getDerivedArtifact(symlinkName, root); + ruleContext.registerAction( + new SolibSymlinkAction(ruleContext.getActionOwner(), library, symlink)); + return LinkerInputs.solibLibraryToLink(symlink, library); + } + + /** + * Returns the name of the symlink that will be created for a library, given + * its name. + * + * @param ruleContext rule context that requests symlink + * @param libraryPath the root-relative path of the library + * @param preserveName true if filename should be preserved + * @param prefixConsumer true if the result should be prefixed with the label of the consumer + * @returns root relative path name + */ + public static PathFragment getMangledName(RuleContext ruleContext, + PathFragment libraryPath, + boolean preserveName, + boolean prefixConsumer, + CppConfiguration cppConfiguration) { + String escapedRulePath = Actions.escapedPath( + "_" + ruleContext.getLabel()); + String soname = getDynamicLibrarySoname(libraryPath, preserveName); + PathFragment solibDir = new PathFragment(cppConfiguration.getSolibDirectory()); + if (preserveName) { + String escapedLibraryPath = + Actions.escapedPath("_" + libraryPath.getParentDirectory().getPathString()); + PathFragment mangledDir = solibDir.getRelative(prefixConsumer + ? escapedRulePath + "__" + escapedLibraryPath + : escapedLibraryPath); + return mangledDir.getRelative(soname); + } else { + return solibDir.getRelative(prefixConsumer + ? escapedRulePath + "__" + soname + : soname); + } + } + + /** + * Compute the SONAME to use for a dynamic library. This name is basically the + * name of the shared library in its final symlinked location. + * + * @param libraryPath name of the shared library that needs to be mangled + * @param preserveName true if filename should be preserved, false - mangled + * @return soname to embed in the dynamic library + */ + public static String getDynamicLibrarySoname(PathFragment libraryPath, + boolean preserveName) { + String mangledName; + if (preserveName) { + mangledName = libraryPath.getBaseName(); + } else { + mangledName = "lib" + Actions.escapedPath(libraryPath.getPathString()); + } + return mangledName; + } + + @Override + public boolean shouldReportPathPrefixConflict(Action action) { + return false; // Always ignore path prefix conflict for the SolibSymlinkAction. + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/TransitiveLipoInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/TransitiveLipoInfoProvider.java new file mode 100644 index 0000000..4094124 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/TransitiveLipoInfoProvider.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A target that can contribute profiling information to LIPO C++ compilations. + * + * <p>This is used in the LIPO context collector tree to collect data from the transitive + * closure of the :lipo_context_collector target. It is eventually passed to the configured + * targets in the target configuration through {@link LipoContextProvider}. + */ +@Immutable +public final class TransitiveLipoInfoProvider implements TransitiveInfoProvider { + public static final TransitiveLipoInfoProvider EMPTY = + new TransitiveLipoInfoProvider( + NestedSetBuilder.<IncludeScannable>emptySet(Order.STABLE_ORDER)); + + private final NestedSet<IncludeScannable> includeScannables; + + public TransitiveLipoInfoProvider(NestedSet<IncludeScannable> includeScannables) { + this.includeScannables = includeScannables; + } + + /** + * Returns the include scannables in the transitive closure. + * + * <p>This is used for constructing the path fragment -> include scannable map in the + * LIPO-enabled target configuration. + */ + public NestedSet<IncludeScannable> getTransitiveIncludeScannables() { + return includeScannables; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/WriteBuildInfoHeaderAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/WriteBuildInfoHeaderAction.java new file mode 100644 index 0000000..58b3330 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/WriteBuildInfoHeaderAction.java
@@ -0,0 +1,194 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.analysis.BuildInfoHelper; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * An action that creates a C++ header containing the build information in the + * form of #define directives. + */ +public final class WriteBuildInfoHeaderAction extends AbstractFileWriteAction { + private static final String GUID = "b0798174-1352-4a54-854a-9785aaea491b"; + + private final ImmutableList<Artifact> valueArtifacts; + + private final boolean writeVolatileInfo; + private final boolean writeStableInfo; + + /** + * Creates an action that writes a C++ header with the build information. + * + * <p>It reads the set of build info keys from an action context that is usually contributed + * to Bazel by the workspace status module, and the value associated with said keys from the + * workspace status files (stable and volatile) written by the workspace status action. + * + * <p>Without input artifacts this action uses redacted build information. + * @param inputs Artifacts that contain build information, or an empty + * collection to use redacted build information + * @param output the C++ header Artifact created by this action + * @param writeVolatileInfo whether to write the volatile part of the build + * information to the generated header + * @param writeStableInfo whether to write the non-volatile part of the + * build information to the generated header + */ + public WriteBuildInfoHeaderAction(Collection<Artifact> inputs, + Artifact output, boolean writeVolatileInfo, boolean writeStableInfo) { + super(BuildInfoHelper.BUILD_INFO_ACTION_OWNER, + inputs, output, /*makeExecutable=*/false); + valueArtifacts = ImmutableList.copyOf(inputs); + if (!inputs.isEmpty()) { + // With non-empty inputs we should not generate both volatile and non-volatile data + // in the same header file. + Preconditions.checkState(writeVolatileInfo ^ writeStableInfo); + } + Preconditions.checkState( + output.isConstantMetadata() == (writeVolatileInfo && !inputs.isEmpty())); + + this.writeVolatileInfo = writeVolatileInfo; + this.writeStableInfo = writeStableInfo; + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) + throws IOException { + WorkspaceStatusAction.Context context = + executor.getContext(WorkspaceStatusAction.Context.class); + + final Map<String, WorkspaceStatusAction.Key> keys = new LinkedHashMap<>(); + if (writeVolatileInfo) { + keys.putAll(context.getVolatileKeys()); + } + + if (writeStableInfo) { + keys.putAll(context.getStableKeys()); + } + + final Map<String, String> values = new LinkedHashMap<>(); + for (Artifact valueFile : valueArtifacts) { + values.putAll(WorkspaceStatusAction.parseValues(valueFile.getPath())); + } + + final boolean redacted = valueArtifacts.isEmpty(); + + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + Writer writer = new OutputStreamWriter(out, UTF_8); + + for (Map.Entry<String, WorkspaceStatusAction.Key> key : keys.entrySet()) { + if (!key.getValue().isInLanguage("C++")) { + continue; + } + + String value = redacted ? key.getValue().getRedactedValue() + : values.containsKey(key.getKey()) ? values.get(key.getKey()) + : key.getValue().getDefaultValue(); + + switch (key.getValue().getType()) { + case VERBATIM: + case INTEGER: + break; + + case STRING: + value = quote(value); + break; + + default: + throw new IllegalStateException(); + } + define(writer, key.getKey(), value); + + } + writer.flush(); + } + }; + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addBoolean(writeStableInfo); + f.addBoolean(writeVolatileInfo); + return f.hexDigestAndReset(); + } + + @Override + public boolean executeUnconditionally() { + // Note: isVolatile must return true if executeUnconditionally can ever return true + // for this instance. + return isUnconditional(); + } + + @Override + public boolean isVolatile() { + return isUnconditional(); + } + + private boolean isUnconditional() { + // Because of special handling in the MetadataHandler, changed volatile build + // information does not trigger relinking of all libraries that have + // linkstamps. But we do want to regenerate the header in case libraries are + // relinked because of other reasons. + // Without inputs the contents of the header do not change, so there is no + // point in executing the action again in that case. + return writeVolatileInfo && !Iterables.isEmpty(getInputs()); + } + + /** + * Quote a string with double quotes. + */ + private String quote(String string) { + // TODO(bazel-team): This is doesn't really work if the string contains quotes. Or a newline. + // Or a backslash. Or anything unusual, really. + return "\"" + string + "\""; + } + + /** + * Write a preprocessor define directive to a Writer. + */ + private void define(Writer writer, String name, String value) throws IOException { + writer.write("#define "); + writer.write(name); + writer.write(' '); + writer.write(value); + writer.write('\n'); + } + + @Override + protected String getRawProgressMessage() { + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ActionListener.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ActionListener.java new file mode 100644 index 0000000..f3b302f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ActionListener.java
@@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.extra; + +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.ImmutableSortedKeyListMultimap; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * Implementation for the 'action_listener' rule. + */ +public final class ActionListener implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + // This rule doesn't produce any output when listed as a build target. + // Only when used via the --experimental_action_listener flag, + // this rule instructs the build system to add additional outputs. + + List<ExtraActionSpec> extraActions; + + Multimap<String, ExtraActionSpec> extraActionMap; + + Set<String> mnemonics = Sets.newHashSet( + ruleContext.attributes().get("mnemonics", Type.STRING_LIST)); + extraActions = retrieveAndValidateExtraActions(ruleContext); + ImmutableSortedKeyListMultimap.Builder<String, ExtraActionSpec> + extraActionMapBuilder = ImmutableSortedKeyListMultimap.builder(); + for (String mnemonic : mnemonics) { + extraActionMapBuilder.putAll(mnemonic, extraActions); + } + extraActionMap = extraActionMapBuilder.build(); + return new RuleConfiguredTargetBuilder(ruleContext) + .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY)) + .add(ExtraActionMapProvider.class, new ExtraActionMapProvider(extraActionMap)) + .build(); + } + + /** + * Loads the targets listed in the 'extra_actions' attribute of this rule. + * Validates these targets to be extra_actions indeed. And checks if the + * blaze version number is in the range of the blaze_version restrictions on the rule. + */ + private List<ExtraActionSpec> retrieveAndValidateExtraActions(RuleContext ruleContext) { + List<ExtraActionSpec> extraActions = new ArrayList<>(); + for (TransitiveInfoCollection prerequisite : + ruleContext.getPrerequisites("extra_actions", Mode.TARGET)) { + ExtraActionSpec spec = prerequisite.getProvider(ExtraActionSpec.class); + if (spec == null) { + ruleContext.attributeError("extra_actions", String.format("target %s is not an " + + "extra_action rule", prerequisite.getLabel().toString())); + } else { + extraActions.add(spec); + } + } + if (extraActions.size() == 0) { + ruleContext.attributeWarning("extra_actions", + "No extra_action is specified for this version of blaze."); + } + return extraActions; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraAction.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraAction.java new file mode 100644 index 0000000..2b53a1f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraAction.java
@@ -0,0 +1,246 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.extra; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.actions.DelegateSpawn; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Action used by extra_action rules to create an action that shadows an existing action. Runs a + * command-line using {@link SpawnActionContext} for executions. + */ +public final class ExtraAction extends SpawnAction { + private final Action shadowedAction; + private final boolean createDummyOutput; + private final Artifact extraActionInfoFile; + private final ImmutableMap<PathFragment, Artifact> runfilesManifests; + private final ImmutableSet<Artifact> extraActionInputs; + private boolean inputsKnown; + + public ExtraAction(ActionOwner owner, + ImmutableSet<Artifact> extraActionInputs, + Map<PathFragment, Artifact> runfilesManifests, + Artifact extraActionInfoFile, + Collection<Artifact> outputs, + Action shadowedAction, + boolean createDummyOutput, + CommandLine argv, + Map<String, String> environment, + String progressMessage, + String mnemonic) { + super(owner, + createInputs(shadowedAction.getInputs(), extraActionInputs), + outputs, + AbstractAction.DEFAULT_RESOURCE_SET, + argv, environment, progressMessage, mnemonic); + this.extraActionInfoFile = extraActionInfoFile; + this.shadowedAction = shadowedAction; + this.runfilesManifests = ImmutableMap.copyOf(runfilesManifests); + this.createDummyOutput = createDummyOutput; + + this.extraActionInputs = extraActionInputs; + inputsKnown = shadowedAction.inputsKnown(); + if (createDummyOutput) { + // extra action file & dummy file + Preconditions.checkArgument(outputs.size() == 2); + } + } + + @Override + public boolean discoversInputs() { + return shadowedAction.discoversInputs(); + } + + @Override + public void discoverInputs(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Preconditions.checkState(discoversInputs(), this); + if (getContext(actionExecutionContext.getExecutor()).isRemotable(getMnemonic(), + isRemotable())) { + // If we're running remotely, we need to update our inputs to take account of any additional + // inputs the shadowed action may need to do its work. + if (shadowedAction.discoversInputs() && shadowedAction instanceof AbstractAction) { + updateInputs( + ((AbstractAction) shadowedAction).getInputFilesForExtraAction(actionExecutionContext)); + } + } + } + + @Override + public boolean inputsKnown() { + return inputsKnown; + } + + private static NestedSet<Artifact> createInputs( + Iterable<Artifact> shadowedActionInputs, ImmutableSet<Artifact> extraActionInputs) { + NestedSetBuilder<Artifact> result = new NestedSetBuilder<>(Order.STABLE_ORDER); + if (shadowedActionInputs instanceof NestedSet) { + result.addTransitive((NestedSet<Artifact>) shadowedActionInputs); + } else { + result.addAll(shadowedActionInputs); + } + return result.addAll(extraActionInputs).build(); + } + + private void updateInputs(Iterable<Artifact> shadowedActionInputs) { + synchronized (this) { + setInputs(createInputs(shadowedActionInputs, extraActionInputs)); + inputsKnown = true; + } + } + + @Override + public void updateInputsFromCache(ArtifactResolver artifactResolver, + Collection<PathFragment> inputPaths) { + // We update the inputs directly from the shadowed action. + Set<PathFragment> extraActionPathFragments = + ImmutableSet.copyOf(Artifact.asPathFragments(extraActionInputs)); + shadowedAction.updateInputsFromCache(artifactResolver, + Collections2.filter(inputPaths, Predicates.in(extraActionPathFragments))); + Preconditions.checkState(shadowedAction.inputsKnown(), "%s %s", this, shadowedAction); + updateInputs(shadowedAction.getInputs()); + } + + /** + * @InheritDoc + * + * This method calls in to {@link AbstractAction#getInputFilesForExtraAction} and + * {@link Action#getExtraActionInfo} of the action being shadowed from the thread executing this + * ExtraAction. It assumes these methods are safe to call from a different thread than the thread + * responsible for the execution of the action being shadowed. + */ + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + // PHASE 1: generate .xa file containing protocol buffer describing + // the action being shadowed + + // We call the getExtraActionInfo command only at execution time + // so actions can store information only known at execution time into the + // protocol buffer. + ExtraActionInfo info = shadowedAction.getExtraActionInfo().build(); + try (OutputStream out = extraActionInfoFile.getPath().getOutputStream()) { + info.writeTo(out); + } catch (IOException e) { + throw new ActionExecutionException(e.getMessage(), e, this, false); + } + Executor executor = actionExecutionContext.getExecutor(); + + // PHASE 2: execution of extra_action. + + if (getContext(executor).isRemotable(getMnemonic(), isRemotable())) { + try { + getContext(executor).exec(getExtraActionSpawn(), actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException(this); + } + } else { + super.execute(actionExecutionContext); + } + + // PHASE 3: create dummy output. + // If the user didn't specify output, we need to create dummy output + // to make blaze schedule this action. + if (createDummyOutput) { + for (Artifact output : getOutputs()) { + try { + FileSystemUtils.touchFile(output.getPath()); + } catch (IOException e) { + throw new ActionExecutionException(e.getMessage(), e, this, false); + } + } + } + synchronized (this) { + inputsKnown = true; + } + } + + /** + * The spawn command for ExtraAction needs to be slightly modified from + * regular SpawnActions: + * -the extraActionInfo file needs to be added to the list of inputs. + * -the extraActionInfo file that is an output file of this task is created + * before the SpawnAction so should not be listed as one of its outputs. + */ + // TODO(bazel-team): Add more tests that execute this code path! + private Spawn getExtraActionSpawn() { + final Spawn base = super.getSpawn(); + return new DelegateSpawn(base) { + @Override public Iterable<? extends ActionInput> getInputFiles() { + return Iterables.concat(base.getInputFiles(), ImmutableSet.of(extraActionInfoFile)); + } + + @Override public List<? extends ActionInput> getOutputFiles() { + return Lists.newArrayList( + Iterables.filter(getOutputs(), new Predicate<Artifact>() { + @Override + public boolean apply(Artifact item) { + return item != extraActionInfoFile; + } + })); + } + + @Override public ImmutableMap<PathFragment, Artifact> getRunfilesManifests() { + ImmutableMap.Builder<PathFragment, Artifact> builder = ImmutableMap.builder(); + builder.putAll(super.getRunfilesManifests()); + builder.putAll(runfilesManifests); + return builder.build(); + } + + @Override public String getMnemonic() { return ExtraAction.this.getMnemonic(); } + }; + } + + /** + * Returns the action this extra action is 'shadowing'. + */ + public Action getShadowedAction() { + return shadowedAction; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionFactory.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionFactory.java new file mode 100644 index 0000000..8040ee0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionFactory.java
@@ -0,0 +1,91 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.extra; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.CommandHelper; +import com.google.devtools.build.lib.analysis.ConfigurationMakeVariableContext; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.MakeVariableExpander; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.List; + +/** + * Factory for 'extra_action'. + */ +public final class ExtraActionFactory implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext context) { + // This rule doesn't produce any output when listed as a build target. + // Only when used via the --experimental_action_listener flag, + // this rule instructs the build system to add additional outputs. + List<Artifact> resolvedData = Lists.newArrayList(); + + Iterable<FilesToRunProvider> tools = + context.getPrerequisites("tools", Mode.HOST, FilesToRunProvider.class); + CommandHelper commandHelper = new CommandHelper( + context, tools, ImmutableMap.<Label, Iterable<Artifact>>of()); + + resolvedData.addAll(context.getPrerequisiteArtifacts("data", Mode.DATA).list()); + List<String>outputTemplates = + context.attributes().get("out_templates", Type.STRING_LIST); + + String command = commandHelper.resolveCommandAndExpandLabels(false, true); + // This is a bit of a hack. We want to run the MakeVariableExpander first, so we expand $ on + // variables that are expanded below with $$, which gets reverted to $ by the + // MakeVariableExpander. This allows us to expand package-specific make variables in the + // package where the extra action is defined, and then later replace the owner-specific make + // variables when the extra action is instantiated. + command = command.replace("$(EXTRA_ACTION_FILE)", "$$(EXTRA_ACTION_FILE)"); + command = command.replace("$(ACTION_ID)", "$$(ACTION_ID)"); + command = command.replace("$(OWNER_LABEL_DIGEST)", "$$(OWNER_LABEL_DIGEST)"); + command = command.replace("$(output ", "$$(output "); + try { + command = MakeVariableExpander.expand( + command, new ConfigurationMakeVariableContext( + context.getTarget().getPackage(), context.getConfiguration())); + } catch (MakeVariableExpander.ExpansionException e) { + context.ruleError(String.format("Unable to expand make variables: %s", + e.getMessage())); + } + + boolean requiresActionOutput = + context.attributes().get("requires_action_output", Type.BOOLEAN); + + ExtraActionSpec spec = new ExtraActionSpec( + commandHelper.getResolvedTools(), + commandHelper.getRemoteRunfileManifestMap(), + resolvedData, + outputTemplates, + command, + context.getLabel(), + requiresActionOutput); + + return new RuleConfiguredTargetBuilder(context) + .addProvider(ExtraActionSpec.class, spec) + .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY)) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionMapProvider.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionMapProvider.java new file mode 100644 index 0000000..ffeebe0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionMapProvider.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.extra; + +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Provides an action type -> set of extra actions to run map. + */ +@Immutable +public final class ExtraActionMapProvider implements TransitiveInfoProvider { + private final ImmutableMultimap<String, ExtraActionSpec> extraActionMap; + + public ExtraActionMapProvider(Multimap<String, ExtraActionSpec> extraActionMap) { + this.extraActionMap = ImmutableMultimap.copyOf(extraActionMap); + } + + /** + * Returns the extra action map. + */ + public ImmutableMultimap<String, ExtraActionSpec> getExtraActionMap() { + return extraActionMap; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionSpec.java b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionSpec.java new file mode 100644 index 0000000..40a063e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/extra/ExtraActionSpec.java
@@ -0,0 +1,220 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.extra; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.CommandHelper; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; + +/** + * The specification for a particular extra action type. + */ +@Immutable +public final class ExtraActionSpec implements TransitiveInfoProvider { + private final ImmutableList<Artifact> resolvedTools; + private final ImmutableMap<PathFragment, Artifact> manifests; + private final ImmutableList<Artifact> resolvedData; + private final ImmutableList<String> outputTemplates; + private final String command; + private final boolean requiresActionOutput; + private final Label label; + + ExtraActionSpec( + Iterable<Artifact> resolvedTools, + Map<PathFragment, Artifact> manifests, + Iterable<Artifact> resolvedData, + Iterable<String> outputTemplates, + String command, + Label label, + boolean requiresActionOutput) { + this.resolvedTools = ImmutableList.copyOf(resolvedTools); + this.manifests = ImmutableMap.copyOf(manifests); + this.resolvedData = ImmutableList.copyOf(resolvedData); + this.outputTemplates = ImmutableList.copyOf(outputTemplates); + this.command = command; + this.label = label; + this.requiresActionOutput = requiresActionOutput; + } + + public Label getLabel() { + return label; + } + + /** + * Adds an extra_action to the action graph based on the action to shadow. + */ + public Collection<Artifact> addExtraAction(RuleContext owningRule, + Action actionToShadow) { + Collection<Artifact> extraActionOutputs = new LinkedHashSet<>(); + ImmutableSet.Builder<Artifact> extraActionInputs = ImmutableSet.builder(); + + ActionOwner owner = actionToShadow.getOwner(); + Label ownerLabel = owner.getLabel(); + if (requiresActionOutput) { + extraActionInputs.addAll(actionToShadow.getOutputs()); + } + extraActionInputs.addAll(resolvedTools); + extraActionInputs.addAll(resolvedData); + + boolean createDummyOutput = false; + + for (String outputTemplate : outputTemplates) { + // We create output for the extra_action based on the 'out_template' attribute. + // See {link #getExtraActionOutputArtifact} for supported variables. + extraActionOutputs.add(getExtraActionOutputArtifact(owningRule, actionToShadow, + owner, outputTemplate)); + } + // extra_action has no output, we need to create some dummy output to keep the build up-to-date. + if (extraActionOutputs.size() == 0) { + createDummyOutput = true; + extraActionOutputs.add(getExtraActionOutputArtifact(owningRule, actionToShadow, + owner, "$(ACTION_ID).dummy")); + } + + // We generate a file containing a protocol buffer describing the action that is being shadowed. + // It is up to each action being shadowed to decide what contents to store here. + Artifact extraActionInfoFile = getExtraActionOutputArtifact(owningRule, actionToShadow, + owner, "$(ACTION_ID).xa"); + extraActionOutputs.add(extraActionInfoFile); + + // Expand extra_action specific variables from the provided command-line. + // See {@link #createExpandedCommand} for list of supported variables. + String command = createExpandedCommand(owningRule, actionToShadow, owner, extraActionInfoFile); + + Map<String, String> env = owningRule.getConfiguration().getDefaultShellEnvironment(); + + List<String> argv = CommandHelper.buildCommandLine(owningRule, + command, extraActionInputs, ".extra_action_script.sh"); + + String commandMessage = String.format("Executing extra_action %s on %s", label, ownerLabel); + owningRule.registerAction(new ExtraAction( + actionToShadow.getOwner(), + extraActionInputs.build(), + manifests, + extraActionInfoFile, + extraActionOutputs, + actionToShadow, + createDummyOutput, + CommandLine.of(argv, false), + env, + commandMessage, + label.getName())); + + return extraActionOutputs; + } + + /** + * Expand extra_action specific variables: + * $(EXTRA_ACTION_FILE): expands to a path of the file containing a protocol buffer + * describing the action being shadowed. + * $(output <out_template>): expands the output template to the execPath of the file. + * e.g. $(output $(ACTION_ID).out) -> + * <build_path>/extra_actions/bar/baz/devtools/build/test_A41234.out + */ + private String createExpandedCommand(RuleContext owningRule, + Action action, ActionOwner owner, Artifact extraActionInfoFile) { + String realCommand = command.replace( + "$(EXTRA_ACTION_FILE)", extraActionInfoFile.getExecPathString()); + + for (String outputTemplate : outputTemplates) { + String outFile = getExtraActionOutputArtifact(owningRule, action, owner, outputTemplate) + .getExecPathString(); + realCommand = realCommand.replace("$(output " + outputTemplate + ")", outFile); + } + return realCommand; + } + + /** + * Creates an output artifact for the extra_action based on the output_template. + * The path will be in the following form: + * <output dir>/<target-configuration-specific-path>/extra_actions/<extra_action_label>/ + + * <configured_target_label>/<expanded_template> + * + * The template can use the following variables: + * $(ACTION_ID): a unique id for the extra_action. + * + * Sample: + * extra_action: foo/bar:extra + * template: $(ACTION_ID).analysis + * target: foo/bar:main + * expands to: output/configuration/extra_actions/\ + * foo/bar/extra/foo/bar/4683026f7ac1dd1a873ccc8c3d764132.analysis + */ + private Artifact getExtraActionOutputArtifact(RuleContext owningRule, Action action, + ActionOwner owner, String template) { + String actionId = getActionId(owner, action); + + template = template.replace("$(ACTION_ID)", actionId); + template = template.replace("$(OWNER_LABEL_DIGEST)", getOwnerDigest(owner)); + + PathFragment rootRelativePath = getRootRelativePath(template, owner); + return owningRule.getAnalysisEnvironment().getDerivedArtifact(rootRelativePath, + owningRule.getConfiguration().getOutputDirectory()); + } + + private PathFragment getRootRelativePath(String template, ActionOwner owner) { + PathFragment extraActionPackageFragment = label.getPackageFragment(); + PathFragment extraActionPrefix = extraActionPackageFragment.getRelative(label.getName()); + + PathFragment ownerFragment = owner.getLabel().getPackageFragment(); + return new PathFragment("extra_actions").getRelative(extraActionPrefix) + .getRelative(ownerFragment).getRelative(template); + } + + /** + * Calculates a digest representing the owner label. We use the digest instead of the + * original value as the original value might lead to a filename that is too long. + * By using a digest, tools can deterministically find all extra_action outputs for a given + * target, without having to open every file in the package. + */ + private static String getOwnerDigest(ActionOwner owner) { + Fingerprint f = new Fingerprint(); + f.addString(owner.getLabel().toString()); + return f.hexDigestAndReset(); + } + + /** + * Creates a unique id for the action shadowed by this extra_action. + * + * We need to have a unique id for the extra_action to use. We build this + * from the owner's label and the shadowed action id (which is only + * guaranteed to be unique per target). Together with the subfolder + * matching the original target's package name, we believe this is enough + * of a uniqueness guarantee. + */ + @VisibleForTesting + public static String getActionId(ActionOwner owner, Action action) { + Fingerprint f = new Fingerprint(); + f.addString(owner.getLabel().toString()); + f.addString(action.getKey()); + return f.hexDigestAndReset(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/filegroup/Filegroup.java b/src/main/java/com/google/devtools/build/lib/rules/filegroup/Filegroup.java new file mode 100644 index 0000000..cb297d8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/filegroup/Filegroup.java
@@ -0,0 +1,103 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.filegroup; + +import com.google.devtools.build.lib.actions.Actions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.CompilationHelper; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.MiddlemanProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProviderImpl; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Iterator; + +/** + * ConfiguredTarget for "filegroup". + */ +public class Filegroup implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + NestedSet<Artifact> filesToBuild = NestedSetBuilder.wrap(Order.STABLE_ORDER, + ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list()); + NestedSet<Artifact> middleman = CompilationHelper.getAggregatingMiddleman( + ruleContext, Actions.escapeLabel(ruleContext.getLabel()), filesToBuild); + + InstrumentedFilesCollector instrumentedFilesCollector = + new InstrumentedFilesCollector(ruleContext, + // what do *we* know about whether this is a source file or not + new InstrumentationSpec(FileTypeSet.ANY_FILE, "srcs", "deps", "data"), + InstrumentedFilesCollector.NO_METADATA_COLLECTOR, filesToBuild); + + RunfilesProvider runfilesProvider = RunfilesProvider.withData( + new Runfiles.Builder() + .addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES) + .build(), + // If you're visiting a filegroup as data, then we also visit its data as data. + new Runfiles.Builder().addTransitiveArtifacts(filesToBuild) + .addDataDeps(ruleContext).build()); + + return new RuleConfiguredTargetBuilder(ruleContext) + .add(RunfilesProvider.class, runfilesProvider) + .setFilesToBuild(filesToBuild) + .setRunfilesSupport(null, getExecutable(filesToBuild)) + .add(InstrumentedFilesProvider.class, new InstrumentedFilesProviderImpl( + instrumentedFilesCollector)) + .add(MiddlemanProvider.class, new MiddlemanProvider(middleman)) + .add(FilegroupPathProvider.class, + new FilegroupPathProvider(getFilegroupPath(ruleContext))) + .build(); + } + + /* + * Returns the single executable output of this filegroup. Returns + * {@code null} if there are multiple outputs or the single output is not + * considered an executable. + */ + private Artifact getExecutable(NestedSet<Artifact> filesToBuild) { + Iterator<Artifact> it = filesToBuild.iterator(); + if (it.hasNext()) { + Artifact out = it.next(); + if (!it.hasNext()) { + return out; + } + } + return null; + } + + private PathFragment getFilegroupPath(RuleContext ruleContext) { + String attr = ruleContext.attributes().get("path", Type.STRING); + if (attr.isEmpty()) { + return PathFragment.EMPTY_FRAGMENT; + } else { + return ruleContext.getLabel().getPackageFragment().getRelative(attr); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/filegroup/FilegroupPathProvider.java b/src/main/java/com/google/devtools/build/lib/rules/filegroup/FilegroupPathProvider.java new file mode 100644 index 0000000..370be07 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/filegroup/FilegroupPathProvider.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.filegroup; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * A transitive info provider for dependent targets to query {@code path} attributes. + */ +@Immutable +public final class FilegroupPathProvider implements TransitiveInfoProvider { + private final PathFragment pathFragment; + + public FilegroupPathProvider(PathFragment pathFragment) { + this.pathFragment = pathFragment; + } + + /** + * Returns the value of the {@code path} attribute or the empty fragment if it is not present. + */ + public PathFragment getFilegroupPath() { + return pathFragment; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContext.java new file mode 100644 index 0000000..056b61e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContext.java
@@ -0,0 +1,34 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.fileset; + +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +import java.util.concurrent.ThreadPoolExecutor; + +/** + * Action context for fileset collection actions. + */ +public interface FilesetActionContext extends ActionContext { + + /** + * Returns a thread pool for fileset symlink tree creation. + */ + ThreadPoolExecutor getFilesetPool(); + + /** + * Returns the name of the workspace the build is run in. + */ + String getWorkspaceName(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContextImpl.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContextImpl.java new file mode 100644 index 0000000..9c03129 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetActionContextImpl.java
@@ -0,0 +1,101 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.fileset; + +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.devtools.build.lib.actions.ActionContextProvider; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BlazeExecutor; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.events.Reporter; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Context for Fileset manifest actions. It currently only provides a ThreadPoolExecutor. + * + * <p>Fileset is a legacy, google-internal mechanism to make parts of the source tree appear as a + * tree in the output directory. + */ +@ExecutionStrategy(contextType = FilesetActionContext.class) +public final class FilesetActionContextImpl implements FilesetActionContext { + // TODO(bazel-team): it would be nice if this weren't shipped in Bazel at all. + + /** + * Factory class. + */ + public static class Provider implements ActionContextProvider { + private FilesetActionContextImpl impl; + private final Reporter reporter; + private final ThreadPoolExecutor filesetPool; + + public Provider(Reporter reporter, String workspaceName) { + this.reporter = reporter; + this.filesetPool = newFilesetPool(100); + this.impl = new FilesetActionContextImpl(filesetPool, workspaceName); + } + + private static ThreadPoolExecutor newFilesetPool(int threads) { + ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 3L, TimeUnit.SECONDS, + new LinkedBlockingQueue<Runnable>()); + // Do not consume threads when not in use. + pool.allowCoreThreadTimeOut(true); + pool.setThreadFactory(new ThreadFactoryBuilder().setNameFormat("Fileset worker %d").build()); + return pool; + } + + @Override + public Iterable<ActionContext> getActionContexts() { + return ImmutableList.<ActionContext>of(impl); + } + + @Override + public void executorCreated(Iterable<ActionContext> usedStrategies) {} + + @Override + public void executionPhaseStarting( + ActionInputFileCache actionInputFileCache, + ActionGraph actionGraph, + Iterable<Artifact> topLevelArtifacts) {} + + @Override + public void executionPhaseEnding() { + BlazeExecutor.shutdownHelperPool(reporter, filesetPool, "Fileset"); + } + } + + private final ThreadPoolExecutor filesetPool; + private final String workspaceName; + + private FilesetActionContextImpl(ThreadPoolExecutor filesetPool, String workspaceName) { + this.filesetPool = filesetPool; + this.workspaceName = workspaceName; + } + + @Override + public ThreadPoolExecutor getFilesetPool() { + return filesetPool; + } + + @Override + public String getWorkspaceName() { + return workspaceName; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetLinks.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetLinks.java new file mode 100644 index 0000000..d523edc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetLinks.java
@@ -0,0 +1,218 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.fileset; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.syntax.FilesetEntry; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * FilesetLinks manages the set of links added to a Fileset. If two links conflict, the first wins. + * + * <p>FilesetLinks is FileSystem-aware. For example, if you first create a link + * (a/b/c, foo), a subsequent call to link (a/b, bar) is a no-op. + * This is because the first link requires us to create a directory "a/b", + * so "a/b" cannot also link to "bar". + * + * <p>TODO(bazel-team): Consider warning if we have such a conflict; we don't do that currently. + */ +public interface FilesetLinks { + + /** + * Get late directory information for a source. + * + * @param src The source to search for. + * @return The late directory info, or null if none was found. + */ + public LateDirectoryInfo getLateDirectoryInfo(PathFragment src); + + public boolean putLateDirectoryInfo(PathFragment src, LateDirectoryInfo lateDir); + + /** + * Add specified file as a symlink. + * + * The behavior when the target file is a symlink depends on the + * symlinkBehavior parameter (see comments for FilesetEntry.SymlinkBehavior). + * + * @param src The root-relative symlink path. + * @param target The symlink target. + */ + public void addFile(PathFragment src, Path target, String metadata, + FilesetEntry.SymlinkBehavior symlinkBehavior) + throws IOException; + + /** + * Add all late directories as symlinks. This function should be called only + * after all recursions have completed, but before getData or getSymlinks are + * called. + */ + public void addLateDirectories() throws IOException; + + /** + * Adds the given symlink to the tree. + * + * @param fromFrag The root-relative symlink path. + * @param toFrag The symlink target. + * @return true iff the symlink was added. + */ + public boolean addLink(PathFragment fromFrag, PathFragment toFrag, String dataVal); + + /** + * @return The unmodifiable map of symlinks. + */ + public Map<PathFragment, PathFragment> getSymlinks(); + + /** + * @return The unmodifiable map of metadata. + */ + public Map<PathFragment, String> getData(); + + /** + * A data structure for containing all the information about a directory that + * is late-added. This means the directory is skipped unless we need to + * recurse into it later. If the directory is never recursed into, we will + * create a symlink directly to it. + */ + public static final class LateDirectoryInfo { + // The constructors are private. Use the factory functions below to create + // instances of this class. + + /** Construct a stub LateDirectoryInfo object. */ + private LateDirectoryInfo() { + this.added = new AtomicBoolean(true); + + // Shut up the compiler. + this.target = null; + this.src = null; + this.pkgMode = SubpackageMode.IGNORE; + this.metadata = null; + this.symlinkBehavior = null; + } + + /** Construct a normal LateDirectoryInfo object. */ + private LateDirectoryInfo(Path target, PathFragment src, SubpackageMode pkgMode, + String metadata, FilesetEntry.SymlinkBehavior symlinkBehavior) { + this.target = target; + this.src = src; + this.pkgMode = pkgMode; + this.metadata = metadata; + this.symlinkBehavior = symlinkBehavior; + this.added = new AtomicBoolean(false); + } + + /** @return The target path for the symlink. The target is the referent. */ + public Path getTarget() { + return target; + } + + /** + * @return The source path for the symlink. The source is the place the + * symlink will be written. */ + public PathFragment getSrc() { + return src; + } + + /** + * @return Whether we should show a warning if we cross a package boundary + * when recursing into this directory. + */ + public SubpackageMode getPkgMode() { + return pkgMode; + } + + /** + * @return The metadata we will write into the manifest if we symlink to + * this directory. + */ + public String getMetadata() { + return metadata; + } + + /** + * @return How to perform the symlinking if the source happens to be a + * symlink itself. + */ + public FilesetEntry.SymlinkBehavior getTargetSymlinkBehavior() { + return Preconditions.checkNotNull(symlinkBehavior, + "should not call this method on stub instances"); + } + + /** + * Atomically checks if the late directory has been added to the manifest + * and marks it as added. If this function returns true, it is the + * responsibility of the caller to recurse into the late directory. + * Otherwise, some other caller has already, or is in the process of + * recursing into it. + * @return Whether the caller should recurse into the late directory. + */ + public boolean shouldAdd() { + return !added.getAndSet(true); + } + + /** + * Create a stub LateDirectoryInfo that is already marked as added. + * @return The new LateDirectoryInfo object. + */ + public static LateDirectoryInfo createStub() { + return new LateDirectoryInfo(); + } + + /** + * Create a LateDirectoryInfo object with the specified attributes. + * @param target The directory to which the symlinks will refer. + * @param src The location at which to create the symlink. + * @param pkgMode How to handle recursion into another package. + * @param metadata The metadata for the directory to write into the + * manifest if we symlink it directly. + * @return The new LateDirectoryInfo object. + */ + public static LateDirectoryInfo create(Path target, PathFragment src, SubpackageMode pkgMode, + String metadata, FilesetEntry.SymlinkBehavior symlinkBehavior) { + return new LateDirectoryInfo(target, src, pkgMode, metadata, symlinkBehavior); + } + + /** + * The target directory to which the symlink will point. + * Note this is a real path on the filesystem and can't be compared to src + * or any source (key) in the links map. + */ + private final Path target; + + /** The referent of the symlink. */ + private final PathFragment src; + + /** Whether to show cross package boundary warnings / errors. */ + private final SubpackageMode pkgMode; + + /** The metadata to write into the manifest file. */ + private final String metadata; + + /** How to perform the symlinking if the source happens to be a symlink itself. */ + private final FilesetEntry.SymlinkBehavior symlinkBehavior; + + /** Whether the directory has already been recursed into. */ + private final AtomicBoolean added; + } + + /** How to handle filesets that cross subpackages. */ + public static enum SubpackageMode { + ERROR, WARNING, IGNORE; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetProvider.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetProvider.java new file mode 100644 index 0000000..6b70aab --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/FilesetProvider.java
@@ -0,0 +1,27 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.fileset; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Information needed by a Fileset to do the right thing when it depends on another Fileset. + */ +public interface FilesetProvider extends TransitiveInfoProvider { + Artifact getFilesetInputManifest(); + PathFragment getFilesetLinkDir(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/fileset/SymlinkTraversal.java b/src/main/java/com/google/devtools/build/lib/rules/fileset/SymlinkTraversal.java new file mode 100644 index 0000000..db13bdb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/fileset/SymlinkTraversal.java
@@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.fileset; + +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; + +import java.io.IOException; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * An interface which contains a method to compute a symlink mapping. + */ +public interface SymlinkTraversal { + + /** + * Adds symlinks to the given FilesetLinks. + * + * @throws IOException if a filesystem operation fails. + * @throws InterruptedException if the traversal is interrupted. + */ + void addSymlinks(EventHandler eventHandler, FilesetLinks links, ThreadPoolExecutor filesetPool) + throws IOException, InterruptedException; + + /** + * Add the traversal's fingerprint to the given Fingerprint. + * @param fp the Fingerprint to combine. + */ + void fingerprint(Fingerprint fp); + + /** + * @return true iff this traversal must be executed unconditionally. + */ + boolean executeUnconditionally(); + + /** + * Returns true if it's ever possible that {@link #executeUnconditionally} + * could evaluate to true during the lifetime of this instance, false + * otherwise. + */ + boolean isVolatile(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/BaseJavaCompilationHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/BaseJavaCompilationHelper.java new file mode 100644 index 0000000..69dc41b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/BaseJavaCompilationHelper.java
@@ -0,0 +1,237 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; + +/** + * A helper class for compiling Java targets. This helper does not rely on the + * presence of rule-specific attributes. + */ +public class BaseJavaCompilationHelper { + /** + * Also see DeployArchiveBuilder.SINGLEJAR_MAX_MEMORY. We don't expect that anyone has more + * than ~500,000 files in a source jar, so 256 MB of memory should be plenty. + */ + private static final String SINGLEJAR_MAX_MEMORY = "-Xmx256m"; + + private final RuleContext ruleContext; + + public BaseJavaCompilationHelper(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + /** + * Returns the artifacts required to invoke {@code javahome} relative binary + * in the action. + */ + public static NestedSet<Artifact> getHostJavabaseInputs(RuleContext ruleContext) { + // This must have a different name than above, because the middleman creation uses the rule's + // configuration, although it should use the host configuration. + return AnalysisUtils.getMiddlemanFor(ruleContext, ":host_jdk"); + } + + private static final ImmutableList<String> SOURCE_JAR_COMMAND_LINE_ARGS = ImmutableList.of( + "--compression", + "--normalize", + "--exclude_build_data", + "--warn_duplicate_resources"); + + private CommandLine sourceJarCommandLine(JavaSemantics semantics, Artifact outputJar, + Iterable<Artifact> resources, Iterable<Artifact> resourceJars) { + CustomCommandLine.Builder args = CustomCommandLine.builder(); + args.addExecPath("--output", outputJar); + args.add(SOURCE_JAR_COMMAND_LINE_ARGS); + args.addExecPaths("--sources", resourceJars); + args.add("--resources"); + for (Artifact resource : resources) { + args.addPaths("%s:%s", resource.getExecPath(), + semantics.getJavaResourcePath(resource.getRootRelativePath())); + } + return args.build(); + } + + /** + * Creates an Action that packages files into a Jar file. + * + * @param semantics delegate semantics for java. + * @param resources the resources to put into the Jar. + * @param resourceJars the resource jars to merge into the jar + * @param outputJar the Jar to create + */ + public void createSourceJarAction(JavaSemantics semantics, Collection<Artifact> resources, + Collection<Artifact> resourceJars, Artifact outputJar) { + ruleContext.registerAction(new SpawnAction.Builder() + .addOutput(outputJar) + .addInputs(resources) + .addInputs(resourceJars) + .addTransitiveInputs(JavaCompilationHelper.getHostJavabaseInputs(ruleContext)) + .setJarExecutable( + ruleContext.getHostConfiguration().getFragment(Jvm.class).getJavaExecutable(), + ruleContext.getPrerequisiteArtifact("$singlejar", Mode.HOST), + ImmutableList.of("-client", SINGLEJAR_MAX_MEMORY)) + .setCommandLine(sourceJarCommandLine(semantics, outputJar, resources, resourceJars)) + .useParameterFile(ParameterFileType.SHELL_QUOTED) + .setProgressMessage("Building source jar " + outputJar.prettyPrint()) + .setMnemonic("JavaSourceJar") + .build(ruleContext)); + } + + /** + * Returns the langtools jar Artifact. + */ + protected final Artifact getLangtoolsJar() { + return ruleContext.getHostPrerequisiteArtifact("$java_langtools"); + } + + /** + * Returns the JavaBuilder jar Artifact. + */ + protected final Artifact getJavaBuilderJar() { + return ruleContext.getPrerequisiteArtifact("$javabuilder", Mode.HOST); + } + + /** + * Returns the javac bootclasspath artifacts. + */ + protected final Iterable<Artifact> getBootClasspath() { + return ruleContext.getPrerequisiteArtifacts("$javac_bootclasspath", Mode.HOST).list(); + } + + private Artifact getIjarArtifact(Artifact jar, boolean addPrefix) { + if (addPrefix) { + PathFragment ruleBase = ruleContext.getLabel().getPackageFragment().getRelative( + ruleContext.getLabel().getName()).getRelative("_ijars"); + PathFragment artifactDirFragment = jar.getRootRelativePath().getParentDirectory(); + String ijarBasename = FileSystemUtils.removeExtension(jar.getFilename()) + "-ijar.jar"; + return getAnalysisEnvironment().getDerivedArtifact( + ruleBase.getRelative(artifactDirFragment).getRelative(ijarBasename), + getConfiguration().getGenfilesDirectory()); + } else { + return derivedArtifact(jar, "", "-ijar.jar"); + } + } + + /** + * Creates the Action that creates ijars from Jar files. + * + * @param inputJar the Jar to create the ijar for + * @param addPrefix whether to prefix the path of the generated ijar with the package and + * name of the current rule + * @return the Artifact to create with the Action + */ + protected Artifact createIjarAction(final Artifact inputJar, boolean addPrefix) { + Artifact interfaceJar = getIjarArtifact(inputJar, addPrefix); + final FilesToRunProvider ijarTarget = + ruleContext.getExecutablePrerequisite("$ijar", Mode.HOST); + if (!ruleContext.hasErrors()) { + ruleContext.registerAction(new SpawnAction.Builder() + .addInput(inputJar) + .addOutput(interfaceJar) + .setExecutable(ijarTarget) + .addArgument(inputJar.getExecPathString()) + .addArgument(interfaceJar.getExecPathString()) + .setProgressMessage("Extracting interface " + ruleContext.getLabel()) + .setMnemonic("JavaIjar") + .build(ruleContext)); + } + return interfaceJar; + } + + protected final JavaCompileAction.Builder createJavaCompileActionBuilder( + JavaSemantics semantics) { + JavaCompileAction.Builder builder = new JavaCompileAction.Builder(ruleContext, semantics); + builder.setJavaExecutable( + ruleContext.getHostConfiguration().getFragment(Jvm.class).getJavaExecutable()); + builder.setJavaBaseInputs(BaseJavaCompilationHelper.getHostJavabaseInputs(ruleContext)); + return builder; + } + + public RuleContext getRuleContext() { + return ruleContext; + } + + public AnalysisEnvironment getAnalysisEnvironment() { + return ruleContext.getAnalysisEnvironment(); + } + + protected BuildConfiguration getConfiguration() { + return ruleContext.getConfiguration(); + } + + protected JavaConfiguration getJavaConfiguration() { + return ruleContext.getFragment(JavaConfiguration.class); + } + + protected PathFragment outputDir(Artifact outputJar) { + return workDir(outputJar, "_files"); + } + + /** + * Produces a derived directory where source files generated by annotation processors should be + * stored. + */ + protected PathFragment sourceGenDir(Artifact outputJar) { + return workDir(outputJar, "_sourcegenfiles"); + } + + protected PathFragment tempDir(Artifact outputJar) { + return workDir(outputJar, "_temp"); + } + + /** + * For an output jar and a suffix, produces a derived directory under + * {@code bin} directory with a given suffix. + */ + private PathFragment workDir(Artifact outputJar, String suffix) { + PathFragment path = outputJar.getRootRelativePath(); + String basename = FileSystemUtils.removeExtension(path.getBaseName()) + suffix; + path = path.replaceName(basename); + return getConfiguration().getBinDirectory().getExecPath().getRelative(path); + } + + /** + * Creates a derived artifact from the given artifact by adding the given + * prefix and removing the extension and replacing it by the given suffix. + * The new artifact will have the same root as the given one. + */ + protected Artifact derivedArtifact(Artifact artifact, String prefix, String suffix) { + return derivedArtifact(artifact, prefix, suffix, artifact.getRoot()); + } + + protected Artifact derivedArtifact(Artifact artifact, String prefix, String suffix, Root root) { + PathFragment path = artifact.getRootRelativePath(); + String basename = FileSystemUtils.removeExtension(path.getBaseName()) + suffix; + path = path.replaceName(prefix + basename); + return getAnalysisEnvironment().getDerivedArtifact(path, root); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/BuildInfoPropertiesTranslator.java b/src/main/java/com/google/devtools/build/lib/rules/java/BuildInfoPropertiesTranslator.java new file mode 100644 index 0000000..053b9e7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/BuildInfoPropertiesTranslator.java
@@ -0,0 +1,33 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import java.util.Map; +import java.util.Properties; + +/** + * A class to describe how build information should be translated into the generated properties + * file. + */ +public interface BuildInfoPropertiesTranslator { + + /** Translate build information into a property file. */ + public void translate(Map<String, String> buildInfo, Properties properties); + + /** + * Returns a unique key for this translator to be used by the + * {@link com.google.devtools.build.lib.actions.Action#getKey()} method + */ + public String computeKey(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/ClasspathConfiguredFragment.java b/src/main/java/com/google/devtools/build/lib/rules/java/ClasspathConfiguredFragment.java new file mode 100644 index 0000000..6510a49 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/ClasspathConfiguredFragment.java
@@ -0,0 +1,96 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; + +/** + * Represents common aspects of all JVM targeting configured targets. + */ +public final class ClasspathConfiguredFragment { + + private final NestedSet<Artifact> runtimeClasspath; + private final NestedSet<Artifact> compileTimeClasspath; + private final ImmutableList<Artifact> bootClasspath; + + /** + * Initializes the runtime and compile time classpaths for this target. This method + * should be called during {@code initializationHook()} once a {@link JavaTargetAttributes} + * object for this target is fully initialized. + * + * @param attributes the processed attributes of this Java target + * @param isNeverLink whether to leave runtimeClasspath empty + */ + public ClasspathConfiguredFragment(JavaCompilationArtifacts javaArtifacts, + JavaTargetAttributes attributes, boolean isNeverLink) { + if (!isNeverLink) { + runtimeClasspath = getRuntimeClasspathList(attributes, javaArtifacts); + } else { + runtimeClasspath = NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); + } + compileTimeClasspath = attributes.getCompileTimeClassPath(); + bootClasspath = attributes.getBootClassPath(); + } + + public ClasspathConfiguredFragment() { + runtimeClasspath = NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); + compileTimeClasspath = NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); + bootClasspath = ImmutableList.of(); + } + + /** + * Returns the runtime class path. It consists of the concatenation of the + * instrumentation class path, output jars and the runtime time class path of + * the transitive dependencies of this rule. + * + * @param attributes the processed attributes of this Java target + * + * @return a {@List} of artifacts that comprise the runtime class path. + */ + private NestedSet<Artifact> getRuntimeClasspathList( + JavaTargetAttributes attributes, JavaCompilationArtifacts javaArtifacts) { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.naiveLinkOrder(); + builder.addAll(javaArtifacts.getRuntimeJars()); + builder.addTransitive(attributes.getRuntimeClassPath()); + return builder.build(); + } + + /** + * Returns the classpath to be passed to the JVM when running a target containing this fragment. + */ + public NestedSet<Artifact> getRuntimeClasspath() { + return runtimeClasspath; + } + + /** + * Returns the classpath to be passed to the Java compiler when compiling a target containing this + * fragment. + */ + public NestedSet<Artifact> getCompileTimeClasspath() { + return compileTimeClasspath; + } + + /** + * Returns the classpath to be passed as a boot classpath to the Java compiler when compiling + * a target containing this fragment. + */ + public ImmutableList<Artifact> getBootClasspath() { + return bootClasspath; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/DeployArchiveBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/java/DeployArchiveBuilder.java new file mode 100644 index 0000000..b9fe186 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/DeployArchiveBuilder.java
@@ -0,0 +1,256 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.IterablesChain; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Utility for configuring an action to generate a deploy archive. + */ +public class DeployArchiveBuilder { + /** + * Memory consumption of SingleJar is about 250 bytes per entry in the output file. Unfortunately, + * the JVM tends to kill the process with an OOM long before we're at the limit. In the most + * recent example, 400 MB of memory was enough for about 500,000 entries. + */ + private static final String SINGLEJAR_MAX_MEMORY = "-Xmx1600m"; + + private final RuleContext ruleContext; + + private final IterablesChain.Builder<Artifact> runtimeJarsBuilder = IterablesChain.builder(); + + private final JavaSemantics semantics; + + private JavaTargetAttributes attributes; + private boolean includeBuildData; + private Compression compression = Compression.UNCOMPRESSED; + @Nullable private Artifact runfilesMiddleman; + private Artifact outputJar; + @Nullable private String javaStartClass; + private ImmutableList<String> deployManifestLines = ImmutableList.of(); + @Nullable private Artifact launcher; + + /** + * Type of compression to apply to output archive. + */ + public enum Compression { + + /** Output should be compressed */ + COMPRESSED, + + /** Output should not be compressed */ + UNCOMPRESSED; + } + + /** + * Creates a builder using the configuration of the rule as the action configuration. + */ + public DeployArchiveBuilder(JavaSemantics semantics, RuleContext ruleContext) { + this.ruleContext = ruleContext; + this.semantics = semantics; + } + + /** + * Sets the processed attributes of the rule generating the deploy archive. + */ + public DeployArchiveBuilder setAttributes(JavaTargetAttributes attributes) { + this.attributes = attributes; + return this; + } + + /** + * Sets whether to include build-data.properties in the deploy archive. + */ + public DeployArchiveBuilder setIncludeBuildData(boolean includeBuildData) { + this.includeBuildData = includeBuildData; + return this; + } + + /** + * Sets whether to enable compression of the output deploy archive. + */ + public DeployArchiveBuilder setCompression(Compression compress) { + this.compression = Preconditions.checkNotNull(compress); + return this; + } + + /** + * Sets additional dependencies to be added to the action that creates the + * deploy jar so that we force the runtime dependencies to be built. + */ + public DeployArchiveBuilder setRunfilesMiddleman(@Nullable Artifact runfilesMiddleman) { + this.runfilesMiddleman = runfilesMiddleman; + return this; + } + + /** + * Sets the artifact to create with the action. + */ + public DeployArchiveBuilder setOutputJar(Artifact outputJar) { + this.outputJar = Preconditions.checkNotNull(outputJar); + return this; + } + + /** + * Sets the class to launch the Java application. + */ + public DeployArchiveBuilder setJavaStartClass(@Nullable String javaStartClass) { + this.javaStartClass = javaStartClass; + return this; + } + + /** + * Adds additional jars that should be on the classpath at runtime. + */ + public DeployArchiveBuilder addRuntimeJars(Iterable<Artifact> jars) { + this.runtimeJarsBuilder.add(jars); + return this; + } + + /** + * Sets the list of extra lines to add to the archive's MANIFEST.MF file. + */ + public DeployArchiveBuilder setDeployManifestLines(Iterable<String> deployManifestLines) { + this.deployManifestLines = ImmutableList.copyOf(deployManifestLines); + return this; + } + + /** + * Sets the optional launcher to be used as the executable for this deploy + * JAR + */ + public DeployArchiveBuilder setLauncher(@Nullable Artifact launcher) { + this.launcher = launcher; + return this; + } + + public static CustomCommandLine.Builder defaultSingleJarCommandLine(Artifact outputJar, + String javaMainClass, + ImmutableList<String> deployManifestLines, Iterable<Artifact> buildInfoFiles, + ImmutableList<Artifact> classpathResources, + Iterable<Artifact> runtimeClasspath, boolean includeBuildData, + Compression compress, Artifact launcher) { + + CustomCommandLine.Builder args = CustomCommandLine.builder(); + args.addExecPath("--output", outputJar); + if (compress == Compression.COMPRESSED) { + args.add("--compression"); + } + args.add("--normalize"); + if (javaMainClass != null) { + args.add("--main_class"); + args.add(javaMainClass); + } + + if (!deployManifestLines.isEmpty()) { + args.add("--deploy_manifest_lines"); + args.add(deployManifestLines); + } + + if (buildInfoFiles != null) { + for (Artifact artifact : buildInfoFiles) { + args.addExecPath("--build_info_file", artifact); + } + } + if (!includeBuildData) { + args.add("--exclude_build_data"); + } + if (launcher != null) { + args.add("--java_launcher"); + args.add(launcher.getExecPathString()); + } + + args.addExecPaths("--classpath_resources", classpathResources); + args.addExecPaths("--sources", runtimeClasspath); + return args; + } + + /** + * Builds the action as configured. + */ + public void build() { + ImmutableList<Artifact> classpathResources = attributes.getClassPathResources(); + Set<String> classPathResourceNames = new HashSet<>(); + for (Artifact artifact : classpathResources) { + String name = artifact.getExecPath().getBaseName(); + if (!classPathResourceNames.add(name)) { + ruleContext.attributeError("classpath_resources", + "entries must have different file names (duplicate: " + name + ")"); + return; + } + } + + IterablesChain<Artifact> runtimeJars = runtimeJarsBuilder.build(); + + IterablesChain.Builder<Artifact> inputs = IterablesChain.builder(); + inputs.add(attributes.getArchiveInputs(true)); + + inputs.add(ImmutableList.copyOf(runtimeJars)); + if (runfilesMiddleman != null) { + inputs.addElement(runfilesMiddleman); + } + + final ImmutableList<Artifact> buildInfoArtifacts = + ruleContext.getAnalysisEnvironment().getBuildInfo(ruleContext, JavaBuildInfoFactory.KEY); + inputs.add(buildInfoArtifacts); + + Iterable<Artifact> runtimeClasspath = Iterables.concat( + runtimeJars, + attributes.getRuntimeClassPathForArchive()); + + if (launcher != null) { + inputs.addElement(launcher); + } + + CommandLine commandLine = semantics.buildSingleJarCommandLine(ruleContext.getConfiguration(), + outputJar, javaStartClass, deployManifestLines, buildInfoArtifacts, classpathResources, + runtimeClasspath, includeBuildData, compression, launcher); + + List<String> jvmArgs = ImmutableList.of("-client", SINGLEJAR_MAX_MEMORY); + ResourceSet resourceSet = + new ResourceSet(/*memoryMb = */200.0, /*cpuUsage = */.2, /*ioUsage=*/.2); + + ruleContext.registerAction(new SpawnAction.Builder() + .addInputs(inputs.build()) + .addTransitiveInputs(JavaCompilationHelper.getHostJavabaseInputs(ruleContext)) + .addOutput(outputJar) + .setResources(resourceSet) + .setJarExecutable( + ruleContext.getHostConfiguration().getFragment(Jvm.class).getJavaExecutable(), + ruleContext.getPrerequisiteArtifact("$singlejar", Mode.HOST), + jvmArgs) + .setCommandLine(commandLine) + .useParameterFile(ParameterFileType.SHELL_QUOTED) + .setProgressMessage("Building deploy jar " + outputJar.prettyPrint()) + .setMnemonic("JavaDeployJar") + .build(ruleContext)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/DirectDependencyProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/DirectDependencyProvider.java new file mode 100644 index 0000000..5b2e106 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/DirectDependencyProvider.java
@@ -0,0 +1,64 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; + +/** + * A provider that returns the direct dependencies of a target. Used for strict dependency + * checking. + */ +@Immutable +public final class DirectDependencyProvider implements TransitiveInfoProvider { + + private final ImmutableList<Dependency> strictDependencies; + + public DirectDependencyProvider(Iterable<Dependency> strictDependencies) { + this.strictDependencies = ImmutableList.copyOf(strictDependencies); + } + + /** + * @returns the direct (strict) dependencies of this provider. All symbols that are directly + * reachable from the sources of the provider should be available in one these artifacts. + */ + public Iterable<Dependency> getStrictDependencies() { + return strictDependencies; + } + + /** + * A pair of label and its generated list of artifacts. + */ + public static class Dependency { + private final Label label; + + // TODO(bazel-team): change this to Artifacts + private final Iterable<String> fileExecPaths; + + public Dependency(Label label, Iterable<String> fileExecPaths) { + this.label = label; + this.fileExecPaths = fileExecPaths; + } + + public Label getLabel() { + return label; + } + + public Iterable<String> getDependencyOutputs() { + return fileExecPaths; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/GenericBuildInfoPropertiesTranslator.java b/src/main/java/com/google/devtools/build/lib/rules/java/GenericBuildInfoPropertiesTranslator.java new file mode 100644 index 0000000..df6a325 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/GenericBuildInfoPropertiesTranslator.java
@@ -0,0 +1,91 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.util.Fingerprint; + +import java.util.Map; +import java.util.Properties; + +/** The generic implementation of {@link BuildInfoPropertiesTranslator} */ +public class GenericBuildInfoPropertiesTranslator implements + BuildInfoPropertiesTranslator { + + private static final String GUID = "e71fe4a8-11af-4ec0-9b38-1d3e7f542f51"; + + // syntax is %ID% for a property that depends on the ID key, %ID|default% to + // always add the property with the "default" key, %% is to add a percent sign + private final Map<String, String> translationKeys; + + /** + * Create a generic translator, for each key,value pair in {@code translationKeys}, the key + * represents the property key that will be written and the value, its value. Inside value every + * %ID% is replaced by the corresponding build information with the same ID key. The property + * won't be added if it's depends on an unresolved build information. Adding a property can + * be forced even if a build information is missing by specifying a default value using the + * %ID|default% syntax. Finally to add a percent sign, just use the %% syntax. + */ + public GenericBuildInfoPropertiesTranslator(Map<String, String> translationKeys) { + this.translationKeys = translationKeys; + } + + @Override + public String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addStringMap(translationKeys); + return f.hexDigestAndReset(); + } + + @Override + public void translate(Map<String, String> buildInfo, Properties properties) { + for (Map.Entry<String, String> entry : translationKeys.entrySet()) { + String translatedValue = translateValue(entry.getValue(), buildInfo); + if (translatedValue != null) { + properties.put(entry.getKey(), translatedValue); + } + } + } + + private String translateValue(String valueDescription, Map<String, String> buildInfo) { + String[] split = valueDescription.split("%"); + StringBuffer result = new StringBuffer(); + boolean isInsideKey = false; + for (String key : split) { + if (isInsideKey) { + if (key.isEmpty()) { + result.append("%"); // empty key means %% + } else { + String defaultValue = null; + int i = key.lastIndexOf('|'); + if (i >= 0) { + defaultValue = key.substring(i + 1); + key = key.substring(0, i); + } + if (buildInfo.containsKey(key)) { + result.append(buildInfo.get(key)); + } else if (defaultValue != null) { + result.append(defaultValue); + } else { // we haven't found the requested key so we ignore the whole value + return null; + } + } + } else { + result.append(key); + } + isInsideKey = !isInsideKey; + } + return result.toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java new file mode 100644 index 0000000..e957f49 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java
@@ -0,0 +1,359 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import static com.google.devtools.build.lib.rules.java.DeployArchiveBuilder.Compression.COMPRESSED; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.TopLevelArtifactProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.CppHelper; +import com.google.devtools.build.lib.rules.cpp.LinkerInput; +import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * An implementation of java_binary. + */ +public class JavaBinary implements RuleConfiguredTargetFactory { + private static final PathFragment CPP_RUNTIMES = new PathFragment("_cpp_runtimes"); + + private final JavaSemantics semantics; + + protected JavaBinary(JavaSemantics semantics) { + this.semantics = semantics; + } + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + final JavaCommon common = new JavaCommon(ruleContext, semantics); + DeployArchiveBuilder deployArchiveBuilder = new DeployArchiveBuilder(semantics, ruleContext); + Runfiles.Builder runfilesBuilder = new Runfiles.Builder(); + List<String> jvmFlags = new ArrayList<>(); + + common.initializeJavacOpts(); + JavaTargetAttributes.Builder attributesBuilder = common.initCommon(); + attributesBuilder.addClassPathResources( + ruleContext.getPrerequisiteArtifacts("classpath_resources", Mode.TARGET).list()); + + List<String> userJvmFlags = common.getJvmFlags(); + + ruleContext.checkSrcsSamePackage(true); + boolean createExecutable = ruleContext.attributes().get("create_executable", Type.BOOLEAN); + List<TransitiveInfoCollection> deps = + Lists.newArrayList(common.targetsTreatedAsDeps(ClasspathType.COMPILE_ONLY)); + semantics.checkRule(ruleContext, common); + String mainClass = semantics.getMainClass(ruleContext, common); + String originalMainClass = mainClass; + if (ruleContext.hasErrors()) { + return null; + } + + // Collect the transitive dependencies. + JavaCompilationHelper helper = new JavaCompilationHelper( + ruleContext, semantics, common.getJavacOpts(), attributesBuilder); + helper.addLibrariesToAttributes(deps); + helper.addProvidersToAttributes(common.compilationArgsFromSources(), /* isNeverLink */ false); + attributesBuilder.addNativeLibraries( + collectNativeLibraries(common.targetsTreatedAsDeps(ClasspathType.BOTH))); + + // deploy_env is valid for java_binary, but not for java_test. + if (ruleContext.getRule().isAttrDefined("deploy_env", Type.LABEL_LIST)) { + for (JavaRuntimeClasspathProvider envTarget : ruleContext.getPrerequisites( + "deploy_env", Mode.TARGET, JavaRuntimeClasspathProvider.class)) { + attributesBuilder.addExcludedArtifacts(envTarget.getRuntimeClasspath()); + } + } + + Artifact srcJar = + ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_SOURCE_JAR); + + Artifact classJar = + ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_CLASS_JAR); + + ImmutableList<Artifact> srcJars = ImmutableList.of(srcJar); + + Artifact launcher = semantics.getLauncher(ruleContext, common, deployArchiveBuilder, + runfilesBuilder, jvmFlags, attributesBuilder); + JavaCompilationArtifacts.Builder javaArtifactsBuilder = new JavaCompilationArtifacts.Builder(); + Artifact instrumentationMetadata = + helper.createInstrumentationMetadata(classJar, javaArtifactsBuilder); + + NestedSetBuilder<Artifact> filesBuilder = NestedSetBuilder.stableOrder(); + Artifact executable = null; + if (createExecutable) { + executable = ruleContext.createOutputArtifact(); // the artifact for the rule itself + filesBuilder.add(classJar).add(executable); + + if (ruleContext.getConfiguration().isCodeCoverageEnabled()) { + mainClass = semantics.addCoverageSupport(helper, attributesBuilder, + executable, instrumentationMetadata, javaArtifactsBuilder, mainClass); + } + } else { + filesBuilder.add(classJar); + } + + JavaTargetAttributes attributes = helper.getAttributes(); + List<Artifact> nativeLibraries = attributes.getNativeLibraries(); + if (!nativeLibraries.isEmpty()) { + jvmFlags.add("-Djava.library.path=" + JavaCommon.javaLibraryPath(nativeLibraries)); + } + + JavaConfiguration javaConfig = ruleContext.getFragment(JavaConfiguration.class); + if (attributes.hasMessages()) { + helper.addTranslations(semantics.translate(ruleContext, javaConfig, + attributes.getMessages())); + } + + if (attributes.hasSourceFiles() || attributes.hasSourceJars() + || attributes.hasResources() || attributes.hasClassPathResources()) { + // We only want to add a jar to the classpath of a dependent rule if it has content. + javaArtifactsBuilder.addRuntimeJar(classJar); + } + + // Any JAR files should be added to the collection of runtime jars. + javaArtifactsBuilder.addRuntimeJars(attributes.getJarFiles()); + + Artifact outputDepsProto = helper.createOutputDepsProtoArtifact(classJar, javaArtifactsBuilder); + + common.setJavaCompilationArtifacts(javaArtifactsBuilder.build()); + + // The gensrcJar is only created if the target uses annotation processing. Otherwise, + // it is null, and the source jar action will not depend on the compile action. + Artifact gensrcJar = helper.createGensrcJar(classJar); + + helper.createCompileAction(classJar, gensrcJar, outputDepsProto, instrumentationMetadata); + helper.createSourceJarAction(srcJar, gensrcJar); + + common.setClassPathFragment(new ClasspathConfiguredFragment( + common.getJavaCompilationArtifacts(), attributes, false)); + + // Collect the action inputs for the runfiles collector here because we need to access the + // analysis environment, and that may no longer be safe when the runfiles collector runs. + Iterable<Artifact> dynamicRuntimeActionInputs = + CppHelper.getToolchain(ruleContext).getDynamicRuntimeLinkInputs(); + + + Iterables.addAll(jvmFlags, semantics.getJvmFlags(ruleContext, common, launcher, userJvmFlags)); + if (ruleContext.hasErrors()) { + return null; + } + + if (createExecutable) { + // Create a shell stub for a Java application + semantics.createStubAction(ruleContext, common, jvmFlags, executable, mainClass, + common.getJavaBinSubstitution(launcher)); + } + + NestedSet<Artifact> transitiveSourceJars = collectTransitiveSourceJars(common, srcJar); + + // TODO(bazel-team): if (getOptions().sourceJars) then make this a dummy prerequisite for the + // DeployArchiveAction ? Needs a few changes there as we can't pass inputs + helper.createSourceJarAction(semantics, ImmutableList.<Artifact>of(), + transitiveSourceJars.toCollection(), + ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_DEPLOY_SOURCE_JAR)); + + RuleConfiguredTargetBuilder builder = + new RuleConfiguredTargetBuilder(ruleContext); + + semantics.addProviders(ruleContext, common, jvmFlags, classJar, srcJar, gensrcJar, + ImmutableMap.<Artifact, Artifact>of(), helper, filesBuilder, builder); + + NestedSet<Artifact> filesToBuild = filesBuilder.build(); + + collectDefaultRunfiles(runfilesBuilder, ruleContext, common, filesToBuild, launcher, + dynamicRuntimeActionInputs); + Runfiles defaultRunfiles = runfilesBuilder.build(); + + RunfilesSupport runfilesSupport = createExecutable + ? runfilesSupport = RunfilesSupport.withExecutable( + ruleContext, defaultRunfiles, executable, + semantics.getExtraArguments(ruleContext, common)) + : null; + + RunfilesProvider runfilesProvider = RunfilesProvider.withData( + defaultRunfiles, + new Runfiles.Builder().merge(runfilesSupport).build()); + + ImmutableList<String> deployManifestLines = + getDeployManifestLines(ruleContext, originalMainClass); + + Artifact deployJar = + ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_DEPLOY_JAR); + + deployArchiveBuilder + .setOutputJar(deployJar) + .setJavaStartClass(mainClass) + .setDeployManifestLines(deployManifestLines) + .setAttributes(attributes) + .addRuntimeJars(common.getJavaCompilationArtifacts().getRuntimeJars()) + .setIncludeBuildData(true) + .setRunfilesMiddleman( + runfilesSupport == null ? null : runfilesSupport.getRunfilesMiddleman()) + .setCompression(COMPRESSED) + .setLauncher(launcher); + + deployArchiveBuilder.build(); + + common.addTransitiveInfoProviders(builder, filesToBuild, classJar); + + return builder + .setFilesToBuild(filesToBuild) + .add(RunfilesProvider.class, runfilesProvider) + .setRunfilesSupport(runfilesSupport, executable) + .add(JavaRuntimeClasspathProvider.class, + new JavaRuntimeClasspathProvider(common.getRuntimeClasspath())) + .add(JavaSourceJarsProvider.class, + new JavaSourceJarsProvider(transitiveSourceJars, srcJars)) + .add(TopLevelArtifactProvider.class, new TopLevelArtifactProvider( + JavaSemantics.SOURCE_JARS_OUTPUT_GROUP, transitiveSourceJars)) + .build(); + } + + // Create the deploy jar and make it dependent on the runfiles middleman if an executable is + // created. Do not add the deploy jar to files to build, so we will only build it when it gets + // requested. + private ImmutableList<String> getDeployManifestLines(RuleContext ruleContext, + String originalMainClass) { + ImmutableList.Builder<String> builder = ImmutableList.<String>builder() + .addAll(ruleContext.attributes().get("deploy_manifest_lines", Type.STRING_LIST)); + if (ruleContext.getConfiguration().isCodeCoverageEnabled()) { + builder.add("Coverage-Main-Class: " + originalMainClass); + } + return builder.build(); + } + + private void collectDefaultRunfiles(Runfiles.Builder builder, RuleContext ruleContext, + JavaCommon common, NestedSet<Artifact> filesToBuild, Artifact launcher, + Iterable<Artifact> dynamicRuntimeActionInputs) { + // Convert to iterable: filesToBuild has a different order. + builder.addArtifacts((Iterable<Artifact>) filesToBuild); + builder.addArtifacts(common.getJavaCompilationArtifacts().getRuntimeJars()); + if (launcher != null) { + final TransitiveInfoCollection defaultLauncher = + JavaHelper.launcherForTarget(semantics, ruleContext); + final Artifact defaultLauncherArtifact = + JavaHelper.launcherArtifactForTarget(semantics, ruleContext); + if (!defaultLauncherArtifact.equals(launcher)) { + builder.addArtifact(launcher); + + // N.B. The "default launcher" referred to here is the launcher target specified through + // an attribute or flag. We wish to retain the runfiles of the default launcher, *except* + // for the original cc_binary artifact, because we've swapped it out with our custom + // launcher. Hence, instead of calling builder.addTarget(), or adding an odd method + // to Runfiles.Builder, we "unravel" the call and manually add things to the builder. + // Because the NestedSet representing each target's launcher runfiles is re-built here, + // we may see increased memory consumption for representing the target's runfiles. + Runfiles runfiles = + defaultLauncher.getProvider(RunfilesProvider.class) + .getDefaultRunfiles(); + NestedSetBuilder<Artifact> unconditionalArtifacts = NestedSetBuilder.compileOrder(); + for (Artifact a : runfiles.getUnconditionalArtifacts()) { + if (!a.equals(defaultLauncherArtifact)) { + unconditionalArtifacts.add(a); + } + } + builder.addTransitiveArtifacts(unconditionalArtifacts.build()); + builder.addSymlinks(runfiles.getSymlinks()); + builder.addRootSymlinks(runfiles.getRootSymlinks()); + builder.addPruningManifests(runfiles.getPruningManifests()); + } else { + builder.addTarget(defaultLauncher, RunfilesProvider.DEFAULT_RUNFILES); + } + } + + semantics.addRunfilesForBinary(ruleContext, launcher, builder); + builder.addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES); + builder.add(ruleContext, JavaRunfilesProvider.TO_RUNFILES); + + List<? extends TransitiveInfoCollection> runtimeDeps = + ruleContext.getPrerequisites("runtime_deps", Mode.TARGET); + builder.addTargets(runtimeDeps, JavaRunfilesProvider.TO_RUNFILES); + builder.addTargets(runtimeDeps, RunfilesProvider.DEFAULT_RUNFILES); + semantics.addDependenciesForRunfiles(ruleContext, builder); + + if (ruleContext.getConfiguration().isCodeCoverageEnabled()) { + Artifact instrumentedJar = common.getJavaCompilationArtifacts().getInstrumentedJar(); + if (instrumentedJar != null) { + builder.addArtifact(instrumentedJar); + } + } + + builder.addArtifacts((Iterable<Artifact>) common.getRuntimeClasspath()); + + // Add the JDK files if it comes from the source repository (see java_stub_template.txt). + TransitiveInfoCollection javabaseTarget = ruleContext.getPrerequisite(":jvm", Mode.HOST); + if (javabaseTarget != null) { + builder.addArtifacts( + (Iterable<Artifact>) javabaseTarget.getProvider(FileProvider.class).getFilesToBuild()); + + // Add symlinks to the C++ runtime libraries under a path that can be built + // into the Java binary without having to embed the crosstool, gcc, and grte + // version information contained within the libraries' package paths. + for (Artifact lib : dynamicRuntimeActionInputs) { + PathFragment path = CPP_RUNTIMES.getRelative(lib.getExecPath().getBaseName()); + builder.addSymlink(path, lib); + } + } + } + + private NestedSet<Artifact> collectTransitiveSourceJars(JavaCommon common, Artifact srcJar) { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + + builder.add(srcJar); + for (JavaSourceJarsProvider dep : common.getDependencies(JavaSourceJarsProvider.class)) { + builder.addTransitive(dep.getTransitiveSourceJars()); + } + return builder.build(); + } + + /** + * Collects the native libraries in the transitive closure of the deps. + * + * @param deps the dependencies to be included as roots of the transitive closure. + * @return the native libraries found in the transitive closure of the deps. + */ + public static Collection<Artifact> collectNativeLibraries( + Iterable<? extends TransitiveInfoCollection> deps) { + NestedSet<LinkerInput> linkerInputs = new NativeLibraryNestedSetBuilder() + .addJavaTargets(deps) + .build(); + ImmutableList.Builder<Artifact> result = ImmutableList.builder(); + for (LinkerInput linkerInput : linkerInputs) { + result.add(linkerInput.getArtifact()); + } + + return result.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaBuildInfoFactory.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBuildInfoFactory.java new file mode 100644 index 0000000..442b85b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBuildInfoFactory.java
@@ -0,0 +1,145 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoCollection; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.rules.java.WriteBuildInfoPropertiesAction.TimestampFormatter; +import com.google.devtools.build.lib.vfs.PathFragment; + +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.util.ArrayList; +import java.util.List; + +/** + * Java build info creation - generates properties file that contain the corresponding build-info + * data. + */ +public abstract class JavaBuildInfoFactory implements BuildInfoFactory { + public static final BuildInfoKey KEY = new BuildInfoKey("Java"); + + static final PathFragment BUILD_INFO_NONVOLATILE_PROPERTIES_NAME = + new PathFragment("build-info-nonvolatile.properties"); + static final PathFragment BUILD_INFO_VOLATILE_PROPERTIES_NAME = + new PathFragment("build-info-volatile.properties"); + static final PathFragment BUILD_INFO_REDACTED_PROPERTIES_NAME = + new PathFragment("build-info-redacted.properties"); + + private static final DateTimeFormatter DEFAULT_TIME_FORMAT = + DateTimeFormat.forPattern("EEE MMM d HH:mm:ss yyyy"); + + // A default formatter that returns a date in UTC format. + private static final TimestampFormatter DEFAULT_FORMATTER = new TimestampFormatter() { + @Override + public String format(long timestamp) { + return new DateTime(timestamp, DateTimeZone.UTC).toString(DEFAULT_TIME_FORMAT) + " (" + + timestamp / 1000 + ')'; + } + }; + + @Override + public final BuildInfoCollection create(BuildInfoContext context, BuildConfiguration config, + Artifact stableStatus, Artifact volatileStatus) { + WriteBuildInfoPropertiesAction redactedInfo = getHeader(context, + config, + BUILD_INFO_REDACTED_PROPERTIES_NAME, + Artifact.NO_ARTIFACTS, + createRedactedTranslator(), + true, + true); + WriteBuildInfoPropertiesAction nonvolatileInfo = getHeader(context, + config, + BUILD_INFO_NONVOLATILE_PROPERTIES_NAME, + ImmutableList.of(stableStatus), + createNonVolatileTranslator(), + false, + true); + WriteBuildInfoPropertiesAction volatileInfo = getHeader(context, + config, + BUILD_INFO_VOLATILE_PROPERTIES_NAME, + ImmutableList.of(volatileStatus), + createVolatileTranslator(), + true, + false); + List<Action> actions = new ArrayList<Action>(3); + actions.add(redactedInfo); + actions.add(nonvolatileInfo); + actions.add(volatileInfo); + return new BuildInfoCollection(actions, + ImmutableList.of(nonvolatileInfo.getPrimaryOutput(), volatileInfo.getPrimaryOutput()), + ImmutableList.of(redactedInfo.getPrimaryOutput())); + } + + /** + * Creates a {@link BuildInfoPropertiesTranslator} to use for volatile keys. + */ + protected abstract BuildInfoPropertiesTranslator createVolatileTranslator(); + + /** + * Creates a {@link BuildInfoPropertiesTranslator} to use for non-volatile keys. + */ + protected abstract BuildInfoPropertiesTranslator createNonVolatileTranslator(); + + /** + * Creates a {@link BuildInfoPropertiesTranslator} to use for redacted version of the build + * informations. + */ + protected abstract BuildInfoPropertiesTranslator createRedactedTranslator(); + + /** + * Specifies the {@link TimestampFormatter} to use to output dates in the properties file. + */ + protected TimestampFormatter getTimestampFormatter() { + return DEFAULT_FORMATTER; + } + + private WriteBuildInfoPropertiesAction getHeader(BuildInfoContext context, + BuildConfiguration config, + PathFragment propertyFileName, + ImmutableList<Artifact> inputs, + BuildInfoPropertiesTranslator translator, + boolean includeVolatile, + boolean includeNonVolatile) { + Root outputPath = config.getIncludeDirectory(); + final Artifact output = context.getBuildInfoArtifact(propertyFileName, outputPath, + includeVolatile && !inputs.isEmpty() ? BuildInfoType.NO_REBUILD + : BuildInfoType.FORCE_REBUILD_IF_CHANGED); + return new WriteBuildInfoPropertiesAction(inputs, + output, + translator, + includeVolatile, + includeNonVolatile, + getTimestampFormatter()); + } + + @Override + public final BuildInfoKey getKey() { + return KEY; + } + + @Override + public boolean isEnabled(BuildConfiguration config) { + return config.hasFragment(JavaConfiguration.class); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCcLinkParamsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCcLinkParamsProvider.java new file mode 100644 index 0000000..5218f3f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCcLinkParamsProvider.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore.CcLinkParamsStoreImpl; + +/** + * A target that provides C++ libraries to be linked into Java targets. + */ +@Immutable +public final class JavaCcLinkParamsProvider implements TransitiveInfoProvider { + private final CcLinkParamsStoreImpl store; + + public JavaCcLinkParamsProvider(CcLinkParamsStore store) { + this.store = new CcLinkParamsStoreImpl(store); + } + + public CcLinkParamsStore getLinkParams() { + return store; + } + + public static final Function<TransitiveInfoCollection, CcLinkParamsStore> TO_LINK_PARAMS = + new Function<TransitiveInfoCollection, CcLinkParamsStore>() { + @Override + public CcLinkParamsStore apply(TransitiveInfoCollection input) { + JavaCcLinkParamsProvider provider = input.getProvider( + JavaCcLinkParamsProvider.class); + return provider == null ? null : provider.getLinkParams(); + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java new file mode 100644 index 0000000..c55a74e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java
@@ -0,0 +1,651 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToCompileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CppCompilationContext; +import com.google.devtools.build.lib.rules.cpp.LinkerInput; +import com.google.devtools.build.lib.rules.java.DirectDependencyProvider.Dependency; +import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType; +import com.google.devtools.build.lib.rules.test.BaselineCoverageAction; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.LocalMetadataCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProviderImpl; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A helper class to create configured targets for Java rules. + */ +public class JavaCommon { + private static final Function<TransitiveInfoCollection, Label> GET_COLLECTION_LABEL = + new Function<TransitiveInfoCollection, Label>() { + @Override + public Label apply(TransitiveInfoCollection collection) { + return collection.getLabel(); + } + }; + + /** + * Collects all metadata files generated by Java compilation actions. + */ + private static final LocalMetadataCollector JAVA_METADATA_COLLECTOR = + new LocalMetadataCollector() { + @Override + public void collectMetadataArtifacts(Iterable<Artifact> objectFiles, + AnalysisEnvironment analysisEnvironment, NestedSetBuilder<Artifact> metadataFilesBuilder) { + for (Artifact artifact : objectFiles) { + Action action = analysisEnvironment.getLocalGeneratingAction(artifact); + if (action instanceof JavaCompileAction) { + addOutputs(metadataFilesBuilder, action, JavaSemantics.COVERAGE_METADATA); + } + } + } + }; + + private ClasspathConfiguredFragment classpathFragment = new ClasspathConfiguredFragment(); + private JavaCompilationArtifacts javaArtifacts = JavaCompilationArtifacts.EMPTY; + private ImmutableList<String> javacOpts; + + // Targets treated as deps in compilation time, runtime time and both + private final ImmutableMap<ClasspathType, ImmutableList<TransitiveInfoCollection>> + targetsTreatedAsDeps; + + private ImmutableList<Artifact> sources = ImmutableList.of(); + private ImmutableList<JavaPluginInfoProvider> activePlugins = ImmutableList.of(); + + private final RuleContext ruleContext; + private final JavaSemantics semantics; + + public JavaCommon(RuleContext ruleContext, JavaSemantics semantics) { + this(ruleContext, semantics, + collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.COMPILE_ONLY), + collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.RUNTIME_ONLY), + collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.BOTH)); + } + + public JavaCommon(RuleContext ruleContext, + JavaSemantics semantics, + ImmutableList<TransitiveInfoCollection> compileDeps, + ImmutableList<TransitiveInfoCollection> runtimeDeps, + ImmutableList<TransitiveInfoCollection> bothDeps) { + this.ruleContext = ruleContext; + this.semantics = semantics; + this.targetsTreatedAsDeps = ImmutableMap.of( + ClasspathType.COMPILE_ONLY, compileDeps, + ClasspathType.RUNTIME_ONLY, runtimeDeps, + ClasspathType.BOTH, bothDeps); + } + + public void setClassPathFragment(ClasspathConfiguredFragment classpathFragment) { + this.classpathFragment = classpathFragment; + } + + public void setJavaCompilationArtifacts(JavaCompilationArtifacts javaArtifacts) { + this.javaArtifacts = javaArtifacts; + } + + public JavaCompilationArtifacts getJavaCompilationArtifacts() { + return javaArtifacts; + } + + public ImmutableList<Artifact> getProcessorClasspathJars() { + Set<Artifact> processorClasspath = new LinkedHashSet<>(); + for (JavaPluginInfoProvider plugin : activePlugins) { + for (Artifact classpathJar : plugin.getProcessorClasspath()) { + processorClasspath.add(classpathJar); + } + } + return ImmutableList.copyOf(processorClasspath); + } + + public ImmutableList<String> getProcessorClassNames() { + Set<String> processorNames = new LinkedHashSet<>(); + for (JavaPluginInfoProvider plugin : activePlugins) { + processorNames.addAll(plugin.getProcessorClasses()); + } + return ImmutableList.copyOf(processorNames); + } + + /** + * Creates the java.library.path from a list of the native libraries. + * Concatenates the parent directories of the shared libraries into a Java + * search path. Each relative path entry is prepended with "${JAVA_RUNFILES}/" + * so it can be resolved at runtime. + * + * @param sharedLibraries a collection of native libraries to create the java + * library path from + * @return a String containing the ":" separated java library path + */ + public static String javaLibraryPath(Collection<Artifact> sharedLibraries) { + StringBuilder buffer = new StringBuilder(); + Set<PathFragment> entries = new HashSet<>(); + for (Artifact sharedLibrary : sharedLibraries) { + PathFragment entry = sharedLibrary.getRootRelativePath().getParentDirectory(); + if (entries.add(entry)) { + if (buffer.length() > 0) { + buffer.append(':'); + } + buffer.append("${JAVA_RUNFILES}/" + Constants.RUNFILES_PREFIX + "/"); + buffer.append(entry.getPathString()); + } + } + return buffer.toString(); + } + + /** + * Collects Java compilation arguments for this target. + * + * @param recursive Whether to scan dependencies recursively. + * @param isNeverLink Whether the target has the 'neverlink' attr. + */ + JavaCompilationArgs collectJavaCompilationArgs(boolean recursive, boolean isNeverLink, + Iterable<SourcesJavaCompilationArgsProvider> compilationArgsFromSources) { + ClasspathType type = isNeverLink ? ClasspathType.COMPILE_ONLY : ClasspathType.BOTH; + JavaCompilationArgs.Builder builder = JavaCompilationArgs.builder() + .merge(getJavaCompilationArtifacts(), isNeverLink) + .addTransitiveTargets(getExports(ruleContext), recursive, type); + if (recursive) { + builder + .addTransitiveTargets(targetsTreatedAsDeps(ClasspathType.COMPILE_ONLY), recursive, type) + .addTransitiveTargets(getRuntimeDeps(ruleContext), recursive, ClasspathType.RUNTIME_ONLY) + .addSourcesTransitiveCompilationArgs(compilationArgsFromSources, recursive, type); + } + return builder.build(); + } + + /** + * Collects Java dependency artifacts for this target. + * + * @param outDeps output (compile-time) dependency artifact of this target + */ + NestedSet<Artifact> collectCompileTimeDependencyArtifacts(Artifact outDeps) { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + if (outDeps != null) { + builder.add(outDeps); + } + + for (JavaCompilationArgsProvider provider : AnalysisUtils.getProviders( + getExports(ruleContext), JavaCompilationArgsProvider.class)) { + builder.addTransitive(provider.getCompileTimeJavaDependencyArtifacts()); + } + return builder.build(); + } + + public static List<TransitiveInfoCollection> getExports(RuleContext ruleContext) { + // We need to check here because there are classes inheriting from this class that implement + // rules that don't have this attribute. + if (ruleContext.getRule().getRuleClassObject().hasAttr("exports", Type.LABEL_LIST)) { + return ImmutableList.copyOf(ruleContext.getPrerequisites("exports", Mode.TARGET)); + } else { + return ImmutableList.of(); + } + } + + /** + * Sanity checks the given runtime dependencies, and emits errors if there is a problem. + * Also called by {@link #initCommon()} for the current target's runtime dependencies. + */ + public void checkRuntimeDeps(List<TransitiveInfoCollection> runtimeDepInfo) { + for (TransitiveInfoCollection c : runtimeDepInfo) { + JavaNeverlinkInfoProvider neverLinkedness = + c.getProvider(JavaNeverlinkInfoProvider.class); + if (neverLinkedness == null) { + continue; + } + boolean reportError = !ruleContext.getConfiguration().getAllowRuntimeDepsOnNeverLink(); + if (neverLinkedness.isNeverlink()) { + String msg = String.format("neverlink dep %s not allowed in runtime deps", c.getLabel()); + if (reportError) { + ruleContext.attributeError("runtime_deps", msg); + } else { + ruleContext.attributeWarning("runtime_deps", msg); + } + } + } + } + + /** + * Returns transitive Java native libraries. + * + * @see JavaNativeLibraryProvider + */ + protected NestedSet<LinkerInput> collectTransitiveJavaNativeLibraries() { + NativeLibraryNestedSetBuilder builder = new NativeLibraryNestedSetBuilder(); + builder.addJavaTargets(targetsTreatedAsDeps(ClasspathType.BOTH)); + + if (ruleContext.getRule().isAttrDefined("data", Type.LABEL_LIST)) { + builder.addJavaTargets(ruleContext.getPrerequisites("data", Mode.DATA)); + } + return builder.build(); + } + + /** + * Collects transitive source jars for the current rule. + * + * @param targetSrcJar The source jar artifact corresponding to the output of the current rule. + * @return A nested set containing all of the source jar artifacts on which the current rule + * transitively depends. + */ + public NestedSet<Artifact> collectTransitiveSourceJars(Artifact targetSrcJar) { + NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder(); + + builder.add(targetSrcJar); + for (JavaSourceJarsProvider dep : getDependencies(JavaSourceJarsProvider.class)) { + builder.addTransitive(dep.getTransitiveSourceJars()); + } + return builder.build(); + } + + /** + * Collects transitive C++ dependencies. + */ + protected CppCompilationContext collectTransitiveCppDeps() { + CppCompilationContext.Builder builder = new CppCompilationContext.Builder(ruleContext); + for (TransitiveInfoCollection dep : targetsTreatedAsDeps(ClasspathType.BOTH)) { + CppCompilationContext context = + dep.getProvider(CppCompilationContext.class); + if (context != null) { + builder.mergeDependentContext(context); + } + } + return builder.build(); + } + + /** + * Collects labels of targets and artifacts reached transitively via the "exports" attribute. + */ + protected NestedSet<Label> collectTransitiveExports() { + NestedSetBuilder<Label> builder = NestedSetBuilder.stableOrder(); + List<TransitiveInfoCollection> currentRuleExports = getExports(ruleContext); + + builder.addAll(Iterables.transform(currentRuleExports, GET_COLLECTION_LABEL)); + + for (TransitiveInfoCollection dep : currentRuleExports) { + JavaExportsProvider exportsProvider = dep.getProvider(JavaExportsProvider.class); + + if (exportsProvider != null) { + builder.addTransitive(exportsProvider.getTransitiveExports()); + } + } + + return builder.build(); + } + + public final void initializeJavacOpts() { + initializeJavacOpts(semantics.getExtraJavacOpts(ruleContext)); + } + + public final void initializeJavacOpts(Iterable<String> extraJavacOpts) { + javacOpts = ImmutableList.copyOf(Iterables.concat( + JavaToolchainProvider.getDefaultJavacOptions(ruleContext), + ruleContext.getTokenizedStringListAttr("javacopts"), extraJavacOpts)); + } + + /** + * Returns the string that the stub should use to determine the JVM + * @param launcher if non-null, the cc_binary used to launch the Java Virtual Machine + */ + public String getJavaBinSubstitution(@Nullable Artifact launcher) { + PathFragment javaExecutable; + + if (launcher != null) { + javaExecutable = launcher.getRootRelativePath(); + } else { + javaExecutable = ruleContext.getFragment(Jvm.class).getJavaExecutable(); + } + + String pathPrefix = + javaExecutable.isAbsolute() ? "" : "${JAVA_RUNFILES}/" + Constants.RUNFILES_PREFIX + "/"; + return "JAVABIN=${JAVABIN:-" + pathPrefix + javaExecutable.getPathString() + "}"; + } + + /** + * Heuristically determines the name of the primary Java class for this + * executable, based on the rule name and the "srcs" list. + * + * <p>(This is expected to be the class containing the "main" method for a + * java_binary, or a JUnit Test class for a java_test.) + * + * @param sourceFiles the source files for this rule + * @return a fully qualified Java class name, or null if none could be + * determined. + */ + public String determinePrimaryClass(Collection<Artifact> sourceFiles) { + if (!sourceFiles.isEmpty()) { + String mainSource = ruleContext.getTarget().getName() + ".java"; + for (Artifact sourceFile : sourceFiles) { + PathFragment path = sourceFile.getRootRelativePath(); + if (path.getBaseName().equals(mainSource)) { + return JavaUtil.getJavaFullClassname(FileSystemUtils.removeExtension(path)); + } + } + } + // Last resort: Use the name and package name of the target. + // TODO(bazel-team): this should be fixed to use a source file from the dependencies to + // determine the package of the Java class. + return JavaUtil.getJavaFullClassname(Util.getWorkspaceRelativePath(ruleContext.getTarget())); + } + + /** + * Gets the value of the "jvm_flags" attribute combining it with the default + * options and expanding any make variables. + */ + public List<String> getJvmFlags() { + List<String> jvmFlags = new ArrayList<>(); + jvmFlags.addAll(ruleContext.getFragment(JavaConfiguration.class).getDefaultJvmFlags()); + jvmFlags.addAll(ruleContext.expandedMakeVariablesList("jvm_flags")); + return jvmFlags; + } + + private static List<TransitiveInfoCollection> getRuntimeDeps(RuleContext ruleContext) { + // We need to check here because there are classes inheriting from this class that implement + // rules that don't have this attribute. + if (ruleContext.getRule().getRuleClassObject().hasAttr("runtime_deps", Type.LABEL_LIST)) { + return ImmutableList.copyOf(ruleContext.getPrerequisites("runtime_deps", Mode.TARGET)); + } else { + return ImmutableList.of(); + } + } + + public JavaTargetAttributes.Builder initCommon() { + return initCommon(Collections.<Artifact>emptySet()); + } + + /** + * Initialize the common actions and build various collections of artifacts + * for the initializationHook() methods of the subclasses. + * + * <p>Note that not all subclasses call this method. + * + * @return the processed attributes + */ + public JavaTargetAttributes.Builder initCommon(Collection<Artifact> extraSrcs) { + Preconditions.checkState(javacOpts != null); + sources = ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list(); + activePlugins = collectPlugins(); + + JavaTargetAttributes.Builder javaTargetAttributes = new JavaTargetAttributes.Builder(semantics); + processSrcs(javaTargetAttributes, javacOpts); + javaTargetAttributes.addSourceArtifacts(extraSrcs); + processRuntimeDeps(javaTargetAttributes); + + semantics.commonDependencyProcessing(ruleContext, javaTargetAttributes, + targetsTreatedAsDeps(ClasspathType.COMPILE_ONLY)); + + // Check that we have do not have both sources and jars. + if ((javaTargetAttributes.hasSourceFiles() || javaTargetAttributes.hasSourceJars()) + && javaTargetAttributes.hasJarFiles()) { + ruleContext.attributeWarning("srcs", "cannot use both Java sources - source " + + "jars or source files - and precompiled jars"); + } + + if (disallowDepsWithoutSrcs(ruleContext.getRule().getRuleClass()) + && ruleContext.attributes().get("srcs", Type.LABEL_LIST).isEmpty() + && ruleContext.getRule().isAttributeValueExplicitlySpecified("deps")) { + ruleContext.attributeError("deps", "deps not allowed without srcs; move to runtime_deps?"); + } + + javaTargetAttributes.addResources(semantics.collectResources(ruleContext)); + addPlugins(javaTargetAttributes); + + javaTargetAttributes.setRuleKind(ruleContext.getRule().getRuleClass()); + javaTargetAttributes.setTargetLabel(ruleContext.getLabel()); + + return javaTargetAttributes; + } + + private boolean disallowDepsWithoutSrcs(String ruleClass) { + return ruleClass.equals("java_library") + || ruleClass.equals("java_binary") + || ruleClass.equals("java_test"); + } + + public ImmutableList<? extends TransitiveInfoCollection> targetsTreatedAsDeps( + ClasspathType type) { + return targetsTreatedAsDeps.get(type); + } + + private static ImmutableList<TransitiveInfoCollection> collectTargetsTreatedAsDeps( + RuleContext ruleContext, JavaSemantics semantics, ClasspathType type) { + ImmutableList.Builder<TransitiveInfoCollection> builder = new Builder<>(); + + if (!type.equals(ClasspathType.COMPILE_ONLY)) { + builder.addAll(getRuntimeDeps(ruleContext)); + builder.addAll(getExports(ruleContext)); + } + builder.addAll(ruleContext.getPrerequisites("deps", Mode.TARGET)); + + semantics.collectTargetsTreatedAsDeps(ruleContext, builder); + + // Implicitly add dependency on java launcher cc_binary when --java_launcher= is enabled, + // or when launcher attribute is specified in a build rule. + TransitiveInfoCollection launcher = JavaHelper.launcherForTarget(semantics, ruleContext); + if (launcher != null) { + builder.add(launcher); + } + + return builder.build(); + } + + public void addTransitiveInfoProviders(RuleConfiguredTargetBuilder builder, + NestedSet<Artifact> filesToBuild, @Nullable Artifact classJar) { + InstrumentedFilesCollector instrumentedFilesCollector = + new InstrumentedFilesCollector(ruleContext, semantics.getCoverageInstrumentationSpec(), + JAVA_METADATA_COLLECTOR, filesToBuild); + + builder + .add(InstrumentedFilesProvider.class, new InstrumentedFilesProviderImpl( + instrumentedFilesCollector)) + .add(FilesToCompileProvider.class, + new FilesToCompileProvider(getFilesToCompile(classJar))) + .add(JavaExportsProvider.class, new JavaExportsProvider(collectTransitiveExports())); + + if (!TargetUtils.isTestRule(ruleContext.getTarget())) { + ImmutableList<Artifact> baselineCoverageArtifacts = + BaselineCoverageAction.getBaselineCoverageArtifacts(ruleContext, + instrumentedFilesCollector.getInstrumentedFiles()); + builder.setBaselineCoverageArtifacts(baselineCoverageArtifacts); + } + } + + /** + * Processes the sources of this target, adding them as messages, proper + * sources or to the list of targets treated as deps as required. + */ + private void processSrcs(JavaTargetAttributes.Builder attributes, + ImmutableList<String> javacOpts) { + for (MessageBundleProvider srcItem : ruleContext.getPrerequisites( + "srcs", Mode.TARGET, MessageBundleProvider.class)) { + attributes.addMessages(srcItem.getMessages()); + } + + attributes.addSourceArtifacts(sources); + + addCompileTimeClassPathEntriesMaybeThroughIjar(attributes, javacOpts); + } + + /** + * Processes the transitive runtime_deps of this target. + */ + private void processRuntimeDeps(JavaTargetAttributes.Builder attributes) { + List<TransitiveInfoCollection> runtimeDepInfo = getRuntimeDeps(ruleContext); + checkRuntimeDeps(runtimeDepInfo); + JavaCompilationArgs args = JavaCompilationArgs.builder() + .addTransitiveTargets(runtimeDepInfo, true, ClasspathType.RUNTIME_ONLY) + .build(); + attributes.addRuntimeClassPathEntries(args.getRuntimeJars()); + attributes.addInstrumentationMetadataEntries(args.getInstrumentationMetadata()); + } + + public Iterable<SourcesJavaCompilationArgsProvider> compilationArgsFromSources() { + return ruleContext.getPrerequisites("srcs", Mode.TARGET, + SourcesJavaCompilationArgsProvider.class); + } + + /** + * Adds jars in the given group of entries to the compile time classpath after + * using ijar to create jar interfaces for the generated jars. + */ + private void addCompileTimeClassPathEntriesMaybeThroughIjar( + JavaTargetAttributes.Builder attributes, ImmutableList<String> javacOpts) { + JavaCompilationHelper helper = new JavaCompilationHelper( + ruleContext, semantics, javacOpts, attributes); + for (FileProvider provider : ruleContext + .getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) { + Iterable<Artifact> jarFiles = helper.filterGeneratedJarsThroughIjar( + FileType.filter(provider.getFilesToBuild(), JavaSemantics.JAR)); + List<Artifact> jarsWithOwners = Lists.newArrayList(jarFiles); + attributes.addDirectCompileTimeClassPathEntries(jarsWithOwners); + attributes.addCompileTimeJarFiles(jarsWithOwners); + } + } + + /** + * Adds information about the annotation processors that should be run for this java target to + * the target attributes. + */ + private void addPlugins(JavaTargetAttributes.Builder attributes) { + for (JavaPluginInfoProvider plugin : activePlugins) { + for (String name : plugin.getProcessorClasses()) { + attributes.addProcessorName(name); + } + // Now get the plugin-libraries runtime classpath. + attributes.addProcessorPath(plugin.getProcessorClasspath()); + } + } + + private ImmutableList<JavaPluginInfoProvider> collectPlugins() { + List<JavaPluginInfoProvider> result = new ArrayList<>(); + Iterables.addAll(result, getPluginInfoProvidersForAttribute(":java_plugins", Mode.HOST)); + Iterables.addAll(result, getPluginInfoProvidersForAttribute("plugins", Mode.HOST)); + Iterables.addAll(result, getPluginInfoProvidersForAttribute("deps", Mode.TARGET)); + return ImmutableList.copyOf(result); + } + + Iterable<JavaPluginInfoProvider> getPluginInfoProvidersForAttribute(String attribute, + Mode mode) { + if (ruleContext.getRule().getRuleClassObject().hasAttr(attribute, Type.LABEL_LIST)) { + return ruleContext.getPrerequisites(attribute, mode, JavaPluginInfoProvider.class); + } + return ImmutableList.of(); + } + + /** + * Gets all the deps. + */ + public final Iterable<? extends TransitiveInfoCollection> getDependencies() { + return targetsTreatedAsDeps(ClasspathType.BOTH); + } + + /** + * Gets all the deps that implement a particular provider. + */ + public final <P extends TransitiveInfoProvider> Iterable<P> getDependencies( + Class<P> provider) { + return AnalysisUtils.getProviders(getDependencies(), provider); + } + + /** + * Returns true if and only if this target has the neverlink attribute set to + * 1, or false if the neverlink attribute does not exist (for example, on + * *_binary targets) + * + * @return the value of the neverlink attribute. + */ + public final boolean isNeverLink() { + return ruleContext.getRule().isAttrDefined("neverlink", Type.BOOLEAN) && + ruleContext.attributes().get("neverlink", Type.BOOLEAN); + } + + private ImmutableList<Artifact> getFilesToCompile(Artifact classJar) { + if (classJar == null) { + // Some subclasses don't produce jars + return ImmutableList.of(); + } + return ImmutableList.of(classJar); + } + + public ImmutableList<Dependency> computeStrictDepsFromJavaAttributes( + JavaTargetAttributes javaTargetAttributes) { + Multimap<Label, String> depMap = HashMultimap.<Label, String>create(); + for (Artifact jar : javaTargetAttributes.getDirectJars()) { + depMap.put(Preconditions.checkNotNull(jar.getOwner()), + jar.getExecPathString()); + } + ImmutableList.Builder<Dependency> depOuts = ImmutableList.builder(); + for (Label label : depMap.keySet()) { + depOuts.add(new Dependency(label, depMap.get(label))); + } + return depOuts.build(); + } + + public ImmutableList<Artifact> getSrcsArtifacts() { + return sources; + } + + public ImmutableList<String> getJavacOpts() { + return javacOpts; + } + + public ImmutableList<Artifact> getBootClasspath() { + return classpathFragment.getBootClasspath(); + } + + public NestedSet<Artifact> getRuntimeClasspath() { + return classpathFragment.getRuntimeClasspath(); + } + + public NestedSet<Artifact> getCompileTimeClasspath() { + return classpathFragment.getCompileTimeClasspath(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgs.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgs.java new file mode 100644 index 0000000..145d646 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgs.java
@@ -0,0 +1,301 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.util.FileType; + +import java.util.Collection; + +/** + * A container of Java compilation artifacts. + */ +public final class JavaCompilationArgs { + // TODO(bazel-team): It would be desirable to use LinkOrderNestedSet here so that + // parents-before-deps is preserved for graphs that are not trees. However, the legacy + // JavaLibraryCollector implemented naive link ordering and many targets in the + // depot depend on the consistency of left-to-right ordering that is not provided by + // LinkOrderNestedSet. They simply list their local dependencies before + // other targets that may use conflicting dependencies, and the local deps + // appear earlier on the classpath, as desired. Behavior of LinkOrderNestedSet + // can be very unintuitive in case of conflicting orders, because the order is + // decided by the rightmost branch in such cases. For example, if A depends on {junit4, + // B}, B depends on {C, D}, C depends on {junit3}, and D depends on {junit4}, + // the classpath of A will have junit3 before junit4. + private final NestedSet<Artifact> runtimeJars; + private final NestedSet<Artifact> compileTimeJars; + private final NestedSet<Artifact> instrumentationMetadata; + + public static final JavaCompilationArgs EMPTY_ARGS = new JavaCompilationArgs( + NestedSetBuilder.<Artifact>create(Order.NAIVE_LINK_ORDER), + NestedSetBuilder.<Artifact>create(Order.NAIVE_LINK_ORDER), + NestedSetBuilder.<Artifact>create(Order.NAIVE_LINK_ORDER)); + + private JavaCompilationArgs(NestedSet<Artifact> runtimeJars, + NestedSet<Artifact> compileTimeJars, + NestedSet<Artifact> instrumentationMetadata) { + this.runtimeJars = runtimeJars; + this.compileTimeJars = compileTimeJars; + this.instrumentationMetadata = instrumentationMetadata; + } + + /** + * Returns transitive runtime jars. + */ + public NestedSet<Artifact> getRuntimeJars() { + return runtimeJars; + } + + /** + * Returns transitive compile-time jars. + */ + public NestedSet<Artifact> getCompileTimeJars() { + return compileTimeJars; + } + + /** + * Returns transitive instrumentation metadata jars. + */ + public NestedSet<Artifact> getInstrumentationMetadata() { + return instrumentationMetadata; + } + + /** + * Returns a new builder instance. + */ + public static final Builder builder() { + return new Builder(); + } + + /** + * Builder for {@link JavaCompilationArgs}. + * + * + */ + public static final class Builder { + private final NestedSetBuilder<Artifact> runtimeJarsBuilder = + NestedSetBuilder.naiveLinkOrder(); + private final NestedSetBuilder<Artifact> compileTimeJarsBuilder = + NestedSetBuilder.naiveLinkOrder(); + private final NestedSetBuilder<Artifact> instrumentationMetadataBuilder = + NestedSetBuilder.naiveLinkOrder(); + + /** + * Use {@code TransitiveJavaCompilationArgs#builder()} to instantiate the builder. + */ + private Builder() { + } + + /** + * Legacy method for dealing with objects which construct + * {@link JavaCompilationArtifacts} objects. + */ + // TODO(bazel-team): Remove when we get rid of JavaCompilationArtifacts. + public Builder merge(JavaCompilationArtifacts other, boolean isNeverLink) { + if (!isNeverLink) { + addRuntimeJars(other.getRuntimeJars()); + } + addCompileTimeJars(other.getCompileTimeJars()); + addInstrumentationMetadata(other.getInstrumentationMetadata()); + return this; + } + + /** + * Legacy method for dealing with objects which construct + * {@link JavaCompilationArtifacts} objects. + */ + public Builder merge(JavaCompilationArtifacts other) { + return merge(other, false); + } + + public Builder addRuntimeJar(Artifact runtimeJar) { + this.runtimeJarsBuilder.add(runtimeJar); + return this; + } + + public Builder addRuntimeJars(Iterable<Artifact> runtimeJars) { + this.runtimeJarsBuilder.addAll(runtimeJars); + return this; + } + + public Builder addCompileTimeJar(Artifact compileTimeJar) { + this.compileTimeJarsBuilder.add(compileTimeJar); + return this; + } + + public Builder addCompileTimeJars(Iterable<Artifact> compileTimeJars) { + this.compileTimeJarsBuilder.addAll(compileTimeJars); + return this; + } + + public Builder addInstrumentationMetadata(Artifact instrumentationMetadata) { + this.instrumentationMetadataBuilder.add(instrumentationMetadata); + return this; + } + + public Builder addInstrumentationMetadata(Collection<Artifact> instrumentationMetadata) { + this.instrumentationMetadataBuilder.addAll(instrumentationMetadata); + return this; + } + + public Builder addTransitiveCompilationArgs( + JavaCompilationArgsProvider dep, boolean recursive, ClasspathType type) { + JavaCompilationArgs args = recursive + ? dep.getRecursiveJavaCompilationArgs() + : dep.getJavaCompilationArgs(); + addTransitiveArgs(args, type); + return this; + } + + public Builder addTransitiveCompilationArgs( + SourcesJavaCompilationArgsProvider dep, boolean recursive, ClasspathType type) { + JavaCompilationArgs args; + if (recursive) { + args = dep.getRecursiveJavaCompilationArgs(); + } else { + args = dep.getJavaCompilationArgs(); + } + addTransitiveArgs(args, type); + return this; + } + + public Builder addSourcesTransitiveCompilationArgs( + Iterable<? extends SourcesJavaCompilationArgsProvider> deps, + boolean recursive, + ClasspathType type) { + for (SourcesJavaCompilationArgsProvider dep : deps) { + addTransitiveCompilationArgs(dep, recursive, type); + } + + return this; + } + + /** + * Merges the artifacts of another target. + */ + public Builder addTransitiveTarget(TransitiveInfoCollection dep, boolean recursive, + ClasspathType type) { + JavaCompilationArgsProvider provider = dep.getProvider(JavaCompilationArgsProvider.class); + if (provider != null) { + addTransitiveCompilationArgs(provider, recursive, type); + return this; + } else { + NestedSet<Artifact> filesToBuild = + dep.getProvider(FileProvider.class).getFilesToBuild(); + for (Artifact jar : FileType.filter(filesToBuild, JavaSemantics.JAR)) { + addCompileTimeJar(jar); + addRuntimeJar(jar); + } + } + return this; + } + + /** + * Merges the artifacts of a collection of targets. + */ + public Builder addTransitiveTargets(Iterable<? extends TransitiveInfoCollection> deps, + boolean recursive, ClasspathType type) { + for (TransitiveInfoCollection dep : deps) { + addTransitiveTarget(dep, recursive, type); + } + return this; + } + + /** + * Merges the artifacts of a collection of targets. + */ + public Builder addTransitiveTargets(Iterable<? extends TransitiveInfoCollection> deps, + boolean recursive) { + return addTransitiveTargets(deps, recursive, ClasspathType.BOTH); + } + + /** + * Merges the artifacts of a collection of targets. + */ + public Builder addTransitiveDependencies(Iterable<JavaCompilationArgsProvider> deps, + boolean recursive) { + for (JavaCompilationArgsProvider dep : deps) { + addTransitiveDependency(dep, recursive, ClasspathType.BOTH); + } + return this; + } + + /** + * Merges the artifacts of another target. + */ + private Builder addTransitiveDependency(JavaCompilationArgsProvider dep, boolean recursive, + ClasspathType type) { + JavaCompilationArgs args = recursive + ? dep.getRecursiveJavaCompilationArgs() + : dep.getJavaCompilationArgs(); + addTransitiveArgs(args, type); + return this; + } + + /** + * Merges the artifacts of a collection of targets. + */ + public Builder addTransitiveTargets(Iterable<? extends TransitiveInfoCollection> deps) { + return addTransitiveTargets(deps, /*recursive=*/true, ClasspathType.BOTH); + } + + /** + * Includes the contents of another instance of JavaCompilationArgs. + * + * @param args the JavaCompilationArgs instance + * @param type the classpath(s) to consider + */ + public Builder addTransitiveArgs(JavaCompilationArgs args, ClasspathType type) { + if (!ClasspathType.RUNTIME_ONLY.equals(type)) { + compileTimeJarsBuilder.addTransitive(args.getCompileTimeJars()); + } + if (!ClasspathType.COMPILE_ONLY.equals(type)) { + runtimeJarsBuilder.addTransitive(args.getRuntimeJars()); + } + instrumentationMetadataBuilder.addTransitive( + args.getInstrumentationMetadata()); + return this; + } + + /** + * Builds a {@link JavaCompilationArgs} object. + */ + public JavaCompilationArgs build() { + return new JavaCompilationArgs( + runtimeJarsBuilder.build(), + compileTimeJarsBuilder.build(), + instrumentationMetadataBuilder.build()); + } + } + + /** + * Enum to specify transitive compilation args traversal + */ + public static enum ClasspathType { + /* treat the same for compile time and runtime */ + BOTH, + + /* Only include on compile classpath */ + COMPILE_ONLY, + + /* Only include on runtime classpath */ + RUNTIME_ONLY; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgsProvider.java new file mode 100644 index 0000000..1958ada --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArgsProvider.java
@@ -0,0 +1,94 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * An interface for objects that provide information on how to include them in + * Java builds. + */ +@Immutable +public final class JavaCompilationArgsProvider implements TransitiveInfoProvider { + private final JavaCompilationArgs javaCompilationArgs; + private final JavaCompilationArgs recursiveJavaCompilationArgs; + private final NestedSet<Artifact> compileTimeJavaDepArtifacts; + private final NestedSet<Artifact> runTimeJavaDepArtifacts; + + public JavaCompilationArgsProvider(JavaCompilationArgs javaCompilationArgs, + JavaCompilationArgs recursiveJavaCompilationArgs, + NestedSet<Artifact> compileTimeJavaDepArtifacts, + NestedSet<Artifact> runTimeJavaDepArtifacts) { + this.javaCompilationArgs = javaCompilationArgs; + this.recursiveJavaCompilationArgs = recursiveJavaCompilationArgs; + this.compileTimeJavaDepArtifacts = compileTimeJavaDepArtifacts; + this.runTimeJavaDepArtifacts = runTimeJavaDepArtifacts; + } + + public JavaCompilationArgsProvider(JavaCompilationArgs javaCompilationArgs, + JavaCompilationArgs recursiveJavaCompilationArgs) { + this.javaCompilationArgs = javaCompilationArgs; + this.recursiveJavaCompilationArgs = recursiveJavaCompilationArgs; + this.compileTimeJavaDepArtifacts = NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER); + this.runTimeJavaDepArtifacts = NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER); + } + + /** + * Returns non-recursively collected Java compilation information for + * building this target (called when strict_java_deps = 1). + * + * <p>Note that some of the parameters are still collected from the complete + * transitive closure. The non-recursive collection applies mainly to + * compile-time jars. + */ + public JavaCompilationArgs getJavaCompilationArgs() { + return javaCompilationArgs; + } + + /** + * Returns recursively collected Java compilation information for building + * this target (called when strict_java_deps = 0). + */ + public JavaCompilationArgs getRecursiveJavaCompilationArgs() { + return recursiveJavaCompilationArgs; + } + + /** + * Returns non-recursively collected Java dependency artifacts for + * computing a restricted classpath when building this target (called when + * strict_java_deps = 1). + * + * <p>Note that dependency artifacts are needed only when non-recursive + * compilation args do not provide a safe super-set of dependencies. + * Non-strict targets such as proto_library, always collecting their + * transitive closure of deps, do not need to provide dependency artifacts. + */ + public NestedSet<Artifact> getCompileTimeJavaDependencyArtifacts() { + return compileTimeJavaDepArtifacts; + } + + /** + * Returns Java dependency artifacts for computing a restricted run-time + * classpath (called when strict_java_deps = 1). + */ + public NestedSet<Artifact> getRunTimeJavaDependencyArtifacts() { + return runTimeJavaDepArtifacts; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArtifacts.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArtifacts.java new file mode 100644 index 0000000..98ccbac --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationArtifacts.java
@@ -0,0 +1,148 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.LinkedHashSet; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A collection of artifacts for java compilations. It concisely describes the + * outputs of a java-related rule, with runtime jars, compile-time jars, + * unfiltered compile-time jars (these are run through ijar if they are + * dependent upon by another target), source ijars, and instrumentation + * manifests. Not all rules generate all kinds of artifacts. Each java-related + * rule should add both a runtime jar and either a compile-time jar or an + * unfiltered compile-time jar. + * + * <p>An instance of this class only collects the data for the current target, + * not for the transitive closure of targets, so these still need to be + * collected using some other mechanism, such as the {@link + * JavaCompilationArgsProvider}. + */ +@Immutable +public final class JavaCompilationArtifacts { + + public static final JavaCompilationArtifacts EMPTY = new Builder().build(); + + private final ImmutableList<Artifact> runtimeJars; + private final ImmutableList<Artifact> compileTimeJars; + private final ImmutableList<Artifact> instrumentationMetadata; + private final Artifact compileTimeDependencyArtifact; + private final Artifact runTimeDependencyArtifact; + private final Artifact instrumentedJar; + + private JavaCompilationArtifacts(ImmutableList<Artifact> runtimeJars, + ImmutableList<Artifact> compileTimeJars, + ImmutableList<Artifact> instrumentationMetadata, + Artifact compileTimeDependencyArtifact, Artifact runTimeDependencyArtifact, + Artifact instrumentedJar) { + this.runtimeJars = runtimeJars; + this.compileTimeJars = compileTimeJars; + this.instrumentationMetadata = instrumentationMetadata; + this.compileTimeDependencyArtifact = compileTimeDependencyArtifact; + this.runTimeDependencyArtifact = runTimeDependencyArtifact; + this.instrumentedJar = instrumentedJar; + } + + public ImmutableList<Artifact> getRuntimeJars() { + return runtimeJars; + } + + public ImmutableList<Artifact> getCompileTimeJars() { + return compileTimeJars; + } + + public ImmutableList<Artifact> getInstrumentationMetadata() { + return instrumentationMetadata; + } + + public Artifact getCompileTimeDependencyArtifact() { + return compileTimeDependencyArtifact; + } + + public Artifact getRunTimeDependencyArtifact() { + return runTimeDependencyArtifact; + } + + public Artifact getInstrumentedJar() { + return instrumentedJar; + } + + /** + * A builder for {@link JavaCompilationArtifacts}. + */ + public static final class Builder { + private final Set<Artifact> runtimeJars = new LinkedHashSet<>(); + private final Set<Artifact> compileTimeJars = new LinkedHashSet<>(); + private final Set<Artifact> instrumentationMetadata = new LinkedHashSet<>(); + private Artifact compileTimeDependencies; + private Artifact runTimeDependencies; + private Artifact instrumentedJar; + + public JavaCompilationArtifacts build() { + return new JavaCompilationArtifacts(ImmutableList.copyOf(runtimeJars), + ImmutableList.copyOf(compileTimeJars), + ImmutableList.copyOf(instrumentationMetadata), + compileTimeDependencies, runTimeDependencies, instrumentedJar); + } + + public Builder addRuntimeJar(Artifact jar) { + this.runtimeJars.add(jar); + return this; + } + + public Builder addRuntimeJars(Iterable<Artifact> jars) { + Iterables.addAll(this.runtimeJars, jars); + return this; + } + + public Builder addCompileTimeJar(Artifact jar) { + this.compileTimeJars.add(jar); + return this; + } + + public Builder addCompileTimeJars(Iterable<Artifact> jars) { + Iterables.addAll(this.compileTimeJars, jars); + return this; + } + + public Builder addInstrumentationMetadata(Artifact instrumentationMetadata) { + this.instrumentationMetadata.add(instrumentationMetadata); + return this; + } + + public Builder setCompileTimeDependencies(@Nullable Artifact compileTimeDependencies) { + this.compileTimeDependencies = compileTimeDependencies; + return this; + } + + public Builder setRunTimeDependencies(@Nullable Artifact runTimeDependencies) { + this.runTimeDependencies = runTimeDependencies; + return this; + } + + public Builder setInstrumentedJar(@Nullable Artifact instrumentedJar) { + this.instrumentedJar = instrumentedJar; + return this; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java new file mode 100644 index 0000000..181dd12 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java
@@ -0,0 +1,436 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import static com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode.OFF; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType; +import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode; +import com.google.devtools.build.lib.vfs.FileSystemUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import javax.annotation.Nullable; + +/** + * A helper class for compiling Java targets. It contains method to create the + * various intermediate Artifacts for using ijars and source ijars. + * <p> + * Also supports the creation of resource and source only Jars. + */ +public class JavaCompilationHelper extends BaseJavaCompilationHelper { + private Artifact outputDepsProtoArtifact; + private JavaTargetAttributes.Builder attributes; + private JavaTargetAttributes builtAttributes; + private final ImmutableList<String> customJavacOpts; + private final List<Artifact> translations = new ArrayList<>(); + private boolean translationsFrozen = false; + private final JavaSemantics semantics; + + public JavaCompilationHelper(RuleContext ruleContext, JavaSemantics semantics, + ImmutableList<String> javacOpts, JavaTargetAttributes.Builder attributes) { + super(ruleContext); + this.attributes = attributes; + this.customJavacOpts = javacOpts; + this.semantics = semantics; + } + + public JavaCompilationHelper(RuleContext ruleContext, JavaSemantics semantics, + JavaTargetAttributes.Builder attributes) { + this(ruleContext, semantics, getDefaultJavacOptsFromRule(ruleContext), attributes); + } + + public JavaTargetAttributes getAttributes() { + if (builtAttributes == null) { + builtAttributes = attributes.build(); + } + return builtAttributes; + } + + /** + * Creates the Action that compiles Java source files. + * + * @param outputJar the class jar Artifact to create with the Action + * @param gensrcOutputJar the generated sources jar Artifact to create with the Action + * (null if no sources will be generated). + * @param outputDepsProto the compiler-generated jdeps file to create with the Action + * (null if not requested) + * @param outputMetadata metadata file (null if no instrumentation is needed). + */ + public void createCompileAction(Artifact outputJar, @Nullable Artifact gensrcOutputJar, + @Nullable Artifact outputDepsProto, @Nullable Artifact outputMetadata) { + JavaTargetAttributes attributes = getAttributes(); + List<String> javacOpts = getJavacOpts(); + JavaCompileAction.Builder builder = createJavaCompileActionBuilder(semantics); + builder.setClasspathEntries(attributes.getCompileTimeClassPath()); + builder.addResources(attributes.getResources()); + builder.addClasspathResources(attributes.getClassPathResources()); + // Only add default bootclasspath entries if not explicitly set in attributes. + if (!attributes.getBootClassPath().isEmpty()) { + builder.setBootclasspathEntries(attributes.getBootClassPath()); + } else { + builder.setBootclasspathEntries(getBootClasspath()); + } + builder.setLangtoolsJar(getLangtoolsJar()); + builder.setJavaBuilderJar(getJavaBuilderJar()); + builder.addTranslations(getTranslations()); + builder.setOutputJar(outputJar); + builder.setGensrcOutputJar(gensrcOutputJar); + builder.setOutputDepsProto(outputDepsProto); + builder.setMetadata(outputMetadata); + builder.addSourceFiles(attributes.getSourceFiles()); + builder.addSourceJars(attributes.getSourceJars()); + builder.setJavacOpts(javacOpts); + builder.setCompressJar(true); + builder.setClassDirectory(outputDir(outputJar)); + builder.setSourceGenDirectory(sourceGenDir(outputJar)); + builder.setTempDirectory(tempDir(outputJar)); + builder.addProcessorPaths(attributes.getProcessorPath()); + builder.addProcessorNames(attributes.getProcessorNames()); + builder.setStrictJavaDeps(attributes.getStrictJavaDeps()); + builder.addDirectJars(attributes.getDirectJars()); + builder.addCompileTimeDependencyArtifacts(attributes.getCompileTimeDependencyArtifacts()); + builder.setRuleKind(attributes.getRuleKind()); + builder.setTargetLabel(attributes.getTargetLabel()); + getAnalysisEnvironment().registerAction(builder.build()); + } + + /** + * Creates the Action that compiles Java source files and optionally instruments them for + * coverage. + * + * @param outputJar the class jar Artifact to create with the Action + * @param gensrcJar the generated sources jar Artifact to create with the Action + * @param outputDepsProto the compiler-generated jdeps file to create with the Action + * @param javaArtifactsBuilder the build to store the instrumentation metadata in + */ + public void createCompileActionWithInstrumentation(Artifact outputJar, Artifact gensrcJar, + Artifact outputDepsProto, JavaCompilationArtifacts.Builder javaArtifactsBuilder) { + createCompileAction(outputJar, gensrcJar, outputDepsProto, + createInstrumentationMetadata(outputJar, javaArtifactsBuilder)); + } + + /** + * Creates the instrumentation metadata artifact if needed. + * + * @return the instrumentation metadata artifact or null if instrumentation is + * disabled + */ + public Artifact createInstrumentationMetadata(Artifact outputJar, + JavaCompilationArtifacts.Builder javaArtifactsBuilder) { + // If we need to instrument the jar, add additional output (the coverage metadata file) to the + // JavaCompileAction. + Artifact instrumentationMetadata = null; + if (shouldInstrumentJar()) { + instrumentationMetadata = semantics.createInstrumentationMetadataArtifact( + getAnalysisEnvironment(), outputJar); + + if (instrumentationMetadata != null) { + javaArtifactsBuilder.addInstrumentationMetadata(instrumentationMetadata); + } + } + return instrumentationMetadata; + } + + private boolean shouldInstrumentJar() { + // TODO(bazel-team): What about source jars? + return getConfiguration().isCodeCoverageEnabled() && attributes.hasSourceFiles() && + getConfiguration().getInstrumentationFilter().isIncluded( + getRuleContext().getLabel().toString()); + } + + /** + * Returns the artifact for a jar file containing source files that were generated by an + * annotation processor or null if no annotation processors are used. + */ + public Artifact createGensrcJar(@Nullable Artifact outputJar) { + if (!usesAnnotationProcessing()) { + return null; + } + return getAnalysisEnvironment().getDerivedArtifact( + FileSystemUtils.appendWithoutExtension(outputJar.getRootRelativePath(), "-gensrc"), + outputJar.getRoot()); + } + + /** + * Returns whether this target uses annotation processing. + */ + private boolean usesAnnotationProcessing() { + JavaTargetAttributes attributes = getAttributes(); + return getJavacOpts().contains("-processor") || !attributes.getProcessorNames().isEmpty(); + } + + public Artifact getOutputDepsProtoArtifact() { + return outputDepsProtoArtifact; + } + /** + * Creates the jdeps file artifact if needed. Returns null if the target can't emit dependency + * information (i.e there is no compilation step, the target acts as an alias). + * + * @param outputJar output jar artifact used to derive the name + * @return the jdeps file artifact or null if the target can't generate such a file + */ + public Artifact createOutputDepsProtoArtifact(Artifact outputJar, + JavaCompilationArtifacts.Builder builder) { + if (!generatesOutputDeps()) { + return null; + } + + outputDepsProtoArtifact = getAnalysisEnvironment().getDerivedArtifact( + FileSystemUtils.replaceExtension(outputJar.getRootRelativePath(), ".jdeps"), + outputJar.getRoot()); + + builder.setRunTimeDependencies(outputDepsProtoArtifact); + return outputDepsProtoArtifact; + } + + /** + * Returns whether this target emits dependency information. Compilation must occur, so certain + * targets acting as aliases have to be filtered out. + */ + private boolean generatesOutputDeps() { + return getJavaConfiguration().getGenerateJavaDeps() && + (attributes.hasSourceFiles() || attributes.hasSourceJars()); + } + + /** + * Creates an Action that packages all of the resources into a Jar. This + * includes the declared resources, the classpath resources and the translated + * messages. + * + * <p>The resource jar artifact is derived from the given original jar, by + * prepending the given prefix and appending the given suffix. The new jar + * uses the same root as the original jar. + */ + // TODO(bazel-team): Extract this method to make it easier to create simple + // zip/jar archives without having to first create a JavaCompilationhelper and + // JavaTargetAttributes. + public Artifact createResourceJarAction(Artifact resourceJar) { + JavaTargetAttributes attributes = getAttributes(); + JavaCompileAction.Builder builder = createJavaCompileActionBuilder(semantics); + builder.setOutputJar(resourceJar); + builder.addResources(attributes.getResources()); + builder.addClasspathResources(attributes.getClassPathResources()); + builder.setLangtoolsJar(getLangtoolsJar()); + builder.addTranslations(getTranslations()); + builder.setCompressJar(true); + builder.setClassDirectory(outputDir(resourceJar)); + builder.setTempDirectory(tempDir(resourceJar)); + builder.setJavaBuilderJar(getJavaBuilderJar()); + getAnalysisEnvironment().registerAction(builder.build()); + return resourceJar; + } + + /** + * Creates an Action that packages the Java source files into a Jar. If {@code gensrcJar} is + * non-null, includes the contents of the {@code gensrcJar} with the output source jar. + * + * @param outputJar the Artifact to create with the Action + * @param gensrcJar the generated sources jar Artifact that should be included with the + * sources in the output Artifact. May be null. + */ + public void createSourceJarAction(Artifact outputJar, @Nullable Artifact gensrcJar) { + JavaTargetAttributes attributes = getAttributes(); + Collection<Artifact> resourceJars = new ArrayList<>(attributes.getSourceJars()); + if (gensrcJar != null) { + resourceJars.add(gensrcJar); + } + createSourceJarAction(semantics, attributes.getSourceFiles(), resourceJars, outputJar); + } + + /** + * Creates the actions that produce the interface jar. Adds the jar artifacts to the given + * JavaCompilationArtifacts builder. + */ + public void createCompileTimeJarAction(Artifact runtimeJar, + @Nullable Artifact runtimeDeps, JavaCompilationArtifacts.Builder builder) { + Artifact jar = getJavaConfiguration().getUseIjars() + ? createIjarAction(runtimeJar, false) + : runtimeJar; + Artifact deps = runtimeDeps; + + builder.addCompileTimeJar(jar); + builder.setCompileTimeDependencies(deps); + } + + /** + * Creates actions that create ijars from generated jars that are an input to + * the Java target. + * + * @return the generated ijars or original jars that are not generated by a + * genrule + */ + public Iterable<Artifact> filterGeneratedJarsThroughIjar(Iterable<Artifact> jars) { + if (!getJavaConfiguration().getUseIjars()) { + return jars; + } + // We need to copy this list in order to avoid generating a new action each time the iterator + // is enumerated + return ImmutableList.copyOf(Iterables.transform(jars, new Function<Artifact, Artifact>() { + @Override + public Artifact apply(Artifact jar) { + return !jar.isSourceArtifact() ? createIjarAction(jar, true) : jar; + } + })); + } + + private void addArgsAndJarsToAttributes(JavaCompilationArgs args, Iterable<Artifact> directJars) { + // Can only be non-null when isStrict() returns true. + if (directJars != null) { + attributes.addDirectCompileTimeClassPathEntries(directJars); + attributes.addDirectJars(directJars); + } + + attributes.merge(args); + } + + private void addLibrariesToAttributesInternal(Iterable<? extends TransitiveInfoCollection> deps) { + JavaCompilationArgs args = JavaCompilationArgs.builder() + .addTransitiveTargets(deps).build(); + + NestedSet<Artifact> directJars = isStrict() + ? getNonRecursiveCompileTimeJarsFromCollection(deps) + : null; + addArgsAndJarsToAttributes(args, directJars); + } + + private void addProvidersToAttributesInternal( + Iterable<? extends SourcesJavaCompilationArgsProvider> deps, boolean isNeverLink) { + JavaCompilationArgs args = JavaCompilationArgs.builder() + .addSourcesTransitiveCompilationArgs(deps, true, + isNeverLink ? ClasspathType.COMPILE_ONLY : ClasspathType.BOTH) + .build(); + + NestedSet<Artifact> directJars = isStrict() + ? getNonRecursiveCompileTimeJarsFromProvider(deps, isNeverLink) + : null; + addArgsAndJarsToAttributes(args, directJars); + } + + private boolean isStrict() { + return getStrictJavaDeps() != OFF; + } + + private NestedSet<Artifact> getNonRecursiveCompileTimeJarsFromCollection( + Iterable<? extends TransitiveInfoCollection> deps) { + JavaCompilationArgs.Builder builder = JavaCompilationArgs.builder(); + builder.addTransitiveTargets(deps, /*recursive=*/false); + return builder.build().getCompileTimeJars(); + } + + private NestedSet<Artifact> getNonRecursiveCompileTimeJarsFromProvider( + Iterable<? extends SourcesJavaCompilationArgsProvider> deps, boolean isNeverLink) { + return JavaCompilationArgs.builder() + .addSourcesTransitiveCompilationArgs(deps, false, + isNeverLink ? ClasspathType.COMPILE_ONLY : ClasspathType.BOTH) + .build().getCompileTimeJars(); + } + + private void addDependencyArtifactsToAttributes( + Iterable<? extends TransitiveInfoCollection> deps) { + NestedSetBuilder<Artifact> compileTimeBuilder = NestedSetBuilder.stableOrder(); + NestedSetBuilder<Artifact> runTimeBuilder = NestedSetBuilder.stableOrder(); + for (JavaCompilationArgsProvider provider : AnalysisUtils.getProviders( + deps, JavaCompilationArgsProvider.class)) { + compileTimeBuilder.addTransitive(provider.getCompileTimeJavaDependencyArtifacts()); + runTimeBuilder.addTransitive(provider.getRunTimeJavaDependencyArtifacts()); + } + attributes.addCompileTimeDependencyArtifacts(compileTimeBuilder.build()); + attributes.addRuntimeDependencyArtifacts(runTimeBuilder.build()); + } + + /** + * Adds the compile time and runtime Java libraries in the transitive closure + * of the deps to the attributes. + * + * @param deps the dependencies to be included as roots of the transitive + * closure + */ + public void addLibrariesToAttributes(Iterable<? extends TransitiveInfoCollection> deps) { + // Enforcing strict Java dependencies: when the --strict_java_deps flag is + // WARN or ERROR, or is DEFAULT and strict_java_deps attribute is unset, + // we use a stricter javac compiler to perform direct deps checks. + attributes.setStrictJavaDeps(getStrictJavaDeps()); + addLibrariesToAttributesInternal(deps); + + JavaClasspathMode classpathMode = getJavaConfiguration().getReduceJavaClasspath(); + if (isStrict() && classpathMode != JavaClasspathMode.OFF) { + addDependencyArtifactsToAttributes(deps); + } + } + + public void addProvidersToAttributes(Iterable<? extends SourcesJavaCompilationArgsProvider> deps, + boolean isNeverLink) { + // see addLibrariesToAttributes() for explanation + attributes.setStrictJavaDeps(getStrictJavaDeps()); + addProvidersToAttributesInternal(deps, isNeverLink); + } + + /** + * Determines whether to enable strict_java_deps. + * + * @return filtered command line flag value, defaulting to ERROR + */ + public StrictDepsMode getStrictJavaDeps() { + return getJavaConfiguration().getFilteredStrictJavaDeps(); + } + + /** + * Gets the value of the "javacopts" attribute combining them with the + * default options. If the current rule has no javacopts attribute, this + * method only returns the default options. + */ + @VisibleForTesting + ImmutableList<String> getJavacOpts() { + return customJavacOpts; + } + + /** + * Obtains the standard list of javac opts needed to build {@code rule}. + * + * This method must only be called during initialization. + * + * @param ruleContext a rule context + * @return a list of options to provide to javac + */ + private static ImmutableList<String> getDefaultJavacOptsFromRule(RuleContext ruleContext) { + return ImmutableList.copyOf(Iterables.concat( + JavaToolchainProvider.getDefaultJavacOptions(ruleContext), + ruleContext.getTokenizedStringListAttr("javacopts"))); + } + + public void addTranslations(Collection<Artifact> translations) { + Preconditions.checkArgument(!translationsFrozen); + this.translations.addAll(translations); + } + + private ImmutableList<Artifact> getTranslations() { + translationsFrozen = true; + return ImmutableList.copyOf(translations); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java new file mode 100644 index 0000000..655270b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java
@@ -0,0 +1,1021 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionInputHelper; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ParameterFile; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.actions.extra.JavaCompileInfo; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.CustomArgv; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.CustomMultiArgv; +import com.google.devtools.build.lib.analysis.actions.ParameterFileWriteAction; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.ImmutableIterable; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.util.StringCanonicalizer; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; + +/** + * Action that represents a Java compilation. + */ +@ThreadCompatible +public class JavaCompileAction extends AbstractAction { + + private static final String GUID = "786e174d-ed97-4e79-9f61-ae74430714cf"; + + private static final ResourceSet LOCAL_RESOURCES = + new ResourceSet(750 /*MB*/, 0.5 /*CPU*/, 0.0 /*IO*/); + + private final CommandLine javaCompileCommandLine; + private final CommandLine commandLine; + + /** + * The directory in which generated classfiles are placed. + * May be erased/created by the JavaBuilder. + */ + private final PathFragment classDirectory; + + private final Artifact outputJar; + + /** + * The list of classpath entries to specify to javac. + */ + private final NestedSet<Artifact> classpath; + + /** + * The list of classpath entries to search for annotation processors. + */ + private final ImmutableList<Artifact> processorPath; + + /** + * The list of annotation processor classes to run. + */ + private final ImmutableList<String> processorNames; + + /** + * The translation messages. + */ + private final ImmutableList<Artifact> messages; + + /** + * The set of resources to put into the jar. + */ + private final ImmutableList<Artifact> resources; + + /** + * The set of classpath resources to put into the jar. + */ + private final ImmutableList<Artifact> classpathResources; + + /** + * The set of files which contain lists of additional Java source files to + * compile. + */ + private final ImmutableList<Artifact> sourceJars; + + /** + * The set of explicit Java source files to compile. + */ + private final ImmutableList<Artifact> sourceFiles; + + /** + * The compiler options to pass to javac. + */ + private final ImmutableList<String> javacOpts; + + /** + * The subset of classpath jars provided by direct dependencies. + */ + private final ImmutableList<Artifact> directJars; + + /** + * The level of strict dependency checks (off, warnings, or errors). + */ + private final BuildConfiguration.StrictDepsMode strictJavaDeps; + + /** + * The set of .deps artifacts provided by direct dependencies. + */ + private final ImmutableList<Artifact> compileTimeDependencyArtifacts; + + /** + * The java semantics to get the list of action outputs. + */ + private final JavaSemantics semantics; + + /** + * Constructs an action to compile a set of Java source files to class files. + * + * @param owner the action owner, typically a java_* RuleConfiguredTarget. + * @param baseInputs the set of the input artifacts of the compile action + * without the parameter file action; + * @param outputs the outputs of the action + * @param javaCompileCommandLine the command line for the java library + * builder - it's actually written to the parameter file, but other + * parts (for example, ide_build_info) need access to the data + * @param commandLine the actual invocation command line + */ + private JavaCompileAction(ActionOwner owner, + Iterable<Artifact> baseInputs, + Collection<Artifact> outputs, + CommandLine javaCompileCommandLine, + CommandLine commandLine, + PathFragment classDirectory, + Artifact outputJar, + NestedSet<Artifact> classpath, + List<Artifact> processorPath, + Artifact langtoolsJar, + Artifact javaBuilderJar, + List<String> processorNames, + Collection<Artifact> messages, + Collection<Artifact> resources, + Collection<Artifact> classpathResources, + Collection<Artifact> sourceJars, + Collection<Artifact> sourceFiles, + List<String> javacOpts, + Collection<Artifact> directJars, + BuildConfiguration.StrictDepsMode strictJavaDeps, + Collection<Artifact> compileTimeDependencyArtifacts, + JavaSemantics semantics) { + super(owner, Iterables.concat(ImmutableList.of( + classpath, processorPath, messages, resources, + classpathResources, sourceJars, sourceFiles, compileTimeDependencyArtifacts, + ImmutableList.of(langtoolsJar, javaBuilderJar), baseInputs)), + outputs); + this.javaCompileCommandLine = javaCompileCommandLine; + this.commandLine = commandLine; + + this.classDirectory = Preconditions.checkNotNull(classDirectory); + this.outputJar = outputJar; + this.classpath = classpath; + this.processorPath = ImmutableList.copyOf(processorPath); + this.processorNames = ImmutableList.copyOf(processorNames); + this.messages = ImmutableList.copyOf(messages); + this.resources = ImmutableList.copyOf(resources); + this.classpathResources = ImmutableList.copyOf(classpathResources); + this.sourceJars = ImmutableList.copyOf(sourceJars); + this.sourceFiles = ImmutableList.copyOf(sourceFiles); + this.javacOpts = ImmutableList.copyOf(javacOpts); + this.directJars = ImmutableList.copyOf(directJars); + this.strictJavaDeps = strictJavaDeps; + this.compileTimeDependencyArtifacts = ImmutableList.copyOf(compileTimeDependencyArtifacts); + this.semantics = semantics; + } + + /** + * Returns the given (passed to constructor) source files. + */ + @VisibleForTesting + public Collection<Artifact> getSourceFiles() { + return sourceFiles; + } + + /** + * Returns the list of paths that represent the resources to be added to the + * jar. + */ + @VisibleForTesting + public Collection<Artifact> getResources() { + return resources; + } + + /** + * Returns the list of paths that represents the classpath. + */ + @VisibleForTesting + public Iterable<Artifact> getClasspath() { + return classpath; + } + + /** + * Returns the list of paths that represents the source jars. + */ + @VisibleForTesting + public Collection<Artifact> getSourceJars() { + return sourceJars; + } + + /** + * Returns the list of paths that represents the processor path. + */ + @VisibleForTesting + public List<Artifact> getProcessorpath() { + return processorPath; + } + + @VisibleForTesting + public List<String> getJavacOpts() { + return javacOpts; + } + + @VisibleForTesting + public Collection<Artifact> getDirectJars() { + return directJars; + } + + @VisibleForTesting + public Collection<Artifact> getCompileTimeDependencyArtifacts() { + return compileTimeDependencyArtifacts; + } + + @VisibleForTesting + public BuildConfiguration.StrictDepsMode getStrictJavaDepsMode() { + return strictJavaDeps; + } + + public PathFragment getClassDirectory() { + return classDirectory; + } + + /** + * Returns the list of class names of processors that should + * be run. + */ + @VisibleForTesting + public List<String> getProcessorNames() { + return processorNames; + } + + /** + * Returns the output jar artifact that gets generated by archiving the + * results of the Java compilation and the declared resources. + */ + public Artifact getOutputJar() { + return outputJar; + } + + @Override + public Artifact getPrimaryOutput() { + return getOutputJar(); + } + + /** + * Constructs a command line that can be used to invoke the + * JavaBuilder. + * + * <p>Do not use this method, except for testing (and for the in-process + * strategy). + */ + @VisibleForTesting + public Iterable<String> buildCommandLine() { + return javaCompileCommandLine.arguments(); + } + + /** + * Returns the command and arguments for a java compile action. + */ + public List<String> getCommand() { + return ImmutableList.copyOf(commandLine.arguments()); + } + + @Override + @ThreadCompatible + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + try { + List<ActionInput> outputs = new ArrayList<>(); + outputs.addAll(getOutputs()); + // Add a few useful side-effect output files to the list to retrieve. + // TODO(bazel-team): Just make these Artifacts. + PathFragment classDirectory = getClassDirectory(); + outputs.addAll(semantics.getExtraJavaCompileOutputs(classDirectory)); + outputs.add(ActionInputHelper.fromPath(classDirectory.getChild("srclist").getPathString())); + + try { + // Make sure the directories exist, else the distributor will bomb. + Path classDirectoryPath = executor.getExecRoot().getRelative(getClassDirectory()); + FileSystemUtils.createDirectoryAndParents(classDirectoryPath); + } catch (IOException e) { + throw new EnvironmentalExecException(e.getMessage()); + } + + final ImmutableList<ActionInput> finalOutputs = ImmutableList.copyOf(outputs); + Spawn spawn = new BaseSpawn(getCommand(), ImmutableMap.<String, String>of(), + ImmutableMap.<String, String>of(), this, LOCAL_RESOURCES) { + @Override + public Collection<? extends ActionInput> getOutputFiles() { + return finalOutputs; + } + }; + + executor.getSpawnActionContext(getMnemonic()).exec(spawn, actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException("Java compilation in rule '" + getOwner().getLabel() + "'", + executor.getVerboseFailures(), this); + } + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addStrings(commandLine.arguments()); + return f.hexDigestAndReset(); + } + + @Override + public String describeKey() { + StringBuilder message = new StringBuilder(); + for (String arg : ShellEscaper.escapeAll(commandLine.arguments())) { + message.append(" Command-line argument: "); + message.append(arg); + message.append('\n'); + } + return message.toString(); + } + + @Override + public String getMnemonic() { + return "Javac"; + } + + @Override + protected String getRawProgressMessage() { + int count = sourceFiles.size(); + if (count == 0) { // nothing to compile, just bundling resources and messages + count = resources.size() + classpathResources.size() + messages.size(); + } + return "Building " + outputJar.prettyPrint() + " (" + count + " files)"; + } + + @Override + public String describeStrategy(Executor executor) { + return getContext(executor).strategyLocality(getMnemonic(), true); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + SpawnActionContext context = getContext(executor); + if (context.isRemotable(getMnemonic(), true)) { + return ResourceSet.ZERO; + } + return LOCAL_RESOURCES; + } + + protected SpawnActionContext getContext(Executor executor) { + return executor.getSpawnActionContext(getMnemonic()); + } + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("JavaBuilder "); + Joiner.on(' ').appendTo(result, commandLine.arguments()); + return result.toString(); + } + + @Override + public ExtraActionInfo.Builder getExtraActionInfo() { + JavaCompileInfo.Builder info = JavaCompileInfo.newBuilder(); + info.addAllSourceFile(Artifact.toExecPaths(getSourceFiles())); + info.addAllClasspath(Artifact.toExecPaths(getClasspath())); + info.addClasspath(getClassDirectory().getPathString()); + info.addAllSourcepath(Artifact.toExecPaths(getSourceJars())); + info.addAllJavacOpt(getJavacOpts()); + info.addAllProcessor(getProcessorNames()); + info.addAllProcessorpath(Artifact.toExecPaths(getProcessorpath())); + info.setOutputjar(getOutputJar().getExecPathString()); + + return super.getExtraActionInfo() + .setExtension(JavaCompileInfo.javaCompileInfo, info.build()); + } + + /** + * Creates an instance. + * + * @param configuration the build configuration, which provides the default options and the path + * to the compiler, etc. + * @param classDirectory the directory in which generated classfiles are placed relative to the + * exec root + * @param sourceGenDirectory the directory where source files generated by annotation processors + * should be stored. + * @param tempDirectory a directory in which the library builder can store temporary files + * relative to the exec root + * @param outputJar output jar + * @param compressJar if true compress the output jar + * @param outputDepsProto the proto file capturing dependency information + * @param classpath the complete classpath, the directory in which generated classfiles are placed + * @param processorPath the classpath where javac should search for annotation processors + * @param processorNames the classes that javac should use as annotation processors + * @param messages the message files for translation + * @param resources the set of resources to put into the jar + * @param classpathResources the set of classpath resources to put into the jar + * @param sourceJars the set of jars containing additional source files to compile + * @param sourceFiles the set of explicit Java source files to compile + * @param javacOpts the compiler options to pass to javac + */ + private static CustomCommandLine.Builder javaCompileCommandLine( + final JavaSemantics semantics, + final BuildConfiguration configuration, + final PathFragment classDirectory, + final PathFragment sourceGenDirectory, + PathFragment tempDirectory, + Artifact outputJar, + Artifact gensrcOutputJar, + boolean compressJar, + Artifact outputDepsProto, + final NestedSet<Artifact> classpath, + List<Artifact> processorPath, + Artifact langtoolsJar, + Artifact javaBuilderJar, + List<String> processorNames, + Collection<Artifact> messages, + Collection<Artifact> resources, + Collection<Artifact> classpathResources, + Collection<Artifact> sourceJars, + Collection<Artifact> sourceFiles, + List<String> javacOpts, + final Collection<Artifact> directJars, + BuildConfiguration.StrictDepsMode strictJavaDeps, + Collection<Artifact> compileTimeDependencyArtifacts, + String ruleKind, + Label targetLabel) { + Preconditions.checkNotNull(classDirectory); + Preconditions.checkNotNull(tempDirectory); + Preconditions.checkNotNull(langtoolsJar); + Preconditions.checkNotNull(javaBuilderJar); + + CustomCommandLine.Builder result = CustomCommandLine.builder(); + + result.add("--classdir").addPath(classDirectory); + + result.add("--tempdir").addPath(tempDirectory); + + if (outputJar != null) { + result.addExecPath("--output", outputJar); + } + + if (gensrcOutputJar != null) { + result.add("--sourcegendir").addPath(sourceGenDirectory); + result.addExecPath("--generated_sources_output", gensrcOutputJar); + } + + if (compressJar) { + result.add("--compress_jar"); + } + + if (outputDepsProto != null) { + result.addExecPath("--output_deps_proto", outputDepsProto); + } + + result.add("--classpath").add(new CustomArgv() { + @Override + public String argv() { + List<PathFragment> classpathEntries = new ArrayList<>(); + for (Artifact classpathArtifact : classpath) { + classpathEntries.add(classpathArtifact.getExecPath()); + } + classpathEntries.add(classDirectory); + return Joiner.on(configuration.getHostPathSeparator()).join(classpathEntries); + } + }); + + if (!processorPath.isEmpty()) { + result.addJoinExecPaths("--processorpath", + configuration.getHostPathSeparator(), processorPath); + } + + if (!processorNames.isEmpty()) { + result.add("--processors", processorNames); + } + + if (!messages.isEmpty()) { + result.add("--messages"); + for (Artifact message : messages) { + addAsResourcePrefixedExecPath(semantics, message, result); + } + } + + if (!resources.isEmpty()) { + result.add("--resources"); + for (Artifact resource : resources) { + addAsResourcePrefixedExecPath(semantics, resource, result); + } + } + + if (!classpathResources.isEmpty()) { + result.addExecPaths("--classpath_resources", classpathResources); + } + + if (!sourceJars.isEmpty()) { + result.addExecPaths("--source_jars", sourceJars); + } + + result.addExecPaths("--sources", sourceFiles); + + if (!javacOpts.isEmpty()) { + result.add("--javacopts", javacOpts); + } + + // strict_java_deps controls whether the mapping from jars to targets is + // written out and whether we try to minimize the compile-time classpath. + if (strictJavaDeps != BuildConfiguration.StrictDepsMode.OFF) { + result.add("--strict_java_deps"); + result.add((semantics.useStrictJavaDeps(configuration) ? strictJavaDeps + : BuildConfiguration.StrictDepsMode.OFF).toString()); + result.add(new CustomMultiArgv() { + @Override + public Iterable<String> argv() { + return addJarsToTargets(classpath, directJars); + } + }); + + if (configuration.getFragment(JavaConfiguration.class).getReduceJavaClasspath() + == JavaClasspathMode.JAVABUILDER) { + result.add("--reduce_classpath"); + + if (!compileTimeDependencyArtifacts.isEmpty()) { + result.addExecPaths("--deps_artifacts", compileTimeDependencyArtifacts); + } + } + } + + if (ruleKind != null) { + result.add("--rule_kind"); + result.add(ruleKind); + } + if (targetLabel != null) { + result.add("--target_label"); + if (targetLabel.getPackageIdentifier().getRepository().isDefault()) { + result.add(targetLabel.toString()); + } else { + // @-prefixed strings will be assumed to be filenames and expanded by + // {@link JavaLibraryBuildRequest}, so add an extra &at; to escape it. + result.add("@" + targetLabel); + } + } + + return result; + } + + private static void addAsResourcePrefixedExecPath(JavaSemantics semantics, + Artifact artifact, CustomCommandLine.Builder builder) { + PathFragment execPath = artifact.getExecPath(); + PathFragment resourcePath = semantics.getJavaResourcePath(artifact.getRootRelativePath()); + if (execPath.equals(resourcePath)) { + builder.addPaths(":%s", resourcePath); + } else { + // execPath must end with resourcePath in all cases + PathFragment rootPrefix = trimTail(execPath, resourcePath); + builder.addPaths("%s:%s", rootPrefix, resourcePath); + } + } + + /** + * Returns the root-part of a given path by trimming off the end specified by + * a given tail. Assumes that the tail is known to match, and simply relies on + * the segment lengths. + */ + private static PathFragment trimTail(PathFragment path, PathFragment tail) { + return path.subFragment(0, path.segmentCount() - tail.segmentCount()); + } + + /** + * Builds the list of mappings between jars on the classpath and their + * originating targets names. + */ + private static ImmutableList<String> addJarsToTargets( + NestedSet<Artifact> classpath, Collection<Artifact> directJars) { + ImmutableList.Builder<String> builder = ImmutableList.builder(); + for (Artifact jar : classpath) { + builder.add(directJars.contains(jar) + ? "--direct_dependency" + : "--indirect_dependency"); + builder.add(jar.getExecPathString()); + Label label = getTargetName(jar); + builder.add(label.getPackageIdentifier().getRepository().isDefault() + ? label.toString() + : label.toPathFragment().toString()); + } + return builder.build(); + } + + /** + * Gets the name of the target that produced the given jar artifact. + * + * When specifying jars directly in the "srcs" attribute of a rule (mostly + * for third_party libraries), there is no generating action, so we just + * return the jar name in label form. + */ + private static Label getTargetName(Artifact jar) { + return Preconditions.checkNotNull(jar.getOwner(), jar); + } + + /** + * The actual command line executed for a compile action. + */ + private static CommandLine spawnCommandLine(PathFragment javaExecutable, Artifact javaBuilderJar, + Artifact langtoolsJar, Artifact paramFile, ImmutableList<String> javaBuilderJvmFlags) { + Preconditions.checkNotNull(langtoolsJar); + Preconditions.checkNotNull(javaBuilderJar); + return CustomCommandLine.builder() + .addPath(javaExecutable) + // Langtools jar is placed on the boot classpath so that it can override classes + // in the JRE. Typically this has no effect since langtools.jar does not have + // classes in common with rt.jar. However, it is necessary when using a version + // of javac.jar generated via ant from the langtools build.xml that is of a + // different version than AND has an overlap in contents with the default + // run-time (eg while upgrading the Java version). + .addPaths("-Xbootclasspath/p:%s", langtoolsJar.getExecPath()) + .add(javaBuilderJvmFlags) + .addExecPath("-jar", javaBuilderJar) + .addPaths("@%s", paramFile.getExecPath()) + .build(); + } + + /** + * Builder class to construct Java compile actions. + */ + public static class Builder { + private final ActionOwner owner; + private final AnalysisEnvironment analysisEnvironment; + private final BuildConfiguration configuration; + private final JavaSemantics semantics; + + private PathFragment javaExecutable; + private List<Artifact> javabaseInputs = ImmutableList.of(); + private Artifact outputJar; + private Artifact gensrcOutputJar; + private Artifact outputDepsProto; + private Artifact paramFile; + private Artifact metadata; + private final Collection<Artifact> sourceFiles = new ArrayList<>(); + private final Collection<Artifact> sourceJars = new ArrayList<>(); + private final Collection<Artifact> resources = new ArrayList<>(); + private final Collection<Artifact> classpathResources = new ArrayList<>(); + private final Collection<Artifact> translations = new LinkedHashSet<>(); + private BuildConfiguration.StrictDepsMode strictJavaDeps = + BuildConfiguration.StrictDepsMode.OFF; + private final Collection<Artifact> directJars = new ArrayList<>(); + private final Collection<Artifact> compileTimeDependencyArtifacts = new ArrayList<>(); + private List<String> javacOpts = new ArrayList<>(); + private boolean compressJar; + private NestedSet<Artifact> classpathEntries = + NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); + private ImmutableList<Artifact> bootclasspathEntries = ImmutableList.of(); + private Artifact javaBuilderJar; + private Artifact langtoolsJar; + private PathFragment classDirectory; + private PathFragment sourceGenDirectory; + private PathFragment tempDirectory; + private final List<Artifact> processorPath = new ArrayList<>(); + private final List<String> processorNames = new ArrayList<>(); + private String ruleKind; + private Label targetLabel; + + /** + * Creates a Builder from an owner and a build configuration. + */ + public Builder(ActionOwner owner, AnalysisEnvironment analysisEnvironment, + BuildConfiguration configuration, JavaSemantics semantics) { + this.owner = owner; + this.analysisEnvironment = analysisEnvironment; + this.configuration = configuration; + this.semantics = semantics; + } + + /** + * Creates a Builder from an owner and a build configuration. + */ + public Builder(RuleContext ruleContext, JavaSemantics semantics) { + this(ruleContext.getActionOwner(), ruleContext.getAnalysisEnvironment(), + ruleContext.getConfiguration(), semantics); + } + + public JavaCompileAction build() { + // TODO(bazel-team): all the params should be calculated before getting here, and the various + // aggregation code below should go away. + List<String> jcopts = new ArrayList<>(javacOpts); + JavaConfiguration javaConfiguration = configuration.getFragment(JavaConfiguration.class); + if (javaConfiguration.getJavaWarns().size() > 0) { + jcopts.add("-Xlint:" + Joiner.on(',').join(javaConfiguration.getJavaWarns())); + } + if (!bootclasspathEntries.isEmpty()) { + jcopts.add("-bootclasspath"); + jcopts.add( + Artifact.joinExecPaths(configuration.getHostPathSeparator(), bootclasspathEntries)); + } + List<String> internedJcopts = new ArrayList<>(); + for (String jcopt : jcopts) { + internedJcopts.add(StringCanonicalizer.intern(jcopt)); + } + + // Invariant: if strictJavaDeps is OFF, then directJars and + // dependencyArtifacts are ignored + if (strictJavaDeps == BuildConfiguration.StrictDepsMode.OFF) { + directJars.clear(); + compileTimeDependencyArtifacts.clear(); + } + + // Invariant: if experimental_java_classpath is not set to 'javabuilder', + // dependencyArtifacts are ignored + if (javaConfiguration.getReduceJavaClasspath() != JavaClasspathMode.JAVABUILDER) { + compileTimeDependencyArtifacts.clear(); + } + + if (paramFile == null) { + paramFile = analysisEnvironment.getDerivedArtifact( + ParameterFile.derivePath(outputJar.getRootRelativePath()), + configuration.getBinDirectory()); + } + + // ImmutableIterable is safe to use here because we know that neither of the components of + // the Iterable.concat() will change. Without ImmutableIterable, AbstractAction will + // waste memory by making a preventive copy of the iterable. + Iterable<Artifact> baseInputs = ImmutableIterable.from(Iterables.concat( + javabaseInputs, + bootclasspathEntries, + ImmutableList.of(paramFile))); + + Preconditions.checkState(javaExecutable != null, owner); + Preconditions.checkState(javaExecutable.isAbsolute() ^ !javabaseInputs.isEmpty(), + javaExecutable); + + Collection<Artifact> outputs; + ImmutableList.Builder<Artifact> outputsBuilder = ImmutableList.builder(); + outputsBuilder.add(outputJar); + if (metadata != null) { + outputsBuilder.add(metadata); + } + if (gensrcOutputJar != null) { + outputsBuilder.add(gensrcOutputJar); + } + if (outputDepsProto != null) { + outputsBuilder.add(outputDepsProto); + } + outputs = outputsBuilder.build(); + + CustomCommandLine.Builder paramFileContentsBuilder = javaCompileCommandLine( + semantics, + configuration, + classDirectory, + sourceGenDirectory, + tempDirectory, + outputJar, + gensrcOutputJar, + compressJar, + outputDepsProto, + classpathEntries, + processorPath, + langtoolsJar, + javaBuilderJar, + processorNames, + translations, + resources, + classpathResources, + sourceJars, + sourceFiles, + internedJcopts, + directJars, + strictJavaDeps, + compileTimeDependencyArtifacts, + ruleKind, + targetLabel); + semantics.buildJavaCommandLine(outputs, configuration, paramFileContentsBuilder); + CommandLine paramFileContents = paramFileContentsBuilder.build(); + Action parameterFileWriteAction = new ParameterFileWriteAction(owner, paramFile, + paramFileContents, ParameterFile.ParameterFileType.UNQUOTED, ISO_8859_1); + analysisEnvironment.registerAction(parameterFileWriteAction); + + CommandLine javaBuilderCommandLine = spawnCommandLine( + javaExecutable, + javaBuilderJar, + langtoolsJar, + paramFile, + javaConfiguration.getDefaultJavaBuilderJvmFlags()); + + return new JavaCompileAction(owner, + baseInputs, + outputs, + paramFileContents, + javaBuilderCommandLine, + classDirectory, + outputJar, + classpathEntries, + processorPath, + langtoolsJar, + javaBuilderJar, + processorNames, + translations, + resources, + classpathResources, + sourceJars, + sourceFiles, + internedJcopts, + directJars, + strictJavaDeps, + compileTimeDependencyArtifacts, + + semantics); + } + + public Builder setParameterFile(Artifact paramFile) { + this.paramFile = paramFile; + return this; + } + + public Builder setJavaExecutable(PathFragment javaExecutable) { + this.javaExecutable = javaExecutable; + return this; + } + + public Builder setJavaBaseInputs(Iterable<Artifact> javabaseInputs) { + this.javabaseInputs = ImmutableList.copyOf(javabaseInputs); + return this; + } + + public Builder setOutputJar(Artifact outputJar) { + this.outputJar = outputJar; + return this; + } + + public Builder setGensrcOutputJar(Artifact gensrcOutputJar) { + this.gensrcOutputJar = gensrcOutputJar; + return this; + } + + public Builder setOutputDepsProto(Artifact outputDepsProto) { + this.outputDepsProto = outputDepsProto; + return this; + } + + public Builder setMetadata(Artifact metadata) { + this.metadata = metadata; + return this; + } + + public Builder addSourceFile(Artifact sourceFile) { + sourceFiles.add(sourceFile); + return this; + } + + public Builder addSourceFiles(Collection<Artifact> sourceFiles) { + this.sourceFiles.addAll(sourceFiles); + return this; + } + + public Builder addSourceJars(Collection<Artifact> sourceJars) { + this.sourceJars.addAll(sourceJars); + return this; + } + + public Builder addResources(Collection<Artifact> resources) { + this.resources.addAll(resources); + return this; + } + + public Builder addClasspathResources(Collection<Artifact> classpathResources) { + this.classpathResources.addAll(classpathResources); + return this; + } + + public Builder addTranslations(Collection<Artifact> translations) { + this.translations.addAll(translations); + return this; + } + + /** + * Sets the strictness of Java dependency checking, see {@link + * com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode}. + */ + public Builder setStrictJavaDeps(BuildConfiguration.StrictDepsMode strictDeps) { + strictJavaDeps = strictDeps; + return this; + } + + /** + * Accumulates the given jar artifacts as being provided by direct dependencies. + */ + public Builder addDirectJars(Collection<Artifact> directJars) { + Iterables.addAll(this.directJars, directJars); + return this; + } + + public Builder addCompileTimeDependencyArtifacts(Collection<Artifact> dependencyArtifacts) { + Iterables.addAll(this.compileTimeDependencyArtifacts, dependencyArtifacts); + return this; + } + + public Builder setJavacOpts(Iterable<String> copts) { + this.javacOpts = ImmutableList.copyOf(copts); + return this; + } + + public Builder setCompressJar(boolean compressJar) { + this.compressJar = compressJar; + return this; + } + + public Builder setClasspathEntries(NestedSet<Artifact> classpathEntries) { + this.classpathEntries = classpathEntries; + return this; + } + + public Builder setBootclasspathEntries(Iterable<Artifact> bootclasspathEntries) { + this.bootclasspathEntries = ImmutableList.copyOf(bootclasspathEntries); + return this; + } + + public Builder setClassDirectory(PathFragment classDirectory) { + this.classDirectory = classDirectory; + return this; + } + + /** + * Sets the directory where source files generated by annotation processors should be stored. + */ + public Builder setSourceGenDirectory(PathFragment sourceGenDirectory) { + this.sourceGenDirectory = sourceGenDirectory; + return this; + } + + public Builder setTempDirectory(PathFragment tempDirectory) { + this.tempDirectory = tempDirectory; + return this; + } + + public Builder addProcessorPaths(Collection<Artifact> processorPaths) { + this.processorPath.addAll(processorPaths); + return this; + } + + public Builder addProcessorNames(Collection<String> processorNames) { + this.processorNames.addAll(processorNames); + return this; + } + + public Builder setLangtoolsJar(Artifact langtoolsJar) { + this.langtoolsJar = langtoolsJar; + return this; + } + + public Builder setJavaBuilderJar(Artifact javaBuilderJar) { + this.javaBuilderJar = javaBuilderJar; + return this; + } + + public Builder setRuleKind(String ruleKind) { + this.ruleKind = ruleKind; + return this; + } + + public Builder setTargetLabel(Label targetLabel) { + this.targetLabel = targetLabel; + return this; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java new file mode 100644 index 0000000..e1c6dc2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java
@@ -0,0 +1,260 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap.Builder; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.common.options.TriState; + +import java.util.List; + +/** + * A java compiler configuration containing the flags required for compilation. + */ +@Immutable +@SkylarkModule(name = "java_configuration", doc = "A java compiler configuration") +public final class JavaConfiguration extends Fragment { + /** + * Values for the --experimental_java_classpath option + */ + public static enum JavaClasspathMode { + /** Use full transitive classpaths, the default behavior. */ + OFF, + /** JavaBuilder computes the reduced classpath before invoking javac. */ + JAVABUILDER, + /** Blaze computes the reduced classpath before invoking JavaBuilder. */ + BLAZE + } + + private final ImmutableList<String> commandLineJavacFlags; + private final Label javaLauncherLabel; + private final Label javaBuilderTop; + private final ImmutableList<String> defaultJavaBuilderJvmOpts; + private final Label javaLangtoolsJar; + private final boolean useIjars; + private final boolean generateJavaDeps; + private final JavaClasspathMode experimentalJavaClasspath; + private final ImmutableList<String> javaWarns; + private final ImmutableList<String> defaultJvmFlags; + private final ImmutableList<String> checkedConstraints; + private final StrictDepsMode strictJavaDeps; + private final Label javacBootclasspath; + private final ImmutableList<String> javacOpts; + private final TriState bundleTranslations; + private final ImmutableList<Label> translationTargets; + private final String javaCpu; + + private final String cacheKey; + private Label javaToolchain; + + JavaConfiguration(boolean generateJavaDeps, + List<String> defaultJvmFlags, JavaOptions javaOptions, Label javaToolchain, String javaCpu, + ImmutableList<String> defaultJavaBuilderJvmOpts) throws InvalidConfigurationException { + this.commandLineJavacFlags = + ImmutableList.copyOf(JavaHelper.tokenizeJavaOptions(javaOptions.javacOpts)); + this.javaLauncherLabel = javaOptions.javaLauncher; + this.javaBuilderTop = javaOptions.javaBuilderTop; + this.defaultJavaBuilderJvmOpts = defaultJavaBuilderJvmOpts; + this.javaLangtoolsJar = javaOptions.javaLangtoolsJar; + this.useIjars = javaOptions.useIjars; + this.generateJavaDeps = generateJavaDeps; + this.experimentalJavaClasspath = javaOptions.experimentalJavaClasspath; + this.javaWarns = ImmutableList.copyOf(javaOptions.javaWarns); + this.defaultJvmFlags = ImmutableList.copyOf(defaultJvmFlags); + this.checkedConstraints = ImmutableList.copyOf(javaOptions.checkedConstraints); + this.strictJavaDeps = javaOptions.strictJavaDeps; + this.javacBootclasspath = javaOptions.javacBootclasspath; + this.javacOpts = ImmutableList.copyOf(javaOptions.javacOpts); + this.bundleTranslations = javaOptions.bundleTranslations; + this.javaCpu = javaCpu; + this.javaToolchain = javaToolchain; + + ImmutableList.Builder<Label> translationsBuilder = ImmutableList.builder(); + for (String s : javaOptions.translationTargets) { + try { + Label label = Label.parseAbsolute(s); + translationsBuilder.add(label); + } catch (SyntaxException e) { + throw new InvalidConfigurationException("Invalid translations target '" + s + "', make " + + "sure it uses correct absolute path syntax.", e); + } + } + this.translationTargets = translationsBuilder.build(); + + this.cacheKey = Joiner.on(" ").join(commandLineJavacFlags); + } + + @SkylarkCallable(name = "default_javac_flags", structField = true, + doc = "The default flags for the Java compiler.") + // TODO(bazel-team): this is the command-line passed options, we should remove from skylark + // probably. + public List<String> getDefaultJavacFlags() { + return commandLineJavacFlags; + } + + @Override + public String cacheKey() { + return cacheKey; + } + + @Override + public void reportInvalidOptions(EventHandler reporter, BuildOptions buildOptions) { + if ((bundleTranslations == TriState.YES) && translationTargets.isEmpty()) { + reporter.handle(Event.error("Translations enabled, but no message translations specified. " + + "Use '--message_translations' to select the message translations to use")); + } + } + + @Override + public void addGlobalMakeVariables(Builder<String, String> globalMakeEnvBuilder) { + globalMakeEnvBuilder.put("JAVA_TRANSLATIONS", buildTranslations() ? "1" : "0"); + globalMakeEnvBuilder.put("JAVA_CPU", javaCpu); + } + + /** + * Returns the Java cpu. + */ + public String getJavaCpu() { + return javaCpu; + } + + /** + * Returns the default javabuilder jar + */ + public Label getDefaultJavaBuilderJar() { + return javaBuilderTop; + } + + /** + * Returns the default JVM flags to be used when invoking javabuilder. + */ + public ImmutableList<String> getDefaultJavaBuilderJvmFlags() { + return defaultJavaBuilderJvmOpts; + } + + /** + * Returns the default java langtools jar + */ + public Label getDefaultJavaLangtoolsJar() { + return javaLangtoolsJar; + } + + /** + * Returns true iff Java compilation should use ijars. + */ + public boolean getUseIjars() { + return useIjars; + } + + /** + * Returns true iff dependency information is generated after compilation. + */ + public boolean getGenerateJavaDeps() { + return generateJavaDeps; + } + + public JavaClasspathMode getReduceJavaClasspath() { + return experimentalJavaClasspath; + } + + /** + * Returns the extra warnings enabled for Java compilation. + */ + public List<String> getJavaWarns() { + return javaWarns; + } + + public List<String> getDefaultJvmFlags() { + return defaultJvmFlags; + } + + public List<String> getCheckedConstraints() { + return checkedConstraints; + } + + public StrictDepsMode getStrictJavaDeps() { + return strictJavaDeps; + } + + public StrictDepsMode getFilteredStrictJavaDeps() { + StrictDepsMode strict = getStrictJavaDeps(); + switch (strict) { + case STRICT: + case DEFAULT: + return StrictDepsMode.ERROR; + default: // OFF, WARN, ERROR + return strict; + } + } + + /** + * @return proper label only if --java_launcher= is specified, otherwise null. + */ + public Label getJavaLauncherLabel() { + return javaLauncherLabel; + } + + public Label getJavacBootclasspath() { + return javacBootclasspath; + } + + public List<String> getJavacOpts() { + return javacOpts; + } + + @Override + public String getName() { + return "Java"; + } + + /** + * Returns the raw translation targets. + */ + public List<Label> getTranslationTargets() { + return translationTargets; + } + + /** + * Returns true if the we should build translations. + */ + public boolean buildTranslations() { + return (bundleTranslations != TriState.NO) && !translationTargets.isEmpty(); + } + + /** + * Returns whether translations were explicitly disabled. + */ + public boolean isTranslationsDisabled() { + return bundleTranslations == TriState.NO; + } + + /** + * Returns the label of the default java_toolchain rule + */ + public Label getToolchainLabel() { + return javaToolchain; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfigurationLoader.java new file mode 100644 index 0000000..53fdfdf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfigurationLoader.java
@@ -0,0 +1,76 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.RedirectChaser; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode; +import com.google.devtools.build.lib.syntax.Label; + +/** + * A loader that creates JavaConfiguration instances based on JavaBuilder configurations and + * command-line options. + */ +public class JavaConfigurationLoader implements ConfigurationFragmentFactory { + private final JavaCpuSupplier cpuSupplier; + + public JavaConfigurationLoader(JavaCpuSupplier cpuSupplier) { + this.cpuSupplier = cpuSupplier; + } + + @Override + public JavaConfiguration create(ConfigurationEnvironment env, BuildOptions buildOptions) + throws InvalidConfigurationException { + JavaOptions javaOptions = buildOptions.get(JavaOptions.class); + + Label javaToolchain = RedirectChaser.followRedirects(env, javaOptions.javaToolchain, + "java_toolchain"); + return create(javaOptions, javaToolchain, cpuSupplier.getJavaCpu(buildOptions, env)); + } + + @Override + public Class<? extends Fragment> creates() { + return JavaConfiguration.class; + } + + public JavaConfiguration create(JavaOptions javaOptions, Label javaToolchain, String javaCpu) + throws InvalidConfigurationException { + + boolean generateJavaDeps = javaOptions.javaDeps || + javaOptions.experimentalJavaClasspath != JavaClasspathMode.OFF; + + ImmutableList<String> defaultJavaBuilderJvmOpts = ImmutableList.<String>builder() + .addAll(getJavacJvmOptions()) + .addAll(JavaHelper.tokenizeJavaOptions(javaOptions.javaBuilderJvmOpts)) + .build(); + + return new JavaConfiguration(generateJavaDeps, javaOptions.jvmOpts, javaOptions, + javaToolchain, javaCpu, defaultJavaBuilderJvmOpts); + } + + /** + * This method returns the list of JVM options when invoking the java compiler. + * + * <p>TODO(bazel-team): Maybe we should put those options in the java_toolchain rule. + */ + protected ImmutableList<String> getJavacJvmOptions() { + return ImmutableList.of("-client"); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCpuSupplier.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCpuSupplier.java new file mode 100644 index 0000000..5492abf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCpuSupplier.java
@@ -0,0 +1,31 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; + +/** + * Determines the CPU to be used for Java compilation from the build options and the + * configuration environment. + */ +public interface JavaCpuSupplier { + /** + * Returns the Java CPU based on the buiold options and the configuration environment. + */ + String getJavaCpu(BuildOptions buildOptions, ConfigurationEnvironment env) + throws InvalidConfigurationException; +} \ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaExportsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaExportsProvider.java new file mode 100644 index 0000000..52857f1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaExportsProvider.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; + +/** + * The collection of labels of exported targets and artifacts reached via "exports" attribute + * transitively. + */ +@Immutable +public final class JavaExportsProvider implements TransitiveInfoProvider { + + private final NestedSet<Label> transitiveExports; + + public JavaExportsProvider(NestedSet<Label> transitiveExports) { + this.transitiveExports = transitiveExports; + } + + /** + * Returns the labels of exported targets and artifacts reached transitively through the "exports" + * attribute. + */ + public NestedSet<Label> getTransitiveExports() { + return transitiveExports; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHelper.java new file mode 100644 index 0000000..b2a7ca0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaHelper.java
@@ -0,0 +1,104 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.shell.ShellUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * Utility methods for use by Java-related parts of Bazel. + */ +// TODO(bazel-team): Merge with JavaUtil. +public abstract class JavaHelper { + + private JavaHelper() {} + + /** + * Returns the java launcher implementation for the given target, if any. + * A null return value means "use the JDK launcher". + */ + public static TransitiveInfoCollection launcherForTarget(JavaSemantics semantics, + RuleContext ruleContext) { + String launcher = filterLauncherForTarget(semantics, ruleContext); + return (launcher == null) ? null : ruleContext.getPrerequisite(launcher, Mode.TARGET); + } + + /** + * Returns the java launcher artifact for the given target, if any. + * A null return value means "use the JDK launcher". + */ + public static Artifact launcherArtifactForTarget(JavaSemantics semantics, + RuleContext ruleContext) { + String launcher = filterLauncherForTarget(semantics, ruleContext); + return (launcher == null) ? null : ruleContext.getPrerequisiteArtifact(launcher, Mode.TARGET); + } + + /** + * Control structure abstraction for safely extracting a prereq from the launcher attribute + * or --java_launcher flag. + */ + private static String filterLauncherForTarget(JavaSemantics semantics, RuleContext ruleContext) { + // BUILD rule "launcher" attribute + if (ruleContext.getRule().isAttrDefined("launcher", Type.LABEL) + && ruleContext.attributes().get("launcher", Type.LABEL) != null) { + if (ruleContext.attributes().get("launcher", Type.LABEL) + .equals(JavaSemantics.JDK_LAUNCHER_LABEL)) { + return null; + } + return "launcher"; + } + // Blaze flag --java_launcher + JavaConfiguration javaConfig = ruleContext.getFragment(JavaConfiguration.class); + if (ruleContext.getRule().isAttrDefined(":java_launcher", Type.LABEL) + && ((javaConfig.getJavaLauncherLabel() != null + && !javaConfig.getJavaLauncherLabel().equals(JavaSemantics.JDK_LAUNCHER_LABEL)) + || semantics.forceUseJavaLauncherTarget(ruleContext))) { + return ":java_launcher"; + } + return null; + } + + /** + * Javac options require special processing - People use them and expect the + * options to be tokenized. + */ + public static List<String> tokenizeJavaOptions(Iterable<String> inOpts) { + // Ideally, this would be in the options parser. Unfortunately, + // the options parser can't handle a converter that expands + // from a value X into a List<X> and allow-multiple at the + // same time. + List<String> result = new ArrayList<>(); + for (String current : inOpts) { + try { + ShellUtils.tokenize(result, current); + } catch (ShellUtils.TokenizationException ex) { + // Tokenization failed; this likely means that the user + // did not want tokenization to happen on his argument. + // (Any tokenization where we should produce an error + // has already been done by the shell that invoked + // blaze). Therefore, pass the argument through to + // the tool, so that we can see the original error. + result.add(current); + } + } + return result; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaImport.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaImport.java new file mode 100644 index 0000000..f978f98 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaImport.java
@@ -0,0 +1,201 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TopLevelArtifactProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.CcLinkParams; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsProvider; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore; +import com.google.devtools.build.lib.rules.cpp.CppCompilationContext; +import com.google.devtools.build.lib.rules.cpp.LinkerInput; +import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType; + +/** + * An implementation for the "java_import" rule. + */ +public class JavaImport implements RuleConfiguredTargetFactory { + private final JavaSemantics semantics; + + protected JavaImport(JavaSemantics semantics) { + this.semantics = semantics; + } + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + ImmutableList<Artifact> srcJars = ImmutableList.of(); + ImmutableList<Artifact> jars = collectJars(ruleContext); + Artifact srcJar = ruleContext.getPrerequisiteArtifact("srcjar", Mode.TARGET); + + if (ruleContext.hasErrors()) { + return null; + } + + ImmutableList<TransitiveInfoCollection> targets = ImmutableList.copyOf( + ruleContext.getPrerequisites("exports", Mode.TARGET)); + final JavaCommon common = new JavaCommon( + ruleContext, semantics, targets, targets, targets); + semantics.checkRule(ruleContext, common); + + // No need for javac options - no compilation happening here. + JavaCompilationHelper helper = new JavaCompilationHelper(ruleContext, semantics, + ImmutableList.<String>of(), new JavaTargetAttributes.Builder(semantics)); + ImmutableMap.Builder<Artifact, Artifact> compilationToRuntimeJarMap = ImmutableMap.builder(); + ImmutableList<Artifact> interfaceJars = + processWithIjar(jars, helper, compilationToRuntimeJarMap); + + common.setJavaCompilationArtifacts(collectJavaArtifacts(jars, interfaceJars)); + + CppCompilationContext transitiveCppDeps = common.collectTransitiveCppDeps(); + NestedSet<LinkerInput> transitiveJavaNativeLibraries = + common.collectTransitiveJavaNativeLibraries(); + + JavaCompilationArgs javaCompilationArgs = common.collectJavaCompilationArgs( + false, common.isNeverLink(), compilationArgsFromSources()); + JavaCompilationArgs recursiveJavaCompilationArgs = common.collectJavaCompilationArgs( + true, common.isNeverLink(), compilationArgsFromSources()); + NestedSet<Artifact> transitiveJavaSourceJars = + collectTransitiveJavaSourceJars(ruleContext, srcJar); + if (srcJar != null) { + srcJars = ImmutableList.of(srcJar); + } + + // The "neverlink" attribute is transitive, so if it is enabled, we don't add any + // runfiles from this target or its dependencies. + Runfiles runfiles = common.isNeverLink() ? + Runfiles.EMPTY : + new Runfiles.Builder() + // add the jars to the runfiles + .addArtifacts(common.getJavaCompilationArtifacts().getRuntimeJars()) + .addTargets(targets, RunfilesProvider.DEFAULT_RUNFILES) + .addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES) + .addTargets(targets, JavaRunfilesProvider.TO_RUNFILES) + .add(ruleContext, JavaRunfilesProvider.TO_RUNFILES) + .build(); + + CcLinkParamsStore ccLinkParamsStore = new CcLinkParamsStore() { + @Override + protected void collect(CcLinkParams.Builder builder, boolean linkingStatically, + boolean linkShared) { + Iterable<? extends TransitiveInfoCollection> deps = + common.targetsTreatedAsDeps(ClasspathType.BOTH); + builder.addTransitiveTargets(deps); + builder.addTransitiveLangTargets(deps, JavaCcLinkParamsProvider.TO_LINK_PARAMS); + } + }; + RuleConfiguredTargetBuilder ruleBuilder = + new RuleConfiguredTargetBuilder(ruleContext); + NestedSetBuilder<Artifact> filesBuilder = NestedSetBuilder.stableOrder(); + filesBuilder.addAll(jars); + + semantics.addProviders( + ruleContext, common, ImmutableList.<String>of(), null, + srcJar, null, compilationToRuntimeJarMap.build(), helper, filesBuilder, ruleBuilder); + + NestedSet<Artifact> filesToBuild = filesBuilder.build(); + + common.addTransitiveInfoProviders(ruleBuilder, filesToBuild, null); + return ruleBuilder + .setFilesToBuild(filesToBuild) + .add(JavaNeverlinkInfoProvider.class, new JavaNeverlinkInfoProvider(common.isNeverLink())) + .add(RunfilesProvider.class, RunfilesProvider.simple(runfiles)) + .add(CcLinkParamsProvider.class, new CcLinkParamsProvider(ccLinkParamsStore)) + .add(JavaCompilationArgsProvider.class, new JavaCompilationArgsProvider( + javaCompilationArgs, recursiveJavaCompilationArgs)) + .add(JavaNativeLibraryProvider.class, new JavaNativeLibraryProvider( + transitiveJavaNativeLibraries)) + .add(CppCompilationContext.class, transitiveCppDeps) + .add(JavaSourceJarsProvider.class, new JavaSourceJarsProvider( + transitiveJavaSourceJars, srcJars)) + .add(TopLevelArtifactProvider.class, new TopLevelArtifactProvider( + JavaSemantics.SOURCE_JARS_OUTPUT_GROUP, transitiveJavaSourceJars)) + .build(); + } + + private NestedSet<Artifact> collectTransitiveJavaSourceJars(RuleContext ruleContext, + Artifact srcJar) { + NestedSetBuilder<Artifact> transitiveJavaSourceJarBuilder = + NestedSetBuilder.stableOrder(); + if (srcJar != null) { + transitiveJavaSourceJarBuilder.add(srcJar); + } + for (JavaSourceJarsProvider other : + ruleContext.getPrerequisites("exports", Mode.TARGET, JavaSourceJarsProvider.class)) { + transitiveJavaSourceJarBuilder.addTransitive(other.getTransitiveSourceJars()); + } + return transitiveJavaSourceJarBuilder.build(); + } + + private JavaCompilationArtifacts collectJavaArtifacts( + ImmutableList<Artifact> jars, + ImmutableList<Artifact> interfaceJars) { + JavaCompilationArtifacts.Builder javaArtifactsBuilder = new JavaCompilationArtifacts.Builder(); + javaArtifactsBuilder.addRuntimeJars(jars); + // interfaceJars Artifacts have proper owner labels + javaArtifactsBuilder.addCompileTimeJars(interfaceJars); + return javaArtifactsBuilder.build(); + } + + private ImmutableList<Artifact> collectJars(RuleContext ruleContext) { + ImmutableList.Builder<Artifact> jarsBuilder = ImmutableList.builder(); + for (TransitiveInfoCollection info : ruleContext.getPrerequisites("jars", Mode.TARGET)) { + if (info.getProvider(JavaCompilationArgsProvider.class) != null) { + ruleContext.attributeError("jars", "should not refer to Java rules"); + } + for (Artifact jar : info.getProvider(FileProvider.class).getFilesToBuild()) { + if (!JavaSemantics.JAR.matches(jar.getFilename())) { + ruleContext.attributeError("jars", jar.getFilename() + " is not a .jar file"); + } else { + jarsBuilder.add(jar); + } + } + } + return jarsBuilder.build(); + } + + private ImmutableList<Artifact> processWithIjar(ImmutableList<Artifact> jars, + JavaCompilationHelper helper, + ImmutableMap.Builder<Artifact, Artifact> compilationToRuntimeJarMap) { + ImmutableList.Builder<Artifact> interfaceJarsBuilder = ImmutableList.builder(); + for (Artifact jar : jars) { + Artifact ijar = helper.createIjarAction(jar, true); + interfaceJarsBuilder.add(ijar); + compilationToRuntimeJarMap.put(ijar, jar); + } + return interfaceJarsBuilder.build(); + } + + private Iterable<SourcesJavaCompilationArgsProvider> compilationArgsFromSources() { + return ImmutableList.of(); + } + + private ImmutableList<String> getJavaConstraints(RuleContext ruleContext) { + return ImmutableList.copyOf(ruleContext.attributes().get("constraints", Type.STRING_LIST)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaImportBaseRule.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaImportBaseRule.java new file mode 100644 index 0000000..b153b58 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaImportBaseRule.java
@@ -0,0 +1,91 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; + +/** + * A base rule for building the java_import rule. + */ +@BlazeRule(name = "$java_import_base", + type = RuleClassType.ABSTRACT, + ancestors = { BaseRuleClasses.RuleBase.class }) +public class JavaImportBaseRule implements RuleDefinition { + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + .add(attr(":host_jdk", LABEL) + .cfg(HOST) + .value(JavaSemantics.HOST_JDK)) + /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(jars) --> + The list of JAR files provided to Java targets that depend on this target. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("jars", LABEL_LIST) + .mandatory() + .nonEmpty() + .allowedFileTypes(JavaSemantics.JAR)) + /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(srcjar) --> + A JAR file that contains source code for the compiled JAR files. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("srcjar", LABEL) + .allowedFileTypes(JavaSemantics.SOURCE_JAR, JavaSemantics.JAR) + .direct_compile_time_input()) + .removeAttribute("deps") // only exports are allowed; nothing is compiled + /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(neverlink) --> + Only use this library for compilation and not at runtime. + ${SYNOPSIS} + Useful if the library will be provided by the runtime environment + during execution. Examples of libraries like this are IDE APIs + for IDE plug-ins or <code>tools.jar</code> for anything running on + a standard JDK. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("neverlink", BOOLEAN).value(false)) + /* <!-- #BLAZE_RULE(java_import).ATTRIBUTE(constraints) --> + Extra constraints imposed on this rule as a Java library. + ${SYNOPSIS} + See <a href="#java_library.constraints">java_library.constraints</a>. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("constraints", STRING_LIST) + .orderIndependent() + .nonconfigurable("used in Attribute.validityPredicate implementations (loading time)")) + .build(); + } +} +/*<!-- #BLAZE_RULE (NAME = java_import, TYPE = LIBRARY, FAMILY = Java) --> + +${ATTRIBUTE_SIGNATURE} + + <p>This rule allows the use of precompiled JAR files as libraries for + <code><a href="#java_library">java_library</a></code> rules.</p> + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibrary.java new file mode 100644 index 0000000..1831ef0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibrary.java
@@ -0,0 +1,244 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TopLevelArtifactProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.CcLinkParams; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsProvider; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore; +import com.google.devtools.build.lib.rules.cpp.CppCompilationContext; +import com.google.devtools.build.lib.rules.cpp.LinkerInput; +import com.google.devtools.build.lib.rules.java.JavaCompilationArgs.ClasspathType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * Implementation for the java_library rule. + */ +public class JavaLibrary implements RuleConfiguredTargetFactory { + private final JavaSemantics semantics; + + protected JavaLibrary(JavaSemantics semantics) { + this.semantics = semantics; + } + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + JavaCommon common = new JavaCommon(ruleContext, semantics); + RuleConfiguredTargetBuilder builder = init(ruleContext, common); + return builder != null ? builder.build() : null; + } + + public RuleConfiguredTargetBuilder init(RuleContext ruleContext, final JavaCommon common) { + common.initializeJavacOpts(); + JavaTargetAttributes.Builder attributesBuilder = common.initCommon(); + + // Collect the transitive dependencies. + JavaCompilationHelper helper = new JavaCompilationHelper( + ruleContext, semantics, common.getJavacOpts(), attributesBuilder); + helper.addLibrariesToAttributes(common.targetsTreatedAsDeps(ClasspathType.COMPILE_ONLY)); + helper.addProvidersToAttributes(common.compilationArgsFromSources(), common.isNeverLink()); + + if (ruleContext.hasErrors()) { + return null; + } + + semantics.checkRule(ruleContext, common); + + JavaCompilationArtifacts.Builder javaArtifactsBuilder = new JavaCompilationArtifacts.Builder(); + + if (ruleContext.hasErrors()) { + common.setJavaCompilationArtifacts(JavaCompilationArtifacts.EMPTY); + return null; + } + + JavaConfiguration javaConfig = ruleContext.getFragment(JavaConfiguration.class); + NestedSetBuilder<Artifact> filesBuilder = NestedSetBuilder.stableOrder(); + + JavaTargetAttributes attributes = helper.getAttributes(); + if (attributes.hasJarFiles()) { + // This rule is repackaging some source jars as a java library. + Set<Artifact> jarFiles = attributes.getJarFiles(); + javaArtifactsBuilder.addRuntimeJars(jarFiles); + javaArtifactsBuilder.addCompileTimeJars(attributes.getCompileTimeJarFiles()); + + filesBuilder.addAll(jarFiles); + } + if (attributes.hasMessages()) { + helper.addTranslations(semantics.translate(ruleContext, javaConfig, + attributes.getMessages())); + } + + ruleContext.checkSrcsSamePackage(true); + + Artifact jar = null; + + Artifact srcJar = ruleContext.getImplicitOutputArtifact( + JavaSemantics.JAVA_LIBRARY_SOURCE_JAR); + + Artifact classJar = ruleContext.getImplicitOutputArtifact( + JavaSemantics.JAVA_LIBRARY_CLASS_JAR); + + if (attributes.hasSourceFiles() || attributes.hasSourceJars() || attributes.hasResources() + || attributes.hasMessages()) { + // We only want to add a jar to the classpath of a dependent rule if it has content. + javaArtifactsBuilder.addRuntimeJar(classJar); + jar = classJar; + } + + filesBuilder.add(classJar); + + // The gensrcJar is only created if the target uses annotation processing. Otherwise, + // it is null, and the source jar action will not depend on the compile action. + Artifact gensrcJar = helper.createGensrcJar(classJar); + + Artifact outputDepsProto = helper.createOutputDepsProtoArtifact(classJar, javaArtifactsBuilder); + + helper.createCompileActionWithInstrumentation(classJar, gensrcJar, outputDepsProto, + javaArtifactsBuilder); + helper.createSourceJarAction(srcJar, gensrcJar); + + if ((attributes.hasSourceFiles() || attributes.hasSourceJars()) && jar != null) { + helper.createCompileTimeJarAction(jar, outputDepsProto, + javaArtifactsBuilder); + } + + common.setJavaCompilationArtifacts(javaArtifactsBuilder.build()); + common.setClassPathFragment(new ClasspathConfiguredFragment( + common.getJavaCompilationArtifacts(), attributes, common.isNeverLink())); + CppCompilationContext transitiveCppDeps = common.collectTransitiveCppDeps(); + + NestedSet<Artifact> transitiveSourceJars = common.collectTransitiveSourceJars(srcJar); + + // If sources are empty, treat this library as a forwarding node for dependencies. + JavaCompilationArgs javaCompilationArgs = common.collectJavaCompilationArgs( + false, common.isNeverLink(), common.compilationArgsFromSources()); + JavaCompilationArgs recursiveJavaCompilationArgs = common.collectJavaCompilationArgs( + true, common.isNeverLink(), common.compilationArgsFromSources()); + NestedSet<Artifact> compileTimeJavaDepArtifacts = common.collectCompileTimeDependencyArtifacts( + common.getJavaCompilationArtifacts().getCompileTimeDependencyArtifact()); + NestedSet<Artifact> runTimeJavaDepArtifacts = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + NestedSet<LinkerInput> transitiveJavaNativeLibraries = + common.collectTransitiveJavaNativeLibraries(); + + ImmutableList<String> exportedProcessorClasses = ImmutableList.of(); + NestedSet<Artifact> exportedProcessorClasspath = + NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); + ImmutableList.Builder<String> processorClasses = ImmutableList.builder(); + NestedSetBuilder<Artifact> processorClasspath = NestedSetBuilder.naiveLinkOrder(); + for (JavaPluginInfoProvider provider : Iterables.concat( + common.getPluginInfoProvidersForAttribute("exported_plugins", Mode.HOST), + common.getPluginInfoProvidersForAttribute("exports", Mode.TARGET))) { + processorClasses.addAll(provider.getProcessorClasses()); + processorClasspath.addTransitive(provider.getProcessorClasspath()); + } + exportedProcessorClasses = processorClasses.build(); + exportedProcessorClasspath = processorClasspath.build(); + + CcLinkParamsStore ccLinkParamsStore = new CcLinkParamsStore() { + @Override + protected void collect(CcLinkParams.Builder builder, boolean linkingStatically, + boolean linkShared) { + Iterable<? extends TransitiveInfoCollection> deps = + common.targetsTreatedAsDeps(ClasspathType.BOTH); + builder.addTransitiveTargets(deps); + builder.addTransitiveLangTargets(deps, JavaCcLinkParamsProvider.TO_LINK_PARAMS); + } + }; + + // The "neverlink" attribute is transitive, so we don't add any + // runfiles from this target or its dependencies. + Runfiles runfiles = Runfiles.EMPTY; + if (!common.isNeverLink()) { + Runfiles.Builder runfilesBuilder = new Runfiles.Builder().addArtifacts( + common.getJavaCompilationArtifacts().getRuntimeJars()); + + + runfilesBuilder.addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES); + runfilesBuilder.add(ruleContext, JavaRunfilesProvider.TO_RUNFILES); + + List<TransitiveInfoCollection> depsForRunfiles = new ArrayList<>(); + if (ruleContext.getRule().isAttrDefined("runtime_deps", Type.LABEL_LIST)) { + depsForRunfiles.addAll(ruleContext.getPrerequisites("runtime_deps", Mode.TARGET)); + } + if (ruleContext.getRule().isAttrDefined("exports", Type.LABEL_LIST)) { + depsForRunfiles.addAll(ruleContext.getPrerequisites("exports", Mode.TARGET)); + } + + runfilesBuilder.addTargets(depsForRunfiles, RunfilesProvider.DEFAULT_RUNFILES); + runfilesBuilder.addTargets(depsForRunfiles, JavaRunfilesProvider.TO_RUNFILES); + + TransitiveInfoCollection launcher = JavaHelper.launcherForTarget(semantics, ruleContext); + if (launcher != null) { + runfilesBuilder.addTarget(launcher, RunfilesProvider.DATA_RUNFILES); + } + + semantics.addRunfilesForLibrary(ruleContext, runfilesBuilder); + runfiles = runfilesBuilder.build(); + } + + RuleConfiguredTargetBuilder builder = + new RuleConfiguredTargetBuilder(ruleContext); + + semantics.addProviders( + ruleContext, common, ImmutableList.<String>of(), classJar, srcJar, gensrcJar, + ImmutableMap.<Artifact, Artifact>of(), helper, filesBuilder, builder); + + NestedSet<Artifact> filesToBuild = filesBuilder.build(); + common.addTransitiveInfoProviders(builder, filesToBuild, classJar); + + builder + .add(RunfilesProvider.class, RunfilesProvider.simple(runfiles)) + .setFilesToBuild(filesToBuild) + .add(JavaNeverlinkInfoProvider.class, new JavaNeverlinkInfoProvider(common.isNeverLink())) + .add(CppCompilationContext.class, transitiveCppDeps) + .add(JavaCompilationArgsProvider.class, new JavaCompilationArgsProvider( + javaCompilationArgs, recursiveJavaCompilationArgs, + compileTimeJavaDepArtifacts, runTimeJavaDepArtifacts)) + .add(CcLinkParamsProvider.class, new CcLinkParamsProvider(ccLinkParamsStore)) + .add(JavaNativeLibraryProvider.class, new JavaNativeLibraryProvider( + transitiveJavaNativeLibraries)) + .add(JavaSourceJarsProvider.class, new JavaSourceJarsProvider( + transitiveSourceJars, ImmutableList.of(srcJar))) + .add(TopLevelArtifactProvider.class, new TopLevelArtifactProvider( + JavaSemantics.SOURCE_JARS_OUTPUT_GROUP, transitiveSourceJars)) + // TODO(bazel-team): this should only happen for java_plugin + .add(JavaPluginInfoProvider.class, new JavaPluginInfoProvider( + exportedProcessorClasses, exportedProcessorClasspath)); + + if (ruleContext.hasErrors()) { + return null; + } + + return builder; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibraryHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibraryHelper.java new file mode 100644 index 0000000..a28d7fd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaLibraryHelper.java
@@ -0,0 +1,382 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import static com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode.OFF; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.UnmodifiableIterator; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.cpp.CcLinkParams.Builder; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore; +import com.google.devtools.build.lib.rules.cpp.CcSpecificLinkParamsProvider; +import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * A class to create Java compile actions in a way that is consistent with java_library. Rules that + * generate source files and emulate java_library on top of that should use this class + * instead of the lower-level API in JavaCompilationHelper. + * + * <p>Rules that want to use this class are required to have an implicit dependency on the + * Java compiler. + */ +public final class JavaLibraryHelper { + /** + * Function for extracting the {@link JavaCompilationArgs} - note that it also handles .jar files. + */ + private static final Function<TransitiveInfoCollection, JavaCompilationArgsProvider> + TO_COMPILATION_ARGS = new Function<TransitiveInfoCollection, JavaCompilationArgsProvider>() { + @Override + public JavaCompilationArgsProvider apply(TransitiveInfoCollection target) { + return forTarget(target); + } + }; + + /** + * Contains the providers as well as the compilation outputs. + */ + public static final class Info { + private final Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers; + private final JavaCompilationArtifacts compilationArtifacts; + + private Info(Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers, + JavaCompilationArtifacts compilationArtifacts) { + this.providers = Collections.unmodifiableMap(providers); + this.compilationArtifacts = compilationArtifacts; + } + + public Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> getProviders() { + return providers; + } + + public JavaCompilationArtifacts getCompilationArtifacts() { + return compilationArtifacts; + } + } + + private final RuleContext ruleContext; + private final BuildConfiguration configuration; + + private Artifact output; + private final List<Artifact> sourceJars = new ArrayList<>(); + /** + * Contains all the dependencies; these are treated as both compile-time and runtime dependencies. + * Some of these may not be complete configured targets; for backwards compatibility with some + * existing code, we sometimes only have pretend dependencies that only have a single {@link + * JavaCompilationArgsProvider}. + */ + private final List<TransitiveInfoCollection> deps = new ArrayList<>(); + private ImmutableList<String> javacOpts = ImmutableList.of(); + + private StrictDepsMode strictDepsMode = StrictDepsMode.OFF; + private JavaClasspathMode classpathMode = JavaClasspathMode.OFF; + private boolean emitProviders = true; + + public JavaLibraryHelper(RuleContext ruleContext) { + this.ruleContext = ruleContext; + this.configuration = ruleContext.getConfiguration(); + this.classpathMode = ruleContext.getFragment(JavaConfiguration.class).getReduceJavaClasspath(); + } + + /** + * Sets the final output jar; if this is not set, then the {@link #build} method throws an {@link + * IllegalStateException}. Note that this class may generate not just the output itself, but also + * a number of additional intermediate files and outputs. + */ + public JavaLibraryHelper setOutput(Artifact output) { + this.output = output; + return this; + } + + /** + * Adds the given source jars. Any .java files in these jars will be compiled. + */ + public JavaLibraryHelper addSourceJars(Iterable<Artifact> sourceJars) { + Iterables.addAll(this.sourceJars, sourceJars); + return this; + } + + /** + * Adds the given source jars. Any .java files in these jars will be compiled. + */ + public JavaLibraryHelper addSourceJars(Artifact... sourceJars) { + return this.addSourceJars(Arrays.asList(sourceJars)); + } + + /** + * Adds the given compilation args as deps. Avoid this method, and prefer {@link #addDeps} + * instead; this method only exists for backward compatibility and may be removed at any time. + */ + public JavaLibraryHelper addProcessedDeps(JavaCompilationArgs... deps) { + for (JavaCompilationArgs dep : deps) { + this.deps.add(toTransitiveInfoCollection(dep)); + } + return this; + } + + private static TransitiveInfoCollection toTransitiveInfoCollection( + final JavaCompilationArgs args) { + return new TransitiveInfoCollection() { + @Override + public <P extends TransitiveInfoProvider> P getProvider(Class<P> provider) { + if (JavaCompilationArgsProvider.class.equals(provider)) { + return provider.cast(new JavaCompilationArgsProvider(args, args)); + } + return null; + } + + @Override + public Label getLabel() { + throw new UnsupportedOperationException(); + } + + @Override + public BuildConfiguration getConfiguration() { + throw new UnsupportedOperationException(); + } + + @Override + public Object get(String providerKey) { + throw new UnsupportedOperationException(); + } + + @Override + public UnmodifiableIterator<TransitiveInfoProvider> iterator() { + throw new UnsupportedOperationException(); + } + }; + } + + /** + * Adds the given targets as deps. These are used as both compile-time and runtime dependencies. + */ + public JavaLibraryHelper addDeps(Iterable<? extends TransitiveInfoCollection> deps) { + for (TransitiveInfoCollection dep : deps) { + Preconditions.checkArgument(dep.getConfiguration() == null + || dep.getConfiguration().equals(configuration)); + this.deps.add(dep); + } + return this; + } + + /** + * Sets the compiler options. + */ + public JavaLibraryHelper setJavacOpts(Iterable<String> javacOpts) { + this.javacOpts = ImmutableList.copyOf(javacOpts); + return this; + } + + /** + * Sets the mode that determines how strictly dependencies are checked. + */ + public JavaLibraryHelper setStrictDepsMode(StrictDepsMode strictDepsMode) { + this.strictDepsMode = strictDepsMode; + return this; + } + + /** + * Disables all providers, i.e., the resulting {@link Info} object will not contain any providers. + * Avoid this method - having this class compute the providers ensures consistency among all + * clients of this code. + */ + public JavaLibraryHelper noProviders() { + this.emitProviders = false; + return this; + } + + /** + * Creates the compile actions and providers. + */ + public Info build(JavaSemantics semantics) { + Preconditions.checkState(output != null, "must have an output file; use setOutput()"); + JavaTargetAttributes.Builder attributes = new JavaTargetAttributes.Builder(semantics); + attributes.addSourceJars(sourceJars); + addDepsToAttributes(attributes); + attributes.setStrictJavaDeps(strictDepsMode); + attributes.setRuleKind(ruleContext.getRule().getRuleClass()); + attributes.setTargetLabel(ruleContext.getLabel()); + + if (isStrict() && classpathMode != JavaClasspathMode.OFF) { + addDependencyArtifactsToAttributes(attributes); + } + + JavaCompilationArtifacts.Builder artifactsBuilder = new JavaCompilationArtifacts.Builder(); + JavaCompilationHelper helper = + new JavaCompilationHelper(ruleContext, semantics, javacOpts, attributes); + Artifact outputDepsProto = helper.createOutputDepsProtoArtifact(output, artifactsBuilder); + helper.createCompileAction(output, null, outputDepsProto, null); + helper.createCompileTimeJarAction(output, outputDepsProto, artifactsBuilder); + artifactsBuilder.addRuntimeJar(output); + JavaCompilationArtifacts compilationArtifacts = artifactsBuilder.build(); + + Map<Class<? extends TransitiveInfoProvider>, TransitiveInfoProvider> providers = + new LinkedHashMap<>(); + if (emitProviders) { + providers.put(JavaCompilationArgsProvider.class, + collectJavaCompilationArgs(compilationArtifacts)); + providers.put(JavaSourceJarsProvider.class, + new JavaSourceJarsProvider(collectTransitiveJavaSourceJars(), sourceJars)); + providers.put(JavaRunfilesProvider.class, collectJavaRunfiles(compilationArtifacts)); + providers.put(JavaCcLinkParamsProvider.class, + new JavaCcLinkParamsProvider(createJavaCcLinkParamsStore())); + } + return new Info(providers, compilationArtifacts); + } + + private void addDepsToAttributes(JavaTargetAttributes.Builder attributes) { + NestedSet<Artifact> directJars = null; + if (isStrict()) { + directJars = getNonRecursiveCompileTimeJarsFromDeps(); + if (directJars != null) { + attributes.addDirectCompileTimeClassPathEntries(directJars); + attributes.addDirectJars(directJars); + } + } + + JavaCompilationArgs args = JavaCompilationArgs.builder() + .addTransitiveDependencies(transformDeps(), true).build(); + attributes.addCompileTimeClassPathEntries(args.getCompileTimeJars()); + attributes.addRuntimeClassPathEntries(args.getRuntimeJars()); + attributes.addInstrumentationMetadataEntries(args.getInstrumentationMetadata()); + } + + private NestedSet<Artifact> getNonRecursiveCompileTimeJarsFromDeps() { + JavaCompilationArgs.Builder builder = JavaCompilationArgs.builder(); + builder.addTransitiveDependencies(transformDeps(), false); + return builder.build().getCompileTimeJars(); + } + + private void addDependencyArtifactsToAttributes(JavaTargetAttributes.Builder attributes) { + NestedSetBuilder<Artifact> compileTimeBuilder = NestedSetBuilder.stableOrder(); + NestedSetBuilder<Artifact> runTimeBuilder = NestedSetBuilder.stableOrder(); + for (JavaCompilationArgsProvider dep : transformDeps()) { + compileTimeBuilder.addTransitive(dep.getCompileTimeJavaDependencyArtifacts()); + runTimeBuilder.addTransitive(dep.getRunTimeJavaDependencyArtifacts()); + } + attributes.addCompileTimeDependencyArtifacts(compileTimeBuilder.build()); + attributes.addRuntimeDependencyArtifacts(runTimeBuilder.build()); + } + + private Iterable<JavaCompilationArgsProvider> transformDeps() { + return Iterables.transform(deps, TO_COMPILATION_ARGS); + } + + private static JavaCompilationArgsProvider forTarget(TransitiveInfoCollection target) { + if (target.getProvider(JavaCompilationArgsProvider.class) != null) { + // If the target has JavaCompilationArgs, we use those. + return target.getProvider(JavaCompilationArgsProvider.class); + } else { + // Otherwise we look for any jar files. It would be good to remove this, and require + // intermediate java_import rules in these cases. + NestedSet<Artifact> filesToBuild = + target.getProvider(FileProvider.class).getFilesToBuild(); + final List<Artifact> jars = new ArrayList<>(); + Iterables.addAll(jars, FileType.filter(filesToBuild, JavaSemantics.JAR)); + JavaCompilationArgs args = JavaCompilationArgs.builder() + .addCompileTimeJars(jars) + .addRuntimeJars(jars) + .build(); + return new JavaCompilationArgsProvider(args, args); + } + } + + private boolean isStrict() { + return strictDepsMode != OFF; + } + + private JavaCompilationArgsProvider collectJavaCompilationArgs( + JavaCompilationArtifacts compilationArtifacts) { + JavaCompilationArgs javaCompilationArgs = + collectJavaCompilationArgs(compilationArtifacts, false); + JavaCompilationArgs recursiveJavaCompilationArgs = + collectJavaCompilationArgs(compilationArtifacts, true); + return new JavaCompilationArgsProvider(javaCompilationArgs, recursiveJavaCompilationArgs); + } + + /** + * Get compilation arguments for java compilation action. + * + * @param recursive a boolean specifying whether to get transitive + * dependencies + * @return java compilation args + */ + private JavaCompilationArgs collectJavaCompilationArgs( + JavaCompilationArtifacts compilationArtifacts, boolean recursive) { + return JavaCompilationArgs.builder() + .merge(compilationArtifacts) + .addTransitiveDependencies(transformDeps(), recursive) + .build(); + } + + private NestedSet<Artifact> collectTransitiveJavaSourceJars() { + NestedSetBuilder<Artifact> transitiveJavaSourceJarBuilder = + NestedSetBuilder.<Artifact>stableOrder(); + transitiveJavaSourceJarBuilder.addAll(sourceJars); + for (JavaSourceJarsProvider other : ruleContext.getPrerequisites( + "deps", Mode.TARGET, JavaSourceJarsProvider.class)) { + transitiveJavaSourceJarBuilder.addTransitive(other.getTransitiveSourceJars()); + } + return transitiveJavaSourceJarBuilder.build(); + } + + private JavaRunfilesProvider collectJavaRunfiles( + JavaCompilationArtifacts javaCompilationArtifacts) { + Runfiles runfiles = new Runfiles.Builder() + // Compiled templates as well, for API. + .addArtifacts(javaCompilationArtifacts.getRuntimeJars()) + .addTargets(deps, JavaRunfilesProvider.TO_RUNFILES) + .build(); + return new JavaRunfilesProvider(runfiles); + } + + private CcLinkParamsStore createJavaCcLinkParamsStore() { + return new CcLinkParamsStore() { + @Override + protected void collect(Builder builder, boolean linkingStatically, boolean linkShared) { + builder.addTransitiveLangTargets( + deps, + JavaCcLinkParamsProvider.TO_LINK_PARAMS); + builder.addTransitiveTargets(deps); + // TODO(bazel-team): This may need to be optional for some clients of this class. + builder.addTransitiveLangTargets( + deps, + CcSpecificLinkParamsProvider.TO_LINK_PARAMS); + } + }; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaNativeLibraryProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaNativeLibraryProvider.java new file mode 100644 index 0000000..8be42c0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaNativeLibraryProvider.java
@@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.LinkerInput; + +/** + * A target that provides native libraries in the transitive closure of its deps that are needed for + * executing Java code. + */ +@Immutable +public final class JavaNativeLibraryProvider implements TransitiveInfoProvider { + + private final NestedSet<LinkerInput> transitiveJavaNativeLibraries; + + public JavaNativeLibraryProvider( + NestedSet<LinkerInput> transitiveJavaNativeLibraries) { + this.transitiveJavaNativeLibraries = transitiveJavaNativeLibraries; + } + + /** + * Collects native libraries in the transitive closure of its deps that are needed for executing + * Java code. + */ + public NestedSet<LinkerInput> getTransitiveJavaNativeLibraries() { + return transitiveJavaNativeLibraries; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaNeverlinkInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaNeverlinkInfoProvider.java new file mode 100644 index 0000000..75b36c1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaNeverlinkInfoProvider.java
@@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A {@link TransitiveInfoProvider} that provides information about whether a Java archive + * is neverlink. + */ +@Immutable +public final class JavaNeverlinkInfoProvider implements TransitiveInfoProvider { + private final boolean isNeverLink; + + public JavaNeverlinkInfoProvider(boolean isNeverLink) { + this.isNeverLink = isNeverLink; + } + + public boolean isNeverlink() { + return isNeverLink; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java new file mode 100644 index 0000000..f7ef0c7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaOptions.java
@@ -0,0 +1,350 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.LabelConverter; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsConverter; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.StrictDepsMode; +import com.google.devtools.build.lib.analysis.config.DefaultsPackage; +import com.google.devtools.build.lib.analysis.config.FragmentOptions; +import com.google.devtools.build.lib.rules.java.JavaConfiguration.JavaClasspathMode; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.common.options.Converters.StringSetConverter; +import com.google.devtools.common.options.EnumConverter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.TriState; + +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Command-line options for building Java targets + */ +public class JavaOptions extends FragmentOptions { + // Defaults value for options + static final String DEFAULT_LANGTOOLS_BOOTCLASSPATH = "//tools/jdk:bootclasspath"; + static final String DEFAULT_LANGTOOLS = "//tools/jdk:langtools"; + static final String DEFAULT_JAVABUILDER = "//tools:java/JavaBuilder_deploy.jar"; + static final String DEFAULT_SINGLEJAR = "//tools:java/SingleJar_deploy.jar"; + static final String DEFAULT_JAVABASE = "//tools/jdk:jdk"; + static final String DEFAULT_IJAR = "//tools:java/ijar"; + static final String DEFAULT_TOOLCHAIN = "//tools/jdk:toolchain"; + + /** + * Converter for the --javawarn option. + */ + public static class JavacWarnConverter extends StringSetConverter { + public JavacWarnConverter() { + super("all", + "cast", + "-cast", + "deprecation", + "-deprecation", + "divzero", + "-divzero", + "empty", + "-empty", + "fallthrough", + "-fallthrough", + "finally", + "-finally", + "none", + "options", + "-options", + "overrides", + "-overrides", + "path", + "-path", + "processing", + "-processing", + "rawtypes", + "-rawtypes", + "serial", + "-serial", + "unchecked", + "-unchecked" + ); + } + } + + /** + * Converter for the --experimental_java_classpath option. + */ + public static class JavaClasspathModeConverter extends EnumConverter<JavaClasspathMode> { + public JavaClasspathModeConverter() { + super(JavaClasspathMode.class, "Java classpath reduction strategy"); + } + } + + @Option(name = "javabase", + defaultValue = DEFAULT_JAVABASE, + category = "version", + help = "JAVABASE used for the JDK invoked by Blaze. This is the " + + "JAVABASE which will be used to execute external Java " + + "commands.") + public String javaBase; + + @Option(name = "java_toolchain", + defaultValue = DEFAULT_TOOLCHAIN, + category = "version", + converter = LabelConverter.class, + help = "The name of the toolchain rule for Java. Default is " + DEFAULT_TOOLCHAIN) + public Label javaToolchain; + + @Option(name = "host_javabase", + defaultValue = DEFAULT_JAVABASE, + category = "version", + help = "JAVABASE used for the host JDK. This is the JAVABASE which is used to execute " + + " tools during a build.") + public String hostJavaBase; + + @Option(name = "javacopt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to javac.") + public List<String> javacOpts; + + @Option(name = "jvmopt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to the Java VM. These options will get added to the " + + "VM startup options of each java_binary target.") + public List<String> jvmOpts; + + @Option(name = "javawarn", + converter = JavacWarnConverter.class, + defaultValue = "", + category = "flags", + allowMultiple = true, + help = "Additional javac warnings to enable when compiling Java source files.") + public List<String> javaWarns; + + @Option(name = "use_ijars", + defaultValue = "true", + category = "strategy", + help = "If enabled, this option causes Java compilation to use interface jars. " + + "This will result in faster incremental compilation, " + + "but error messages can be different.") + public boolean useIjars; + + @Deprecated + @Option(name = "use_src_ijars", + defaultValue = "false", + category = "undocumented", + help = "No-op. Kept here for backwards compatibility.") + public boolean useSourceIjars; + + @Deprecated + @Option(name = "experimental_incremental_ijars", + defaultValue = "false", + category = "undocumented", + help = "No-op. Kept here for backwards compatibility.") + public boolean incrementalIjars; + + @Option(name = "java_deps", + defaultValue = "true", + category = "strategy", + help = "Generate dependency information (for now, compile-time classpath) per Java target.") + public boolean javaDeps; + + @Option(name = "experimental_java_deps", + defaultValue = "false", + category = "experimental", + expansion = "--java_deps", + deprecationWarning = "Use --java_deps instead") + public boolean experimentalJavaDeps; + + @Option(name = "experimental_java_classpath", + allowMultiple = false, + defaultValue = "javabuilder", + converter = JavaClasspathModeConverter.class, + category = "semantics", + help = "Enables reduced classpaths for Java compilations.") + public JavaClasspathMode experimentalJavaClasspath; + + @Option(name = "java_cpu", + defaultValue = "null", + category = "semantics", + help = "The Java target CPU. Default is k8.") + public String javaCpu; + + @Option(name = "java_debug", + defaultValue = "null", + category = "testing", + expansion = {"--test_arg=--wrapper_script_flag=--debug", "--test_output=streamed", + "--test_strategy=exclusive", "--test_timeout=9999", "--nocache_test_results"}, + help = "Causes the Java virtual machine of a java test to wait for a connection from a " + + "JDWP-compliant debugger (such as jdb) before starting the test. Implies " + + "-test_output=streamed." + ) + public Void javaTestDebug; + + @Option(name = "strict_java_deps", + allowMultiple = false, + defaultValue = "default", + converter = StrictDepsConverter.class, + category = "semantics", + help = "If true, checks that a Java target explicitly declares all directly used " + + "targets as dependencies.") + public StrictDepsMode strictJavaDeps; + + @Option(name = "javabuilder_top", + defaultValue = DEFAULT_JAVABUILDER, + category = "version", + converter = LabelConverter.class, + help = "Label of the filegroup that contains the JavaBuilder jar.") + public Label javaBuilderTop; + + @Option(name = "javabuilder_jvmopt", + allowMultiple = true, + defaultValue = "", + category = "undocumented", + help = "Additional options to pass to the JVM when invoking JavaBuilder.") + public List<String> javaBuilderJvmOpts; + + @Option(name = "singlejar_top", + defaultValue = DEFAULT_SINGLEJAR, + category = "version", + converter = LabelConverter.class, + help = "Label of the filegroup that contains the SingleJar jar.") + public Label singleJarTop; + + @Option(name = "ijar_top", + defaultValue = DEFAULT_IJAR, + category = "version", + converter = LabelConverter.class, + help = "Label of the filegroup that contains the ijar binary.") + public Label iJarTop; + + @Option(name = "java_langtools", + defaultValue = DEFAULT_LANGTOOLS, + category = "version", + converter = LabelConverter.class, + help = "Label of the rule that produces the Java langtools jar.") + public Label javaLangtoolsJar; + + @Option(name = "javac_bootclasspath", + defaultValue = DEFAULT_LANGTOOLS_BOOTCLASSPATH, + category = "version", + converter = LabelConverter.class, + help = "Label of the rule that produces the bootclasspath jars for javac to use.") + public Label javacBootclasspath; + + @Option(name = "java_launcher", + defaultValue = "null", + converter = LabelConverter.class, + category = "semantics", + help = "If enabled, a specific Java launcher is used. " + + "The \"launcher\" attribute overrides this flag. ") + public Label javaLauncher; + + @Option(name = "translations", + defaultValue = "auto", + category = "semantics", + help = "Translate Java messages; bundle all translations into the jar " + + "for each affected rule.") + public TriState bundleTranslations; + + @Option(name = "message_translations", + defaultValue = "", + category = "semantics", + allowMultiple = true, + help = "The message translations used for translating messages in Java targets.") + public List<String> translationTargets; + + @Option(name = "check_constraint", + allowMultiple = true, + defaultValue = "", + category = "checking", + help = "Check the listed constraint.") + public List<String> checkedConstraints; + + @Override + public FragmentOptions getHost(boolean fallback) { + JavaOptions host = (JavaOptions) getDefault(); + + host.javaBase = hostJavaBase; + host.jvmOpts = ImmutableList.of("-client", "-XX:ErrorFile=/dev/stderr"); + + host.javacOpts = javacOpts; + host.javaLangtoolsJar = javaLangtoolsJar; + host.javaBuilderTop = javaBuilderTop; + host.javaToolchain = javaToolchain; + host.singleJarTop = singleJarTop; + host.iJarTop = iJarTop; + + // Java builds often contain complicated code generators for which + // incremental build performance is important. + host.useIjars = useIjars; + + host.javaDeps = javaDeps; + host.experimentalJavaClasspath = experimentalJavaClasspath; + + return host; + } + + @Override + public void addAllLabels(Multimap<String, Label> labelMap) { + addOptionalLabel(labelMap, "jdk", javaBase); + addOptionalLabel(labelMap, "jdk", hostJavaBase); + if (javaLauncher != null) { + labelMap.put("java_launcher", javaLauncher); + } + labelMap.put("javabuilder", javaBuilderTop); + labelMap.put("singlejar", singleJarTop); + labelMap.put("ijar", iJarTop); + labelMap.put("java_toolchain", javaToolchain); + labelMap.putAll("translation", getTranslationLabels()); + } + + @Override + public Map<String, Set<Label>> getDefaultsLabels(BuildConfiguration.Options commonOptions) { + Set<Label> jdkLabels = new LinkedHashSet<>(); + DefaultsPackage.parseAndAdd(jdkLabels, javaBase); + DefaultsPackage.parseAndAdd(jdkLabels, hostJavaBase); + Map<String, Set<Label>> result = new HashMap<>(); + result.put("JDK", jdkLabels); + result.put("JAVA_LANGTOOLS", ImmutableSet.of(javaLangtoolsJar)); + result.put("JAVAC_BOOTCLASSPATH", ImmutableSet.of(javacBootclasspath)); + result.put("JAVABUILDER", ImmutableSet.of(javaBuilderTop)); + result.put("SINGLEJAR", ImmutableSet.of(singleJarTop)); + result.put("IJAR", ImmutableSet.of(iJarTop)); + result.put("JAVA_TOOLCHAIN", ImmutableSet.of(javaToolchain)); + + return result; + } + + private Set<Label> getTranslationLabels() { + Set<Label> result = new LinkedHashSet<>(); + for (String s : translationTargets) { + try { + Label label = Label.parseAbsolute(s); + result.add(label); + } catch (SyntaxException e) { + // We ignore this exception here - it will cause an error message at a later time. + } + } + return result; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPlugin.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPlugin.java new file mode 100644 index 0000000..526d52c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPlugin.java
@@ -0,0 +1,57 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * Implementation for the java_plugin rule. + */ +public class JavaPlugin implements RuleConfiguredTargetFactory { + + private final JavaSemantics semantics; + + protected JavaPlugin(JavaSemantics semantics) { + this.semantics = semantics; + } + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + JavaLibrary javaLibrary = new JavaLibrary(semantics); + JavaCommon common = new JavaCommon(ruleContext, semantics); + RuleConfiguredTargetBuilder builder = javaLibrary.init(ruleContext, common); + if (builder == null) { + return null; + } + builder.add(JavaPluginInfoProvider.class, new JavaPluginInfoProvider( + getProcessorClasses(ruleContext), common.getRuntimeClasspath())); + return builder.build(); + } + + /** + * Returns the class that should be passed to javac in order + * to run the annotation processor this class represents. + */ + private ImmutableList<String> getProcessorClasses(RuleContext ruleContext) { + if (ruleContext.getRule().isAttributeValueExplicitlySpecified("processor_class")) { + return ImmutableList.of(ruleContext.attributes().get("processor_class", Type.STRING)); + } + return ImmutableList.of(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPluginInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPluginInfoProvider.java new file mode 100644 index 0000000..520a228 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPluginInfoProvider.java
@@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Provider for users of Java plugins. + */ +@Immutable +public final class JavaPluginInfoProvider implements TransitiveInfoProvider { + + private final ImmutableList<String> processorClasses; + private final NestedSet<Artifact> processorClasspath; + + public JavaPluginInfoProvider(ImmutableList<String> processorClasses, + NestedSet<Artifact> processorClasspath) { + this.processorClasses = processorClasses; + this.processorClasspath = processorClasspath; + } + + /** + * Returns the class that should be passed to javac in order + * to run the annotation processor this class represents. + */ + public ImmutableList<String> getProcessorClasses() { + return processorClasses; + } + + /** + * Returns the artifacts to add to the runtime classpath for this plugin. + */ + public NestedSet<Artifact> getProcessorClasspath() { + return processorClasspath; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPrimaryClassProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPrimaryClassProvider.java new file mode 100644 index 0000000..fd90011 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPrimaryClassProvider.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Provides the fully qualified name of the primary class to invoke for java targets. + */ +@Immutable +public final class JavaPrimaryClassProvider implements TransitiveInfoProvider { + + private final String primaryClass; + + public JavaPrimaryClassProvider(String primaryClass) { + this.primaryClass = primaryClass; + } + + /** + * Returns either the Java class whose main() method is to be invoked (when + * use_testrunner=0) or the Java subclass of junit.framework.Test that + * is to be tested by the test runner class (when use_testrunner=1). + * + * @return a fully qualified Java class name, or null if none could be + * determined. + */ + public String getPrimaryClass() { + return primaryClass; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaRunfilesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRunfilesProvider.java new file mode 100644 index 0000000..b742d62 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRunfilesProvider.java
@@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A {@link TransitiveInfoProvider} that supplies runfiles for Java dependencies. + */ +@Immutable +public final class JavaRunfilesProvider implements TransitiveInfoProvider { + private final Runfiles runfiles; + + public JavaRunfilesProvider(Runfiles runfiles) { + this.runfiles = runfiles; + } + + public Runfiles getRunfiles() { + return runfiles; + } + + /** + * Returns a function that gets the Java runfiles from a {@link TransitiveInfoCollection} or + * the empty runfiles instance if it does not contain that provider. + */ + public static final Function<TransitiveInfoCollection, Runfiles> TO_RUNFILES = + new Function<TransitiveInfoCollection, Runfiles>() { + @Override + public Runfiles apply(TransitiveInfoCollection input) { + JavaRunfilesProvider provider = input.getProvider(JavaRunfilesProvider.class); + return provider == null + ? Runfiles.EMPTY + : provider.getRunfiles(); + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaRuntimeClasspathProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRuntimeClasspathProvider.java new file mode 100644 index 0000000..c8090df --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRuntimeClasspathProvider.java
@@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Provider for the runtime classpath contributions of a Java binary. + * + * Used to exclude already-available artifacts from related binaries + * (e.g. plugins). + */ +@Immutable +public final class JavaRuntimeClasspathProvider implements TransitiveInfoProvider { + + private final NestedSet<Artifact> runtimeClasspath; + + public JavaRuntimeClasspathProvider(NestedSet<Artifact> runtimeClasspath) { + this.runtimeClasspath = runtimeClasspath; + } + + /** + * Returns the artifacts included on the runtime classpath of this binary. + */ + public NestedSet<Artifact> getRuntimeClasspath() { + return runtimeClasspath; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaSemantics.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSemantics.java new file mode 100644 index 0000000..64b6214 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSemantics.java
@@ -0,0 +1,351 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.LanguageDependentFragment.LibraryLanguage; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.Runfiles.Builder; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel; +import com.google.devtools.build.lib.packages.Attribute.LateBoundLabelList; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.rules.java.DeployArchiveBuilder.Compression; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.List; + +import javax.annotation.Nullable; + +/** + * Pluggable Java compilation semantics. + */ +public interface JavaSemantics { + + public static final LibraryLanguage LANGUAGE = new LibraryLanguage("Java"); + + public static final SafeImplicitOutputsFunction JAVA_LIBRARY_CLASS_JAR = + fromTemplates("lib%{name}.jar"); + public static final SafeImplicitOutputsFunction JAVA_LIBRARY_SOURCE_JAR = + fromTemplates("lib%{name}-src.jar"); + public static final SafeImplicitOutputsFunction JAVA_BINARY_CLASS_JAR = + fromTemplates("%{name}.jar"); + public static final SafeImplicitOutputsFunction JAVA_BINARY_SOURCE_JAR = + fromTemplates("%{name}-src.jar"); + public static final SafeImplicitOutputsFunction JAVA_BINARY_DEPLOY_JAR = + fromTemplates("%{name}_deploy.jar"); + public static final SafeImplicitOutputsFunction JAVA_BINARY_DEPLOY_SOURCE_JAR = + fromTemplates("%{name}_deploy-src.jar"); + + public static final FileType JAVA_SOURCE = FileType.of(".java"); + public static final FileType JAR = FileType.of(".jar"); + public static final FileType PROPERTIES = FileType.of(".properties"); + public static final FileType SOURCE_JAR = FileType.of(".srcjar"); + // TODO(bazel-team): Rename this metadata extension to something meaningful. + public static final FileType COVERAGE_METADATA = FileType.of(".em"); + + /** + * Label to the Java Toolchain rule. It is resolved from a label given in the java options. + */ + static final String JAVA_TOOLCHAIN_LABEL = "//tools/defaults:java_toolchain"; + + public static final LateBoundLabel<BuildConfiguration> JAVA_TOOLCHAIN = + new LateBoundLabel<BuildConfiguration>(JAVA_TOOLCHAIN_LABEL) { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.getFragment(JavaConfiguration.class).getToolchainLabel(); + } + }; + + /** + * Name of the output group used for source jars. + */ + public static final String SOURCE_JARS_OUTPUT_GROUP = "source_jars"; + + /** + * Label of a pseudo-filegroup that contains all jdk files for all + * configurations, as specified on the command-line. + */ + public static final String JDK_LABEL = "//tools/defaults:jdk"; + + /** + * Label of a pseudo-filegroup that contains the boot-classpath entries. + */ + public static final String JAVAC_BOOTCLASSPATH_LABEL = "//tools/defaults:javac_bootclasspath"; + + /** + * Label of the JavaBuilder JAR used for compiling Java source code. + */ + public static final String JAVABUILDER_LABEL = "//tools/defaults:javabuilder"; + + /** + * Label of the SingleJar JAR used for creating deploy jars. + */ + public static final String SINGLEJAR_LABEL = "//tools/defaults:singlejar"; + + /** + * Label of pseudo-cc_binary that tells Blaze a java target's JAVABIN is never to be replaced by + * the contents of --java_launcher; only the JDK's launcher will ever be used. + */ + public static final Label JDK_LAUNCHER_LABEL = + Label.parseAbsoluteUnchecked("//third_party/java/jdk:jdk_launcher"); + + /** + * Implementation for the :jvm attribute. + */ + public static final LateBoundLabel<BuildConfiguration> JVM = + new LateBoundLabel<BuildConfiguration>(JDK_LABEL) { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.getFragment(Jvm.class).getJvmLabel(); + } + }; + + /** + * Implementation for the :host_jdk attribute. + */ + public static final LateBoundLabel<BuildConfiguration> HOST_JDK = + new LateBoundLabel<BuildConfiguration>(JDK_LABEL) { + @Override + public boolean useHostConfiguration() { + return true; + } + + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.getFragment(Jvm.class).getJvmLabel(); + } + }; + + /** + * Implementation for the :java_launcher attribute. Note that the Java launcher is disabled by + * default, so it returns null for the configuration-independent default value. + */ + public static final LateBoundLabel<BuildConfiguration> JAVA_LAUNCHER = + new LateBoundLabel<BuildConfiguration>() { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.getFragment(JavaConfiguration.class).getJavaLauncherLabel(); + } + }; + + public static final LateBoundLabelList<BuildConfiguration> JAVA_PLUGINS = + new LateBoundLabelList<BuildConfiguration>() { + @Override + public List<Label> getDefault(Rule rule, BuildConfiguration configuration) { + return ImmutableList.copyOf(configuration.getPlugins()); + } + }; + + public static final String IJAR_LABEL = "//tools/defaults:ijar"; + + /** + * Verifies if the rule contains and errors. + * + * <p>Errors should be signaled through {@link RuleContext}. + */ + void checkRule(RuleContext ruleContext, JavaCommon javaCommon); + + /** + * Returns the main class of a Java binary. + */ + String getMainClass(RuleContext ruleContext, JavaCommon javaCommon); + + /** + * Returns the resources contributed by a Java rule (usually the contents of the + * {@code resources} attribute) + */ + ImmutableList<Artifact> collectResources(RuleContext ruleContext); + + /** + * Creates the instrumentation metadata artifact for the specified output .jar . + */ + @Nullable Artifact createInstrumentationMetadataArtifact( + AnalysisEnvironment analysisEnvironment, Artifact outputJar); + + /** + * May add extra command line options to the Java compile command line. + */ + void buildJavaCommandLine(Collection<Artifact> outputs, BuildConfiguration configuration, + CustomCommandLine.Builder result); + + + /** + * Constructs the command line to call SingleJar to join all artifacts from + * {@code classpath} (java code) and {@code resources} into {@code output}. + */ + CustomCommandLine buildSingleJarCommandLine(BuildConfiguration configuration, + Artifact output, String mainClass, ImmutableList<String> manifestLines, + Iterable<Artifact> buildInfoFiles, ImmutableList<Artifact> resources, + Iterable<Artifact> classpath, boolean includeBuildData, + Compression compression, Artifact launcher); + + /** + * Creates the action that writes the Java executable stub script. + */ + void createStubAction(RuleContext ruleContext, final JavaCommon javaCommon, + List<String> jvmFlags, Artifact executable, String javaStartClass, + String javaExecutable); + + /** + * Adds extra runfiles for a {@code java_binary} rule. + */ + void addRunfilesForBinary(RuleContext ruleContext, Artifact launcher, + Runfiles.Builder runfilesBuilder); + + /** + * Adds extra runfiles for a {@code java_library} rule. + */ + void addRunfilesForLibrary(RuleContext ruleContext, Runfiles.Builder runfilesBuilder); + + /** + * Returns the coverage instrumentation specification to be used in Java rules. + */ + InstrumentationSpec getCoverageInstrumentationSpec(); + + /** + * Returns the additional options to be passed to javac. + */ + Iterable<String> getExtraJavacOpts(RuleContext ruleContext); + + /** + * Add additional targets to be treated as direct dependencies. + */ + void collectTargetsTreatedAsDeps( + RuleContext ruleContext, ImmutableList.Builder<TransitiveInfoCollection> builder); + + /** + * Enables coverage support for the java target - adds instrumented jar to the classpath and + * modifies main class. + * + * @return new main class + */ + String addCoverageSupport(JavaCompilationHelper helper, + JavaTargetAttributes.Builder attributes, + Artifact executable, Artifact instrumentationMetadata, + JavaCompilationArtifacts.Builder javaArtifactsBuilder, String mainClass); + + /** + * Return the JVM flags to be used in a Java binary. + */ + Iterable<String> getJvmFlags(RuleContext ruleContext, JavaCommon javaCommon, + Artifact launcher, List<String> userJvmFlags); + + /** + * Adds extra providers to a Java target. + */ + void addProviders(RuleContext ruleContext, + JavaCommon javaCommon, + List<String> jvmFlags, + Artifact classJar, + Artifact srcJar, + Artifact gensrcJar, + ImmutableMap<Artifact, Artifact> compilationToRuntimeJarMap, + JavaCompilationHelper helper, + NestedSetBuilder<Artifact> filesBuilder, + RuleConfiguredTargetBuilder ruleBuilder); + + /** + * Tell if a build with the given configuration should use strict java dependencies. This method + * enforces strict java dependencies off if it returns false. + */ + boolean useStrictJavaDeps(BuildConfiguration configuration); + + /** + * Translates XMB messages to translations artifact suitable for Java targets. + */ + Collection<Artifact> translate(RuleContext ruleContext, JavaConfiguration javaConfig, + List<Artifact> messages); + + /** + * Get the launcher artifact for a java binary, creating the necessary actions for it. + * + * @param ruleContext The rule context + * @param common The common helper class. + * @param deployArchiveBuilder the builder to construct the deploy archive action (mutable). + * @param runfilesBuilder the builder to construct the list of runfiles (mutable). + * @param jvmFlags the list of flags to pass to the JVM when running the Java binary (mutable). + * @param attributesBuilder the builder to construct the list of attributes of this target + * (mutable). + * @return the launcher as an artifact + */ + Artifact getLauncher(final RuleContext ruleContext, final JavaCommon common, + DeployArchiveBuilder deployArchiveBuilder, Runfiles.Builder runfilesBuilder, + List<String> jvmFlags, JavaTargetAttributes.Builder attributesBuilder); + + /** + * Add extra dependencies for runfiles of a Java binary. + */ + void addDependenciesForRunfiles(RuleContext ruleContext, Builder builder); + + /** + * Determines if we should enforce the use of the :java_launcher target to determine the java + * launcher artifact even if the --java_launcher option was not specified. + */ + boolean forceUseJavaLauncherTarget(RuleContext ruleContext); + + /** + * Add a source artifact to a {@link JavaTargetAttributes.Builder}. It is called when a source + * artifact is processed but is not matched by default patterns in the + * {@link JavaTargetAttributes.Builder#addSourceArtifacts(Iterable)} method. The semantics can + * then detect its custom artifact types and add it to the builder. + */ + void addArtifactToJavaTargetAttribute(JavaTargetAttributes.Builder builder, Artifact srcArtifact); + + /** + * Works on the list of dependencies of a java target to builder the {@link JavaTargetAttributes}. + * This work is performed in {@link JavaCommon} for all java targets. + */ + void commonDependencyProcessing(RuleContext ruleContext, JavaTargetAttributes.Builder attributes, + Collection<? extends TransitiveInfoCollection> deps); + + /** + * Returns an list of {@link ActionInput} that the {@link JavaCompileAction} generates and + * that should be cached. + */ + Collection<ActionInput> getExtraJavaCompileOutputs(PathFragment classDirectory); + + /** + * Takes the path of a Java resource and tries to determine the Java + * root relative path of the resource. + * + * @param path the root relative path of the resource. + * @return the Java root relative path of the resource of the root + * relative path of the resource if no Java root relative path can be + * determined. + */ + PathFragment getJavaResourcePath(PathFragment path); + + /** + * @return a list of extra arguments to appends to the runfiles support. + */ + List<String> getExtraArguments(RuleContext ruleContext, JavaCommon javaCommon); +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaSourceJarsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSourceJarsProvider.java new file mode 100644 index 0000000..2bb3597 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSourceJarsProvider.java
@@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * The collection of source jars from the transitive closure. + */ +@Immutable +public final class JavaSourceJarsProvider implements TransitiveInfoProvider { + + private final NestedSet<Artifact> transitiveSourceJars; + private final ImmutableList<Artifact> sourceJars; + + public JavaSourceJarsProvider(NestedSet<Artifact> transitiveSourceJars, + Iterable<Artifact> sourceJars) { + this.transitiveSourceJars = transitiveSourceJars; + this.sourceJars = ImmutableList.copyOf(sourceJars); + } + + /** + * Returns all the source jars in the transitive closure, that can be reached by a chain of + * JavaSourceJarsProvider instances. + */ + public NestedSet<Artifact> getTransitiveSourceJars() { + return transitiveSourceJars; + } + + /** + * Return the source jars that are to be built when the target is on the command line. + */ + public ImmutableList<Artifact> getSourceJars() { + return sourceJars; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaTargetAttributes.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaTargetAttributes.java new file mode 100644 index 0000000..a7fc497 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaTargetAttributes.java
@@ -0,0 +1,603 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.IterablesChain; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.cpp.CppFileTypes; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * An object that captures the temporary state we need to pass around while + * the initialization hook for a java rule is running. + */ +public class JavaTargetAttributes { + + private static void checkJar(Artifact classPathEntry) { + if (!JavaSemantics.JAR.matches(classPathEntry.getFilename())) { + throw new IllegalArgumentException( + "not a jar file: " + classPathEntry.prettyPrint()); + } + } + + /** + * A builder class for JavaTargetAttributes. + */ + public static class Builder { + + // The order of source files is important, and there must not be duplicates. + // Unfortunately, there is no interface in Java that represents a collection + // without duplicates that has a stable and deterministic iteration order, + // but is not sorted according to a property of the elements. Thus we are + // stuck with Set. + private final Set<Artifact> sourceFiles = new LinkedHashSet<>(); + private final Set<Artifact> jarFiles = new LinkedHashSet<>(); + private final Set<Artifact> compileTimeJarFiles = new LinkedHashSet<>(); + + private final NestedSetBuilder<Artifact> runtimeClassPath = + NestedSetBuilder.naiveLinkOrder(); + + private final NestedSetBuilder<Artifact> compileTimeClassPath = + NestedSetBuilder.naiveLinkOrder(); + + private final List<Artifact> bootClassPath = new ArrayList<>(); + private final List<Artifact> nativeLibraries = new ArrayList<>(); + + private final Set<Artifact> processorPath = new LinkedHashSet<>(); + private final Set<String> processorNames = new LinkedHashSet<>(); + + private final List<Artifact> resources = new ArrayList<>(); + private final List<Artifact> messages = new ArrayList<>(); + private final List<Artifact> instrumentationMetadata = new ArrayList<>(); + private final List<Artifact> sourceJars = new ArrayList<>(); + + private final List<Artifact> classPathResources = new ArrayList<>(); + + private BuildConfiguration.StrictDepsMode strictJavaDeps = + BuildConfiguration.StrictDepsMode.OFF; + private final List<Artifact> directJars = new ArrayList<>(); + private final List<Artifact> compileTimeDependencyArtifacts = new ArrayList<>(); + private final List<Artifact> runtimeDependencyArtifacts = new ArrayList<>(); + private String ruleKind; + private Label targetLabel; + + private final NestedSetBuilder<Artifact> excludedArtifacts = + NestedSetBuilder.naiveLinkOrder(); + + private boolean built = false; + + private final JavaSemantics semantics; + + public Builder(JavaSemantics semantics) { + this.semantics = semantics; + } + + public Builder addSourceArtifacts(Iterable<Artifact> sourceArtifacts) { + Preconditions.checkArgument(!built); + for (Artifact srcArtifact : sourceArtifacts) { + String srcFilename = srcArtifact.getExecPathString(); + if (JavaSemantics.JAR.matches(srcFilename)) { + runtimeClassPath.add(srcArtifact); + jarFiles.add(srcArtifact); + } else if (JavaSemantics.SOURCE_JAR.matches(srcFilename)) { + sourceJars.add(srcArtifact); + } else if (JavaSemantics.PROPERTIES.matches(srcFilename)) { + // output files of the message compiler + resources.add(srcArtifact); + } else if (JavaSemantics.JAVA_SOURCE.matches(srcFilename)) { + sourceFiles.add(srcArtifact); + } else { + // try specific cases from the semantics. + semantics.addArtifactToJavaTargetAttribute(this, srcArtifact); + } + } + return this; + } + + public Builder addSourceFiles(Iterable<Artifact> sourceFiles) { + Preconditions.checkArgument(!built); + for (Artifact artifact : sourceFiles) { + if (JavaSemantics.JAVA_SOURCE.matches(artifact.getFilename())) { + this.sourceFiles.add(artifact); + } + } + return this; + } + + public Builder merge(JavaCompilationArgs context) { + Preconditions.checkArgument(!built); + addCompileTimeClassPathEntries(context.getCompileTimeJars()); + addRuntimeClassPathEntries(context.getRuntimeJars()); + addInstrumentationMetadataEntries(context.getInstrumentationMetadata()); + return this; + } + + public Builder addSourceJars(Collection<Artifact> sourceJars) { + Preconditions.checkArgument(!built); + this.sourceJars.addAll(sourceJars); + return this; + } + + public Builder addSourceJar(Artifact sourceJar) { + Preconditions.checkArgument(!built); + this.sourceJars.add(sourceJar); + return this; + } + + public Builder addCompileTimeJarFiles(Iterable<Artifact> jars) { + Preconditions.checkArgument(!built); + Iterables.addAll(compileTimeJarFiles, jars); + return this; + } + + public Builder addRuntimeClassPathEntry(Artifact classPathEntry) { + Preconditions.checkArgument(!built); + checkJar(classPathEntry); + runtimeClassPath.add(classPathEntry); + return this; + } + + public Builder addRuntimeClassPathEntries(NestedSet<Artifact> classPathEntries) { + Preconditions.checkArgument(!built); + runtimeClassPath.addTransitive(classPathEntries); + return this; + } + + public Builder addCompileTimeClassPathEntries(NestedSet<Artifact> entries) { + Preconditions.checkArgument(!built); + compileTimeClassPath.addTransitive(entries); + return this; + } + + public Builder addDirectCompileTimeClassPathEntries(Iterable<Artifact> entries) { + Preconditions.checkArgument(!built); + // The other version is preferred as it is more memory-efficient. + for (Artifact classPathEntry : entries) { + compileTimeClassPath.add(classPathEntry); + } + return this; + } + + public Builder setRuleKind(String ruleKind) { + Preconditions.checkArgument(!built); + this.ruleKind = ruleKind; + return this; + } + + public Builder setTargetLabel(Label targetLabel) { + Preconditions.checkArgument(!built); + this.targetLabel = targetLabel; + return this; + } + + /** + * Sets the bootclasspath to be passed to the Java compiler. + * + * <p>If this method is called, then the bootclasspath specified in this JavaTargetAttributes + * instance overrides the default bootclasspath. + */ + public Builder setBootClassPath(List<Artifact> jars) { + Preconditions.checkArgument(!built); + Preconditions.checkArgument(!jars.isEmpty()); + Preconditions.checkState(bootClassPath.isEmpty()); + bootClassPath.addAll(jars); + return this; + } + + public Builder addExcludedArtifacts(NestedSet<Artifact> toExclude) { + Preconditions.checkArgument(!built); + excludedArtifacts.addTransitive(toExclude); + return this; + } + + /** + * Controls how strict the javac compiler will be in checking correct use of + * direct dependencies. + * + * @param strictDeps one of WARN, ERROR or OFF + */ + public Builder setStrictJavaDeps(BuildConfiguration.StrictDepsMode strictDeps) { + Preconditions.checkArgument(!built); + strictJavaDeps = strictDeps; + return this; + } + + /** + * In tandem with strictJavaDeps, directJars represents a subset of the + * compile-time, classpath jars that were provided by direct dependencies. + * When strictJavaDeps is OFF, there is no need to provide directJars, and + * no extra information is passed to javac. When strictJavaDeps is set to + * WARN or ERROR, the compiler command line will include extra flags to + * indicate the warning/error policy and to map the classpath jars to direct + * or transitive dependencies, using the information in directJars. The extra + * flags are formatted like this (same for --indirect_dependency): + * --direct_dependency + * foo/bar/lib.jar + * //java/com/google/foo:bar + * + * @param directJars + */ + public Builder addDirectJars(Iterable<Artifact> directJars) { + Preconditions.checkArgument(!built); + Iterables.addAll(this.directJars, directJars); + return this; + } + + public Builder addCompileTimeDependencyArtifacts(Iterable<Artifact> dependencyArtifacts) { + Preconditions.checkArgument(!built); + Iterables.addAll(this.compileTimeDependencyArtifacts, dependencyArtifacts); + return this; + } + + public Builder addRuntimeDependencyArtifacts(Iterable<Artifact> dependencyArtifacts) { + Preconditions.checkArgument(!built); + Iterables.addAll(this.runtimeDependencyArtifacts, dependencyArtifacts); + return this; + } + + public Builder addInstrumentationMetadataEntries(Iterable<Artifact> metadataEntries) { + Preconditions.checkArgument(!built); + Iterables.addAll(instrumentationMetadata, metadataEntries); + return this; + } + + public Builder addNativeLibrary(Artifact nativeLibrary) { + Preconditions.checkArgument(!built); + String name = nativeLibrary.getFilename(); + if (CppFileTypes.INTERFACE_SHARED_LIBRARY.matches(name)) { + return this; + } + if (!(CppFileTypes.SHARED_LIBRARY.matches(name) + || CppFileTypes.VERSIONED_SHARED_LIBRARY.matches(name))) { + throw new IllegalArgumentException("not a shared library :" + nativeLibrary.prettyPrint()); + } + nativeLibraries.add(nativeLibrary); + return this; + } + + public Builder addNativeLibraries(Iterable<Artifact> nativeLibraries) { + Preconditions.checkArgument(!built); + for (Artifact nativeLibrary : nativeLibraries) { + addNativeLibrary(nativeLibrary); + } + return this; + } + + public Builder addMessages(Collection<Artifact> messages) { + Preconditions.checkArgument(!built); + this.messages.addAll(messages); + return this; + } + + public Builder addMessage(Artifact messagesArtifact) { + Preconditions.checkArgument(!built); + this.messages.add(messagesArtifact); + return this; + } + + public Builder addResources(Collection<Artifact> resources) { + Preconditions.checkArgument(!built); + this.resources.addAll(resources); + return this; + } + + public Builder addResource(Artifact resource) { + Preconditions.checkArgument(!built); + resources.add(resource); + return this; + } + + public Builder addProcessorName(String processor) { + Preconditions.checkArgument(!built); + processorNames.add(processor); + return this; + } + + public Builder addProcessorPath(Iterable<Artifact> jars) { + Preconditions.checkArgument(!built); + Iterables.addAll(processorPath, jars); + return this; + } + + public Builder addClassPathResources(List<Artifact> classPathResources) { + Preconditions.checkArgument(!built); + this.classPathResources.addAll(classPathResources); + return this; + } + + public Builder addClassPathResource(Artifact classPathResource) { + Preconditions.checkArgument(!built); + this.classPathResources.add(classPathResource); + return this; + } + + public JavaTargetAttributes build() { + built = true; + return new JavaTargetAttributes( + sourceFiles, + jarFiles, + compileTimeJarFiles, + runtimeClassPath, + compileTimeClassPath, + bootClassPath, + nativeLibraries, + processorPath, + processorNames, + resources, + messages, + sourceJars, + classPathResources, + directJars, + compileTimeDependencyArtifacts, + ruleKind, + targetLabel, + excludedArtifacts, + strictJavaDeps); + } + + // TODO(bazel-team): Remove these 5 methods. + @Deprecated + public Set<Artifact> getSourceFiles() { + return sourceFiles; + } + + @Deprecated + public boolean hasSourceFiles() { + return !sourceFiles.isEmpty(); + } + + @Deprecated + public List<Artifact> getInstrumentationMetadata() { + return instrumentationMetadata; + } + + @Deprecated + public boolean hasSourceJars() { + return !sourceJars.isEmpty(); + } + + @Deprecated + public boolean hasJarFiles() { + return !jarFiles.isEmpty(); + } + } + + // + // -------------------------- END OF BUILDER CLASS ------------------------- + // + + private final ImmutableSet<Artifact> sourceFiles; + private final ImmutableSet<Artifact> jarFiles; + private final ImmutableSet<Artifact> compileTimeJarFiles; + + private final NestedSet<Artifact> runtimeClassPath; + private final NestedSet<Artifact> compileTimeClassPath; + + private final ImmutableList<Artifact> bootClassPath; + private final ImmutableList<Artifact> nativeLibraries; + + private final ImmutableSet<Artifact> processorPath; + private final ImmutableSet<String> processorNames; + + private final ImmutableList<Artifact> resources; + private final ImmutableList<Artifact> messages; + private final ImmutableList<Artifact> sourceJars; + + private final ImmutableList<Artifact> classPathResources; + + private final ImmutableList<Artifact> directJars; + private final ImmutableList<Artifact> compileTimeDependencyArtifacts; + private final String ruleKind; + private final Label targetLabel; + + private final NestedSet<Artifact> excludedArtifacts; + private final BuildConfiguration.StrictDepsMode strictJavaDeps; + + /** + * Constructor of JavaTargetAttributes. + */ + private JavaTargetAttributes( + Set<Artifact> sourceFiles, + Set<Artifact> jarFiles, + Set<Artifact> compileTimeJarFiles, + NestedSetBuilder<Artifact> runtimeClassPath, + NestedSetBuilder<Artifact> compileTimeClassPath, + List<Artifact> bootClassPath, + List<Artifact> nativeLibraries, + Set<Artifact> processorPath, + Set<String> processorNames, + List<Artifact> resources, + List<Artifact> messages, + List<Artifact> sourceJars, + List<Artifact> classPathResources, + List<Artifact> directJars, + List<Artifact> compileTimeDependencyArtifacts, + String ruleKind, + Label targetLabel, + NestedSetBuilder<Artifact> excludedArtifacts, + BuildConfiguration.StrictDepsMode strictJavaDeps) { + this.sourceFiles = ImmutableSet.copyOf(sourceFiles); + this.jarFiles = ImmutableSet.copyOf(jarFiles); + this.compileTimeJarFiles = ImmutableSet.copyOf(compileTimeJarFiles); + this.runtimeClassPath = runtimeClassPath.build(); + this.compileTimeClassPath = compileTimeClassPath.build(); + this.bootClassPath = ImmutableList.copyOf(bootClassPath); + this.nativeLibraries = ImmutableList.copyOf(nativeLibraries); + this.processorPath = ImmutableSet.copyOf(processorPath); + this.processorNames = ImmutableSet.copyOf(processorNames); + this.resources = ImmutableList.copyOf(resources); + this.messages = ImmutableList.copyOf(messages); + this.sourceJars = ImmutableList.copyOf(sourceJars); + this.classPathResources = ImmutableList.copyOf(classPathResources); + this.directJars = ImmutableList.copyOf(directJars); + this.compileTimeDependencyArtifacts = ImmutableList.copyOf(compileTimeDependencyArtifacts); + this.ruleKind = ruleKind; + this.targetLabel = targetLabel; + this.excludedArtifacts = excludedArtifacts.build(); + this.strictJavaDeps = strictJavaDeps; + } + + public List<Artifact> getDirectJars() { + return directJars; + } + + public List<Artifact> getCompileTimeDependencyArtifacts() { + return compileTimeDependencyArtifacts; + } + + public List<Artifact> getSourceJars() { + return sourceJars; + } + + public Collection<Artifact> getResources() { + return resources; + } + + public List<Artifact> getMessages() { + return messages; + } + + public ImmutableList<Artifact> getClassPathResources() { + return classPathResources; + } + + private NestedSet<Artifact> getExcludedArtifacts() { + return excludedArtifacts; + } + + /** + * Returns the artifacts needed on the runtime classpath of this target. + * + * See also {@link #getRuntimeClassPathForArchive()}. + */ + public NestedSet<Artifact> getRuntimeClassPath() { + return runtimeClassPath; + } + + /** + * Returns the classpath artifacts needed in a deploy jar for this target. + * + * This excludes the artifacts made available by jars in the deployment + * environment. + */ + public Iterable<Artifact> getRuntimeClassPathForArchive() { + Iterable<Artifact> runtimeClasspath = getRuntimeClassPath(); + + if (getExcludedArtifacts().isEmpty()) { + return runtimeClasspath; + } else { + return Iterables.filter(runtimeClasspath, + Predicates.not(Predicates.in(getExcludedArtifacts().toSet()))); + } + } + + public NestedSet<Artifact> getCompileTimeClassPath() { + return compileTimeClassPath; + } + + public ImmutableList<Artifact> getBootClassPath() { + return bootClassPath; + } + + public ImmutableSet<Artifact> getProcessorPath() { + return processorPath; + } + + public Set<Artifact> getSourceFiles() { + return sourceFiles; + } + + public Set<Artifact> getJarFiles() { + return jarFiles; + } + + public Set<Artifact> getCompileTimeJarFiles() { + return compileTimeJarFiles; + } + + public List<Artifact> getNativeLibraries() { + return nativeLibraries; + } + + public Collection<String> getProcessorNames() { + return processorNames; + } + + public boolean hasSourceFiles() { + return !sourceFiles.isEmpty(); + } + + public boolean hasSourceJars() { + return !sourceJars.isEmpty(); + } + + public boolean hasJarFiles() { + return !jarFiles.isEmpty(); + } + + public boolean hasResources() { + return !resources.isEmpty(); + } + + public boolean hasMessages() { + return !messages.isEmpty(); + } + + public boolean hasClassPathResources() { + return !classPathResources.isEmpty(); + } + + public Iterable<Artifact> getArchiveInputs(boolean includeClasspath) { + IterablesChain.Builder<Artifact> inputs = IterablesChain.builder(); + if (includeClasspath) { + inputs.add(ImmutableList.copyOf(getRuntimeClassPathForArchive())); + } + inputs.add(getResources()); + inputs.add(getClassPathResources()); + if (getExcludedArtifacts().isEmpty()) { + return inputs.build(); + } else { + Set<Artifact> excludedJars = Sets.newHashSet(getExcludedArtifacts()); + return ImmutableList.copyOf(Iterables.filter( + inputs.build(), Predicates.not(Predicates.in(excludedJars)))); + } + } + + public String getRuleKind() { + return ruleKind; + } + + public Label getTargetLabel() { + return targetLabel; + } + + public BuildConfiguration.StrictDepsMode getStrictJavaDeps() { + return strictJavaDeps; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java new file mode 100644 index 0000000..65ed97a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java
@@ -0,0 +1,53 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +import java.util.List; + +/** + * Implementation for the {@code java_toolchain} rule. + */ +public final class JavaToolchain implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + final String source = ruleContext.attributes().get("source_version", Type.STRING); + final String target = ruleContext.attributes().get("target_version", Type.STRING); + final String encoding = ruleContext.attributes().get("encoding", Type.STRING); + final List<String> xlint = ruleContext.attributes().get("xlint", Type.STRING_LIST); + final List<String> misc = ruleContext.attributes().get("misc", Type.STRING_LIST); + final JavaConfiguration configuration = ruleContext.getFragment(JavaConfiguration.class); + JavaToolchainProvider provider = new JavaToolchainProvider(source, target, encoding, + ImmutableList.copyOf(xlint), ImmutableList.copyOf(misc), + configuration.getDefaultJavacFlags()); + RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(ruleContext) + .add(JavaToolchainProvider.class, provider) + .setFilesToBuild(new NestedSetBuilder<Artifact>(Order.STABLE_ORDER).build()) + .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY)); + + return builder.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainData.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainData.java new file mode 100644 index 0000000..0338fb8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainData.java
@@ -0,0 +1,55 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Information about the JDK used by the <code>java_*</code> rules. + * + * <p>This class contains the data of the {@code java_toolchain} rules, it is a separate object so + * it can be shared with other tools. + */ +@Immutable +public class JavaToolchainData { + private final ImmutableList<String> options; + + public JavaToolchainData(String source, String target, String encoding, + ImmutableList<String> xlint, ImmutableList<String> misc) { + Builder<String> builder = ImmutableList.<String>builder(); + if (!source.isEmpty()) { + builder.add("-source", source); + } + if (!target.isEmpty()) { + builder.add("-target", target); + } + if (!encoding.isEmpty()) { + builder.add("-encoding", encoding); + } + if (!xlint.isEmpty()) { + builder.add("-Xlint:" + Joiner.on(",").join(xlint)); + } + this.options = builder.addAll(misc).build(); + } + + /** + * @return the list of options as given by the {@code java_toolchain} rule. + */ + public ImmutableList<String> getJavacOptions() { + return options; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java new file mode 100644 index 0000000..3e210d8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java
@@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.List; + +/** + * Information about the JDK used by the <code>java_*</code> rules. + */ +@Immutable +public final class JavaToolchainProvider implements TransitiveInfoProvider { + + private final ImmutableList<String> javacOptions; + + public JavaToolchainProvider(String source, String target, String encoding, + ImmutableList<String> xlint, ImmutableList<String> misc, List<String> defaultJavacFlags) { + super(); + // merges the defaultJavacFlags from + // {@link JavaConfiguration} with the flags from the {@code java_toolchain} rule. + JavaToolchainData data = new JavaToolchainData(source, target, encoding, xlint, misc); + this.javacOptions = ImmutableList.<String>builder() + .addAll(data.getJavacOptions()) + .addAll(defaultJavacFlags) + .build(); + } + + /** + * @return the list of default options for the java compiler + */ + public ImmutableList<String> getJavacOptions() { + return javacOptions; + } + + /** + * An helper method to construct the list of javac options. + * + * @param ruleContext The rule context of the current rule. + * @return the list of flags provided by the {@code java_toolchain} rule merged with the one + * provided by the {@link JavaConfiguration} fragment. + */ + public static List<String> getDefaultJavacOptions(RuleContext ruleContext) { + JavaToolchainProvider javaToolchain = + ruleContext.getPrerequisite(":java_toolchain", Mode.TARGET, JavaToolchainProvider.class); + if (javaToolchain == null) { + ruleContext.ruleError("No java_toolchain implicit dependency found. This is probably because" + + " your java configuration is not up-to-date."); + return ImmutableList.of(); + } + return javaToolchain.getJavacOptions(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java new file mode 100644 index 0000000..16801ee --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java
@@ -0,0 +1,92 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.java; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.STRING; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for {@code java_toolchain} + */ +@BlazeRule(name = "java_toolchain", ancestors = {BaseRuleClasses.BaseRule.class}, + factoryClass = JavaToolchain.class) +public final class JavaToolchainRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder.setUndocumented() + /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(source_version) --> + The Java source version (e.g., '6' or '7'). It specifies which set of code structures + are allowed in the Java source code. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("source_version", STRING).mandatory()) // javac -source flag value. + /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(target_version) --> + The Java target version (e.g., '6' or '7'). It specifies for which Java runtime the class + should be build. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("target_version", STRING).mandatory()) // javac -target flag value. + /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(encoding) --> + The encoding of the java files (e.g., 'UTF-8'). + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("encoding", STRING).mandatory()) // javac -encoding flag value. + /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(xlint) --> + The list of warning to add or removes from default list. Precedes it with a dash to + removes it. Please see the Javac documentation on the -Xlint options for more information. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("xlint", STRING_LIST).value(ImmutableList.<String>of())) + /* <!-- #BLAZE_RULE(java_toolchain).ATTRIBUTE(xlint) --> + The list of extra arguments for the Java compiler. Please refer to the Java compiler + documentation for the extensive list of possible Java compiler flags. + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("misc", STRING_LIST).value(ImmutableList.<String>of())) + .build(); + } +} +/*<!-- #BLAZE_RULE (NAME = java_toolchain, TYPE = OTHER, FAMILY = Java) --> + +${ATTRIBUTE_SIGNATURE} + +<p> +Specifies the configuration for the Java compiler. Which toolchain to be used can be changed through +the --java_toolchain argument. Normally you should not write those kind of rules unless you want to +tune your Java compiler. +</p> + +${ATTRIBUTE_DEFINITION} + +<h4 id="java_binary_examples">Examples</h4> + +<p>A simple example would be: +</p> + +<pre class="code"> +java_toolchain( + name = "toolchain", + source_version = "7", + target_version = "7", + encoding = "UTF-8", + xlint = [ "classfile", "divzero", "empty", "options", "path" ], + misc = [ "-g" ], +) +</pre> + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaUtil.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaUtil.java new file mode 100644 index 0000000..b2a84ec --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaUtil.java
@@ -0,0 +1,147 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Utility methods for use by Java-related parts of the build system. + */ +public abstract class JavaUtil { + + private JavaUtil() {} + + //---------- Java related methods + + /* + * TODO(bazel-team): (2009) + * + * This way of figuring out Java source roots is basically + * broken. I think we need to support these two use cases: + * (1) A user puts his / her shell into a directory named java. + * (2) Someplace in the tree, there's a package named java. + * + * (1) is more important than (2); and (2) cannot always be guaranteed + * due to sloppy implementations in the past; most notably the old + * tools/boilerplate_rules.mk code for compiling Java. + * + * Basically, to implement correct semantics, we will need to configure + * Java source roots based on the package path, plus some heuristics to + * support legacy code, maybe. + * + * Roughly: + * Given a path, find the source root that applies to it by + * - walk over the elements in the package path + * - add "java", "javatests" to them + * - find the first element that is a maximal prefix to the Java file + * - for experimental, some legacy support that basically has some + * arbitrary padding before the Java sourceroot. + */ + + /** + * Given the filename of a Java source file, returns the name of the toplevel Java class defined + * within it. + */ + public static String getJavaClassName(PathFragment path) { + return FileSystemUtils.removeExtension(path.getBaseName()); + } + + /** + * Find the index of the "java" or "javatests" segment in a Java path fragment + * that precedes the source root. + * + * @param path a Java source dir or file path + * @return the index of the java segment or -1 iff no java segment was found. + */ + private static int javaSegmentIndex(PathFragment path) { + if (path.isAbsolute()) { + throw new IllegalArgumentException("path must not be absolute: '" + path + "'"); + } + return path.getFirstSegment(ImmutableSet.of("java", "javatests")); + } + + /** + * Given the PathFragment of a Java source file, returns the Java package to which it belongs. + */ + public static String getJavaPackageName(PathFragment path) { + int index = javaSegmentIndex(path) + 1; + path = path.subFragment(index, path.segmentCount() - 1); + return path.getPathString().replace('/', '.'); + } + + /** + * Given the PathFragment of a file without extension, returns the + * Java fully qualified class name based on the Java root relative path of the + * specified path or 'null' if no java root can be determined. + * <p> + * For example, "java/foo/bar/wiz" and "javatests/foo/bar/wiz" both + * result in "foo.bar.wiz". + * + * TODO(bazel-team): (2011) We need to have a more robust way to determine the Java root + * of a relative path rather than simply trying to find the "java" or + * "javatests" directory. + */ + public static String getJavaFullClassname(PathFragment path) { + PathFragment javaPath = getJavaPath(path); + if (javaPath != null) { + return javaPath.getPathString().replace('/', '.'); + } + return null; + } + + /** + * Given the PathFragment of a Java source file, returns the Java root relative path or 'null' if + * no java root can be determined. + * + * <p> + * For example, "{workspace}/java/foo/bar/wiz" and "{workspace}/javatests/foo/bar/wiz" + * both result in "foo/bar/wiz". + * + * TODO(bazel-team): (2011) We need to have a more robust way to determine the Java root + * of a relative path rather than simply trying to find the "java" or + * "javatests" directory. + */ + public static PathFragment getJavaPath(PathFragment path) { + int index = javaSegmentIndex(path); + if (index >= 0) { + return path.subFragment(index + 1, path.segmentCount()); + } + return null; + } + + /** + * Given the PathFragment of a Java source file, returns the + * Java root of the specified path or 'null' if no java root can be + * determined. + * <p> + * Example 1: "{workspace}/java/foo/bar/wiz" and "{workspace}/javatests/foo/bar/wiz" + * result in "{workspace}/java" and "{workspace}/javatests" Example 2: + * "java/foo/bar/wiz" and "javatests/foo/bar/wiz" result in "java" and + * "javatests" + * + * TODO(bazel-team): (2011) We need to have a more robust way to determine the Java root + * of a relative path rather than simply trying to find the "java" or + * "javatests" directory. + */ + public static PathFragment getJavaRoot(PathFragment path) { + int index = javaSegmentIndex(path); + if (index >= 0) { + return path.subFragment(0, index + 1); + } + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/Jvm.java b/src/main/java/com/google/devtools/build/lib/rules/java/Jvm.java new file mode 100644 index 0000000..eda7360 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/Jvm.java
@@ -0,0 +1,120 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap.Builder; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * This class represents a Java virtual machine with a host system and a path. + * If the JVM comes from the client, it can optionally also contain a label + * pointing to a target that contains all the necessary files. + */ +@SkylarkModule(name = "jvm", + doc = "A configuration fragment representing the Java virtual machine.") +@Immutable +public final class Jvm extends BuildConfiguration.Fragment { + private final PathFragment javaHome; + private final Label jvmLabel; + + /** + * Creates a Jvm instance. Either the {@code javaHome} parameter is absolute, + * or the {@code jvmLabel} parameter must be non-null. This restriction might + * be lifted in the future. Only the {@code jvmLabel} is optional. + */ + public Jvm(PathFragment javaHome, Label jvmLabel) { + Preconditions.checkArgument(javaHome.isAbsolute() ^ (jvmLabel != null)); + this.javaHome = javaHome; + this.jvmLabel = jvmLabel; + } + + @Override + public String getName() { + return "Jvm"; + } + + @Override + public void addImplicitLabels(Multimap<String, Label> implicitLabels) { + if (jvmLabel != null) { + implicitLabels.put(getName(), jvmLabel); + } + } + + /** + * Returns a path fragment that determines the path to the installation + * directory. It is either absolute or relative to the execution root. + */ + public PathFragment getJavaHome() { + return javaHome; + } + + /** + * Returns the path to the javac binary. + */ + public PathFragment getJavacExecutable() { + return getJavaHome().getRelative("bin/javac"); + } + + /** + * Returns the path to the jar binary. + */ + public PathFragment getJarExecutable() { + return getJavaHome().getRelative("bin/jar"); + } + + /** + * Returns the path to the java binary. + */ + @SkylarkCallable(name = "java_executable", structField = true, + doc = "The the java executable, i.e. bin/java relative to the Java home.") + public PathFragment getJavaExecutable() { + return getJavaHome().getRelative("bin/java"); + } + + /** + * Returns a label. Adding this label to the dependencies of an action that + * depends on this JVM is sufficient to ensure that all the required files are + * present. Can be <code>null</code>, in which case nothing needs to be added + * to the dependencies of an action. We rely on convention to make sure that + * this case works, since we can't know which JVMs are installed on the build host. + */ + public Label getJvmLabel() { + return jvmLabel; + } + + /** + * Returns a string that uniquely identifies the JVM for the life time of this + * Blaze instance. This value is intended for analysis caching, so it need not + * reflect changes in the individual files making up the JVM. + */ + @Override + public String cacheKey() { + return javaHome.getSafePathString(); + } + + @Override + public void addGlobalMakeVariables(Builder<String, String> globalMakeEnvBuilder) { + globalMakeEnvBuilder.put("JAVABASE", getJavaHome().getPathString()); + globalMakeEnvBuilder.put("JAVA", getJavaExecutable().getPathString()); + globalMakeEnvBuilder.put("JAVAC", getJavacExecutable().getPathString()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JvmConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/java/JvmConfigurationLoader.java new file mode 100644 index 0000000..7483f88 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JvmConfigurationLoader.java
@@ -0,0 +1,163 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.lib.analysis.RedirectChaser; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.RawAttributeMapper; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.List; + +import javax.annotation.Nullable; + +/** + * A provider to load jvm configurations from the package path. + * + * <p>If the given {@code javaHome} is a label, i.e. starts with {@code "//"}, + * then the loader will look at the target it refers to. If the target is a + * filegroup, then the loader will look in it's srcs for a filegroup that ends + * with {@code -<cpu>}. It will use that filegroup to construct the actual + * {@link Jvm} instance, using the filegroups {@code path} attribute to + * construct the new {@code javaHome} path. + * + * <p>The loader also supports legacy mode, where the JVM can be defined with an abolute path. + */ +public final class JvmConfigurationLoader implements ConfigurationFragmentFactory { + private final boolean forceLegacy; + private final JavaCpuSupplier cpuSupplier; + + public JvmConfigurationLoader(boolean forceLegacy, JavaCpuSupplier cpuSupplier) { + this.forceLegacy = forceLegacy; + this.cpuSupplier = cpuSupplier; + } + + public JvmConfigurationLoader(JavaCpuSupplier cpuSupplier) { + this(/*forceLegacy=*/ false, cpuSupplier); + } + + @Override + public Jvm create(ConfigurationEnvironment env, BuildOptions buildOptions) + throws InvalidConfigurationException { + JavaOptions javaOptions = buildOptions.get(JavaOptions.class); + String javaHome = javaOptions.javaBase; + String cpu = cpuSupplier.getJavaCpu(buildOptions, env); + if (cpu == null) { + return null; + } + + if (!forceLegacy && javaHome.startsWith("//")) { + return createDefault(env, javaHome, cpu); + } else { + return createLegacy(javaHome); + } + } + + @Override + public Class<? extends Fragment> creates() { + return Jvm.class; + } + + @Nullable + private Jvm createDefault(ConfigurationEnvironment lookup, String javaHome, String cpu) + throws InvalidConfigurationException { + try { + Label label = Label.parseAbsolute(javaHome); + label = RedirectChaser.followRedirects(lookup, label, "jdk"); + if (label == null) { + return null; + } + Target javaHomeTarget = lookup.getTarget(label); + if (javaHomeTarget == null) { + return null; + } + if ((javaHomeTarget instanceof Rule) && + "filegroup".equals(((Rule) javaHomeTarget).getRuleClass())) { + RawAttributeMapper javaHomeAttributes = RawAttributeMapper.of((Rule) javaHomeTarget); + if (javaHomeAttributes.isConfigurable("srcs", Type.LABEL_LIST)) { + throw new InvalidConfigurationException("\"srcs\" in " + javaHome + + " is configurable. JAVABASE targets don't support configurable attributes"); + } + List<Label> labels = javaHomeAttributes.get("srcs", Type.LABEL_LIST); + for (Label jvmLabel : labels) { + if (jvmLabel.getName().endsWith("-" + cpu)) { + Target jvmTarget = lookup.getTarget(jvmLabel); + if (jvmTarget == null) { + return null; + } + PathFragment javaHomePath = jvmLabel.getPackageFragment(); + if ((jvmTarget instanceof Rule) && + "filegroup".equals(((Rule) jvmTarget).getRuleClass())) { + RawAttributeMapper jvmTargetAttributes = RawAttributeMapper.of((Rule) jvmTarget); + if (jvmTargetAttributes.isConfigurable("path", Type.STRING)) { + throw new InvalidConfigurationException("\"path\" in " + jvmTarget + + " is configurable. JVM targets don't support configurable attributes"); + } + String path = jvmTargetAttributes.get("path", Type.STRING); + if (path != null) { + javaHomePath = javaHomePath.getRelative(path); + } + } + return new Jvm(javaHomePath, jvmLabel); + } + } + } + throw new InvalidConfigurationException("No JVM target found under " + javaHome + + " that would work for " + cpu); + } catch (NoSuchPackageException e) { + throw new InvalidConfigurationException(e.getMessage(), e); + } catch (NoSuchTargetException e) { + throw new InvalidConfigurationException("No such target: " + e.getMessage(), e); + } catch (SyntaxException e) { + throw new InvalidConfigurationException(e.getMessage(), e); + } + } + + private Jvm createLegacy(String javaHome) + throws InvalidConfigurationException { + if (!javaHome.startsWith("/")) { + throw new InvalidConfigurationException("Illegal javabase value '" + javaHome + + "', javabase must be an absolute path or label"); + } + return new Jvm(new PathFragment(javaHome), null); + } + + /** + * Converts the cpu name to a GNU system name. If the cpu is not a known value, it returns + * <code>"unknown-unknown-linux-gnu"</code>. + */ + @VisibleForTesting + static String convertCpuToGnuSystemName(String cpu) { + if ("piii".equals(cpu)) { + return "i686-unknown-linux-gnu"; + } else if ("k8".equals(cpu)) { + return "x86_64-unknown-linux-gnu"; + } else { + return "unknown-unknown-linux-gnu"; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/MessageBundleProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/MessageBundleProvider.java new file mode 100644 index 0000000..f78e386 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/MessageBundleProvider.java
@@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Marks configured targets that are able to supply message bundles to their + * dependents. + */ +@Immutable +public final class MessageBundleProvider implements TransitiveInfoProvider { + + private final ImmutableList<Artifact> messages; + + public MessageBundleProvider(ImmutableList<Artifact> messages) { + this.messages = messages; + } + + /** + * The set of XML source files containing the message definitions. + */ + public ImmutableList<Artifact> getMessages() { + return messages; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/NativeLibraryNestedSetBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/java/NativeLibraryNestedSetBuilder.java new file mode 100644 index 0000000..09ac59f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/NativeLibraryNestedSetBuilder.java
@@ -0,0 +1,115 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.cpp.CcNativeLibraryProvider; +import com.google.devtools.build.lib.rules.cpp.CppFileTypes; +import com.google.devtools.build.lib.rules.cpp.LinkerInput; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs; +import com.google.devtools.build.lib.util.FileType; + +/** + * A builder that helps construct nested sets of native libraries. + */ +public final class NativeLibraryNestedSetBuilder { + + private final NestedSetBuilder<LinkerInput> builder = NestedSetBuilder.linkOrder(); + + /** + * Build a nested set of native libraries. + */ + public NestedSet<LinkerInput> build() { + return builder.build(); + } + + /** + * Include specified artifacts as native libraries in the nested set. + */ + public NativeLibraryNestedSetBuilder addAll(Iterable<Artifact> deps) { + for (Artifact dep : deps) { + builder.add(new LinkerInputs.SimpleLinkerInput(dep)); + } + return this; + } + + /** + * Include native libraries of specified dependencies into the nested set. + */ + public NativeLibraryNestedSetBuilder addJavaTargets( + Iterable<? extends TransitiveInfoCollection> deps) { + for (TransitiveInfoCollection dep : deps) { + addJavaTarget(dep); + } + return this; + } + + /** + * Include native Java libraries of a specified target into the nested set. + */ + private void addJavaTarget(TransitiveInfoCollection dep) { + JavaNativeLibraryProvider javaProvider = dep.getProvider(JavaNativeLibraryProvider.class); + if (javaProvider != null) { + builder.addTransitive(javaProvider.getTransitiveJavaNativeLibraries()); + return; + } + + CcNativeLibraryProvider ccProvider = dep.getProvider(CcNativeLibraryProvider.class); + if (ccProvider != null) { + builder.addTransitive(ccProvider.getTransitiveCcNativeLibraries()); + return; + } + + addTarget(dep); + } + + /** + * Include native C/C++ libraries of specified dependencies into the nested set. + */ + public NativeLibraryNestedSetBuilder addCcTargets( + Iterable<? extends TransitiveInfoCollection> deps) { + for (TransitiveInfoCollection dep : deps) { + addCcTarget(dep); + } + return this; + } + + /** + * Include native Java libraries of a specified target into the nested set. + */ + private void addCcTarget(TransitiveInfoCollection dep) { + CcNativeLibraryProvider provider = dep.getProvider(CcNativeLibraryProvider.class); + if (provider != null) { + builder.addTransitive(provider.getTransitiveCcNativeLibraries()); + } else { + addTarget(dep); + } + } + + /** + * Include files and genrule artifacts. + */ + private void addTarget(TransitiveInfoCollection dep) { + for (Artifact artifact : FileType.filterList( + dep.getProvider(FileProvider.class).getFilesToBuild(), + CppFileTypes.SHARED_LIBRARY)) { + builder.add(new LinkerInputs.SimpleLinkerInput(artifact)); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/SourcesJavaCompilationArgsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/SourcesJavaCompilationArgsProvider.java new file mode 100644 index 0000000..ff8507e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/SourcesJavaCompilationArgsProvider.java
@@ -0,0 +1,60 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * An interface that marks configured targets that can provide Java compilation arguments through + * the 'srcs' attribute of Java rules. + * + * <p>In a perfect world, this would not be necessary for a million reasons, but + * this world is far from perfect, thus, we need this. + * + * <p>Please do not implement this interface with configured target implementations. + */ +@Immutable +public final class SourcesJavaCompilationArgsProvider implements TransitiveInfoProvider { + private final JavaCompilationArgs javaCompilationArgs; + private final JavaCompilationArgs recursiveJavaCompilationArgs; + + public SourcesJavaCompilationArgsProvider( + JavaCompilationArgs javaCompilationArgs, + JavaCompilationArgs recursiveJavaCompilationArgs) { + this.javaCompilationArgs = javaCompilationArgs; + this.recursiveJavaCompilationArgs = recursiveJavaCompilationArgs; + } + + /** + * Returns non-recursively collected Java compilation information for + * building this target (called when strict_java_deps = 1). + * + * <p>Note that some of the parameters are still collected from the complete + * transitive closure. The non-recursive collection applies mainly to + * compile-time jars. + */ + public JavaCompilationArgs getJavaCompilationArgs() { + return javaCompilationArgs; + } + + /** + * Returns recursively collected Java compilation information for building + * this target (called when strict_java_deps = 0). + */ + public JavaCompilationArgs getRecursiveJavaCompilationArgs() { + return recursiveJavaCompilationArgs; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/WriteBuildInfoPropertiesAction.java b/src/main/java/com/google/devtools/build/lib/rules/java/WriteBuildInfoPropertiesAction.java new file mode 100644 index 0000000..08dcbba --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/java/WriteBuildInfoPropertiesAction.java
@@ -0,0 +1,211 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.java; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.analysis.BuildInfoHelper; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Key; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +/** + * An action that creates a Java properties file containing the build informations. + */ +public class WriteBuildInfoPropertiesAction extends AbstractFileWriteAction { + private static final String GUID = "922949ca-1391-4046-a300-74810618dcdc"; + + private final ImmutableList<Artifact> valueArtifacts; + private final BuildInfoPropertiesTranslator keyTranslations; + private final boolean includeVolatile; + private final boolean includeNonVolatile; + + private final TimestampFormatter timestampFormatter; + /** + * An interface to format a timestamp. We are using our custom one to avoid external dependency. + */ + public static interface TimestampFormatter { + /** + * Return a human readable string for the given {@code timestamp}. {@code timestamp} is given + * in milliseconds since 1st of January 1970 at 0am UTC. + */ + public String format(long timestamp); + } + + /** + * A wrapper around a {@link Writer} that skips the first line assuming the line is pure ASCII. It + * can be used to strip the timestamp comment that {@link Properties#store(Writer, String)} adds. + */ + @VisibleForTesting + static class StripFirstLineWriter extends Writer { + private final Writer writer; + private boolean newlineFound = false; + + StripFirstLineWriter(OutputStream out) { + this.writer = new OutputStreamWriter(out, UTF_8); + } + + @Override + public void write(char[] cbuf, int off, int len) throws IOException { + if (!newlineFound) { + while (len > 0 && cbuf[off] != '\n') { + off++; + len--; + } + if (len > 0) { + newlineFound = true; + off++; + len--; + } + } + if (len > 0) { + writer.write(cbuf, off, len); + } + } + + @Override + public void flush() throws IOException { + writer.flush(); + } + + @Override + public void close() throws IOException { + writer.close(); + } + + } + + /** + * Creates an action that writes a Java property files with build information. + * + * <p>It reads the set of build info keys from an action context that is usually contributed to + * Blaze by the workspace status module, and the value associated with said keys from the + * workspace status files (stable and volatile) written by the workspace status action. The files + * generated by this action serve as input to the + * {@link com.google.devtools.build.singlejar.SingleJar} program. + * + * <p>Without input artifacts, this action uses redacted build information. + * + * @param inputs Artifacts that contain build information, or an empty collection to use redacted + * build information + * @param output output the properties file Artifact created by this action + * @param keyTranslations how to translates available keys. See + * {@link BuildInfoPropertiesTranslator}. + * @param includeVolatile whether the set of key to write are giving volatile keys or not + * @param includeNonVolatile whether the set of key to write are giving non-volatile keys or not + * @param timestampFormatter formats dates printed in the properties file + */ + public WriteBuildInfoPropertiesAction(Collection<Artifact> inputs, Artifact output, + BuildInfoPropertiesTranslator keyTranslations, boolean includeVolatile, + boolean includeNonVolatile, TimestampFormatter timestampFormatter) { + super(BuildInfoHelper.BUILD_INFO_ACTION_OWNER, inputs, output, /* makeExecutable= */false); + this.keyTranslations = keyTranslations; + this.includeVolatile = includeVolatile; + this.includeNonVolatile = includeNonVolatile; + this.timestampFormatter = timestampFormatter; + valueArtifacts = ImmutableList.copyOf(inputs); + + if (!inputs.isEmpty()) { + // With non-empty inputs we should not generate both volatile and non-volatile data + // in the same properties file. + Preconditions.checkState(includeVolatile ^ includeNonVolatile); + } + Preconditions.checkState( + output.isConstantMetadata() == (includeVolatile && !inputs.isEmpty())); + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, + final Executor executor) { + final long timestamp = System.currentTimeMillis(); + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + WorkspaceStatusAction.Context context = + executor.getContext(WorkspaceStatusAction.Context.class); + Map<String, String> values = new LinkedHashMap<>(); + for (Artifact valueFile : valueArtifacts) { + values.putAll(WorkspaceStatusAction.parseValues(valueFile.getPath())); + } + + Map<String, String> keys = new HashMap<>(); + if (includeVolatile) { + addValues(keys, values, context.getVolatileKeys()); + keys.put("BUILD_TIMESTAMP", Long.toString(timestamp / 1000)); + keys.put("BUILD_TIME", timestampFormatter.format(timestamp)); + } + addValues(keys, values, context.getStableKeys()); + Properties properties = new Properties(); + keyTranslations.translate(keys, properties); + properties.store(new StripFirstLineWriter(out), null); + } + }; + } + + private void addValues(Map<String, String> result, Map<String, String> values, + Map<String, Key> keys) { + boolean redacted = values.isEmpty(); + for (Map.Entry<String, WorkspaceStatusAction.Key> key : keys.entrySet()) { + if (key.getValue().isInLanguage("Java")) { + result.put(key.getKey(), gePropertyValue(values, redacted, key)); + } + } + } + + private static String gePropertyValue(Map<String, String> values, boolean redacted, + Map.Entry<String, WorkspaceStatusAction.Key> key) { + return redacted ? key.getValue().getRedactedValue() + : values.containsKey(key.getKey()) ? values.get(key.getKey()) + : key.getValue().getDefaultValue(); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addString(keyTranslations.computeKey()); + f.addBoolean(includeVolatile); + f.addBoolean(includeNonVolatile); + return f.hexDigestAndReset(); + } + + @Override + public boolean executeUnconditionally() { + return isVolatile(); + } + + @Override + public boolean isVolatile() { + return includeVolatile && !Iterables.isEmpty(getInputs()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ApplicationSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ApplicationSupport.java new file mode 100644 index 0000000..73554e5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ApplicationSupport.java
@@ -0,0 +1,588 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates; +import static com.google.devtools.build.xcode.common.TargetDeviceFamily.UI_DEVICE_FAMILY_VALUES; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.BinaryFileWriteAction; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraActoolArgs; +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.common.InvalidFamilyNameException; +import com.google.devtools.build.xcode.common.Platform; +import com.google.devtools.build.xcode.common.RepeatedFamilyNameException; +import com.google.devtools.build.xcode.common.TargetDeviceFamily; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.XcodeprojBuildSetting; + +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Support for application-generating ObjC rules. An application is generally composed of a + * top-level {@link BundleSupport bundle}, potentially signed, as well as some debug information, if + * {@link ObjcConfiguration#generateDebugSymbols() requested}. + * + * <p>Contains actions, validation logic and provider value generation. + * + * <p>Methods on this class can be called in any order without impacting the result. + */ +public final class ApplicationSupport { + + /** + * Template for the containing application folder. + */ + public static final SafeImplicitOutputsFunction IPA = fromTemplates("%{name}.ipa"); + + @VisibleForTesting + static final String NO_ASSET_CATALOG_ERROR_FORMAT = + "a value was specified (%s), but this app does not have any asset catalogs"; + @VisibleForTesting + static final String INVALID_FAMILIES_ERROR = + "Expected one or two strings from the list 'iphone', 'ipad'"; + @VisibleForTesting + static final String DEVICE_NO_PROVISIONING_PROFILE = + "Provisioning profile must be set for device build"; + + @VisibleForTesting + static final String PROVISIONING_PROFILE_BUNDLE_FILE = "embedded.mobileprovision"; + + private final Attributes attributes; + private final BundleSupport bundleSupport; + private final RuleContext ruleContext; + private final Bundling bundling; + private final ObjcProvider objcProvider; + private final LinkedBinary linkedBinary; + private final ImmutableSet<TargetDeviceFamily> families; + private final IntermediateArtifacts intermediateArtifacts; + + /** + * Indicator as to whether this rule generates a binary directly or whether only dependencies + * should be considered. + */ + enum LinkedBinary { + /** + * This rule generates its own binary which should be included as well as dependency-generated + * binaries. + */ + LOCAL_AND_DEPENDENCIES, + + /** + * This rule does not generate its own binary, only consider binaries from dependencies. + */ + DEPENDENCIES_ONLY + } + + /** + * Creates a new application support within the given rule context. + * + * @param ruleContext context for the application-generating rule + * @param objcProvider provider containing all dependencies' information as well as some of this + * rule's + * @param optionsProvider provider containing options and plist settings for this rule and its + * dependencies + * @param linkedBinary whether to look for a linked binary from this rule and dependencies or just + * the latter + */ + ApplicationSupport( + RuleContext ruleContext, ObjcProvider objcProvider, OptionsProvider optionsProvider, + LinkedBinary linkedBinary) { + this.linkedBinary = linkedBinary; + this.attributes = new Attributes(ruleContext); + this.ruleContext = ruleContext; + this.objcProvider = objcProvider; + this.families = ImmutableSet.copyOf(attributes.families()); + this.intermediateArtifacts = ObjcRuleClasses.intermediateArtifacts(ruleContext); + bundling = bundling(ruleContext, objcProvider, optionsProvider); + bundleSupport = new BundleSupport(ruleContext, families, bundling, extraActoolArgs()); + } + + /** + * Validates application-related attributes set on this rule and registers any errors with the + * rule context. + * + * @return this application support + */ + ApplicationSupport validateAttributes() { + bundleSupport.validateAttributes(); + + // No asset catalogs. That means you cannot specify app_icon or + // launch_image attributes, since they must not exist. However, we don't + // run actool in this case, which means it does not do validity checks, + // and we MUST raise our own error somehow... + if (!objcProvider.hasAssetCatalogs()) { + if (attributes.appIcon() != null) { + ruleContext.attributeError("app_icon", + String.format(NO_ASSET_CATALOG_ERROR_FORMAT, attributes.appIcon())); + } + if (attributes.launchImage() != null) { + ruleContext.attributeError("launch_image", + String.format(NO_ASSET_CATALOG_ERROR_FORMAT, attributes.launchImage())); + } + } + + if (families.isEmpty()) { + ruleContext.attributeError("families", INVALID_FAMILIES_ERROR); + } + + return this; + } + + /** + * Registers actions required to build an application. This includes any + * {@link BundleSupport#registerActions(ObjcProvider) bundle} and bundle merge actions, signing + * this application if appropriate and combining several single-architecture binaries into one + * multi-architecture binary. + * + * @return this application support + */ + ApplicationSupport registerActions() { + bundleSupport.registerActions(objcProvider); + + registerCombineArchitecturesAction(); + + ObjcConfiguration objcConfiguration = ObjcRuleClasses.objcConfiguration(ruleContext); + Artifact ipaOutput = ruleContext.getImplicitOutputArtifact(IPA); + + Artifact maybeSignedIpa; + if (objcConfiguration.getPlatform() == Platform.SIMULATOR) { + maybeSignedIpa = ipaOutput; + } else if (attributes.provisioningProfile() == null) { + throw new IllegalStateException(DEVICE_NO_PROVISIONING_PROFILE); + } else { + maybeSignedIpa = registerBundleSigningActions(ipaOutput); + } + + BundleMergeControlBytes bundleMergeControlBytes = new BundleMergeControlBytes( + bundling, maybeSignedIpa, objcConfiguration, families); + registerBundleMergeActions( + maybeSignedIpa, bundling.getBundleContentArtifacts(), bundleMergeControlBytes); + + return this; + } + + private Artifact registerBundleSigningActions(Artifact ipaOutput) { + PathFragment entitlementsDirectory = ruleContext.getUniqueDirectory("entitlements"); + Artifact teamPrefixFile = ruleContext.getRelatedArtifact( + entitlementsDirectory, ".team_prefix_file"); + registerExtractTeamPrefixAction(teamPrefixFile); + + Artifact entitlementsNeedingSubstitution = attributes.entitlements(); + if (entitlementsNeedingSubstitution == null) { + entitlementsNeedingSubstitution = ruleContext.getRelatedArtifact( + entitlementsDirectory, ".entitlements_with_variables"); + registerExtractEntitlementsAction(entitlementsNeedingSubstitution); + } + Artifact entitlements = ruleContext.getRelatedArtifact( + entitlementsDirectory, ".entitlements"); + registerEntitlementsVariableSubstitutionAction( + entitlementsNeedingSubstitution, entitlements, teamPrefixFile); + Artifact ipaUnsigned = ObjcRuleClasses.artifactByAppendingToRootRelativePath( + ruleContext, ipaOutput.getExecPath(), ".unsigned"); + registerSignBundleAction(entitlements, ipaOutput, ipaUnsigned); + return ipaUnsigned; + } + + /** + * Adds bundle- and application-related settings to the given Xcode provider builder. + * + * @return this application support + */ + ApplicationSupport addXcodeSettings(XcodeProvider.Builder xcodeProviderBuilder) { + bundleSupport.addXcodeSettings(xcodeProviderBuilder); + xcodeProviderBuilder.addXcodeprojBuildSettings(buildSettings()); + + return this; + } + + /** + * Adds any files to the given nested set builder that should be built if this application is the + * top level target in a blaze invocation. + * + * @return this application support + */ + ApplicationSupport addFilesToBuild(NestedSetBuilder<Artifact> filesToBuild) { + NestedSetBuilder<Artifact> debugSymbolBuilder = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(objcProvider.get(ObjcProvider.DEBUG_SYMBOLS)); + + if (linkedBinary == LinkedBinary.LOCAL_AND_DEPENDENCIES + && ObjcRuleClasses.objcConfiguration(ruleContext).generateDebugSymbols()) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + debugSymbolBuilder.add(intermediateArtifacts.dsymPlist()) + .add(intermediateArtifacts.dsymSymbol()) + .add(intermediateArtifacts.breakpadSym()); + } + + filesToBuild.add(ruleContext.getImplicitOutputArtifact(ApplicationSupport.IPA)) + // TODO(bazel-team): Fat binaries may require some merging of these file rather than just + // making them available. + .addTransitive(debugSymbolBuilder.build()); + return this; + } + + /** + * Creates the {@link XcTestAppProvider} that can be used if this application is used as an + * {@code xctest_app}. + */ + XcTestAppProvider xcTestAppProvider() { + // We want access to #import-able things from our test rig's dependency graph, but we don't + // want to link anything since that stuff is shared automatically by way of the + // -bundle_loader linker flag. + ObjcProvider partialObjcProvider = new ObjcProvider.Builder() + .addTransitiveAndPropagate(ObjcProvider.HEADER, objcProvider) + .addTransitiveAndPropagate(ObjcProvider.INCLUDE, objcProvider) + .addTransitiveAndPropagate(ObjcProvider.SDK_DYLIB, objcProvider) + .addTransitiveAndPropagate(ObjcProvider.SDK_FRAMEWORK, objcProvider) + .addTransitiveAndPropagate(ObjcProvider.WEAK_SDK_FRAMEWORK, objcProvider) + .addTransitiveAndPropagate(ObjcProvider.FRAMEWORK_DIR, objcProvider) + .addTransitiveAndPropagate(ObjcProvider.FRAMEWORK_FILE, objcProvider) + .build(); + // TODO(bazel-team): Handle the FRAMEWORK_DIR key properly. We probably want to add it to + // framework search paths, but not actually link it with the -framework flag. + return new XcTestAppProvider(intermediateArtifacts.singleArchitectureBinary(), + ruleContext.getImplicitOutputArtifact(IPA), partialObjcProvider); + } + + private ExtraActoolArgs extraActoolArgs() { + ImmutableList.Builder<String> extraArgs = ImmutableList.builder(); + if (attributes.appIcon() != null) { + extraArgs.add("--app-icon", attributes.appIcon()); + } + if (attributes.launchImage() != null) { + extraArgs.add("--launch-image", attributes.launchImage()); + } + return new ExtraActoolArgs(extraArgs.build()); + } + + private Bundling bundling( + RuleContext ruleContext, ObjcProvider objcProvider, OptionsProvider optionsProvider) { + ImmutableList<BundleableFile> extraBundleFiles; + ObjcConfiguration objcConfiguration = ObjcRuleClasses.objcConfiguration(ruleContext); + if (objcConfiguration.getPlatform() == Platform.DEVICE) { + extraBundleFiles = ImmutableList.of(new BundleableFile( + attributes.provisioningProfile(), + PROVISIONING_PROFILE_BUNDLE_FILE)); + } else { + extraBundleFiles = ImmutableList.of(); + } + + return new Bundling.Builder() + .setName(ruleContext.getLabel().getName()) + .setBundleDirSuffix(".app") + .setExtraBundleFiles(extraBundleFiles) + .setObjcProvider(objcProvider) + .setInfoplistMerging( + BundleSupport.infoPlistMerging(ruleContext, objcProvider, optionsProvider)) + .setIntermediateArtifacts(intermediateArtifacts) + .build(); + } + + private void registerCombineArchitecturesAction() { + Artifact resultingLinkedBinary = intermediateArtifacts.combinedArchitectureBinary(".app"); + NestedSet<Artifact> linkedBinaries = linkedBinaries(); + + ruleContext.registerAction(ObjcActionsBuilder.spawnOnDarwinActionBuilder() + .setMnemonic("ObjcCombiningArchitectures") + .addTransitiveInputs(linkedBinaries) + .addOutput(resultingLinkedBinary) + .setExecutable(ObjcActionsBuilder.LIPO) + .setCommandLine(CustomCommandLine.builder() + .addExecPaths("-create", linkedBinaries) + .addExecPath("-o", resultingLinkedBinary) + .build()) + .build(ruleContext)); + } + + private NestedSet<Artifact> linkedBinaries() { + NestedSetBuilder<Artifact> linkedBinariesBuilder = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(attributes.dependentLinkedBinaries()); + if (linkedBinary == LinkedBinary.LOCAL_AND_DEPENDENCIES) { + linkedBinariesBuilder.add(intermediateArtifacts.singleArchitectureBinary()); + } + return linkedBinariesBuilder.build(); + } + + /** Returns this target's Xcode build settings. */ + private Iterable<XcodeprojBuildSetting> buildSettings() { + ImmutableList.Builder<XcodeprojBuildSetting> buildSettings = new ImmutableList.Builder<>(); + if (attributes.appIcon() != null) { + buildSettings.add(XcodeprojBuildSetting.newBuilder() + .setName("ASSETCATALOG_COMPILER_APPICON_NAME") + .setValue(attributes.appIcon()) + .build()); + } + if (attributes.launchImage() != null) { + buildSettings.add(XcodeprojBuildSetting.newBuilder() + .setName("ASSETCATALOG_COMPILER_LAUNCHIMAGE_NAME") + .setValue(attributes.launchImage()) + .build()); + } + + // Convert names to a sequence containing "1" and/or "2" for iPhone and iPad, respectively. + Iterable<Integer> familyIndexes = + families.isEmpty() ? ImmutableList.<Integer>of() : UI_DEVICE_FAMILY_VALUES.get(families); + buildSettings.add(XcodeprojBuildSetting.newBuilder() + .setName("TARGETED_DEVICE_FAMILY") + .setValue(Joiner.on(',').join(familyIndexes)) + .build()); + + Artifact entitlements = attributes.entitlements(); + if (entitlements != null) { + buildSettings.add(XcodeprojBuildSetting.newBuilder() + .setName("CODE_SIGN_ENTITLEMENTS") + .setValue("$(WORKSPACE_ROOT)/" + entitlements.getExecPathString()) + .build()); + } + + return buildSettings.build(); + } + + private ApplicationSupport registerSignBundleAction( + Artifact entitlements, Artifact ipaOutput, Artifact ipaUnsigned) { + // TODO(bazel-team): Support variable substitution + ruleContext.registerAction(ObjcActionsBuilder.spawnOnDarwinActionBuilder() + .setMnemonic("IosSignBundle") + .setProgressMessage("Signing iOS bundle: " + ruleContext.getLabel()) + .setExecutable(new PathFragment("/bin/bash")) + .addArgument("-c") + // TODO(bazel-team): Support --resource-rules for resources + .addArgument("set -e && " + + "t=$(mktemp -d -t signing_intermediate) && " + // Get an absolute path since we need to cd into the temp directory for zip. + + "signed_ipa=${PWD}/" + ipaOutput.getExecPathString() + " && " + + "unzip -qq " + ipaUnsigned.getExecPathString() + " -d ${t} && " + + codesignCommand( + attributes.provisioningProfile(), + entitlements, + String.format("${t}/Payload/%s.app", ruleContext.getLabel().getName())) + " && " + // Using zip since we need to preserve permissions + + "cd \"${t}\" && /usr/bin/zip -q -r \"${signed_ipa}\" .") + .addInput(ipaUnsigned) + .addInput(attributes.provisioningProfile()) + .addInput(entitlements) + .addOutput(ipaOutput) + .build(ruleContext)); + + return this; + } + + private void registerBundleMergeActions(Artifact ipaUnsigned, + NestedSet<Artifact> bundleContentArtifacts, BundleMergeControlBytes controlBytes) { + Artifact bundleMergeControlArtifact = + ObjcRuleClasses.artifactByAppendingToBaseName(ruleContext, ".ipa-control"); + + ruleContext.registerAction( + new BinaryFileWriteAction( + ruleContext.getActionOwner(), bundleMergeControlArtifact, controlBytes, + /*makeExecutable=*/false)); + + ruleContext.registerAction(new SpawnAction.Builder() + .setMnemonic("IosBundle") + .setProgressMessage("Bundling iOS application: " + ruleContext.getLabel()) + .setExecutable(attributes.bundleMergeExecutable()) + .addInputArgument(bundleMergeControlArtifact) + .addTransitiveInputs(bundleContentArtifacts) + .addOutput(ipaUnsigned) + .build(ruleContext)); + } + + private void registerExtractTeamPrefixAction(Artifact teamPrefixFile) { + ruleContext.registerAction(ObjcActionsBuilder.spawnOnDarwinActionBuilder() + .setMnemonic("ExtractIosTeamPrefix") + .setExecutable(new PathFragment("/bin/bash")) + .addArgument("-c") + .addArgument("set -e &&" + + " PLIST=$(" + extractPlistCommand(attributes.provisioningProfile()) + ") && " + + // We think PlistBuddy uses PRead internally to seek through the file. Or possibly + // mmaps the file. Or something similar. + // + // Pipe FDs do not support PRead or mmap, though. + // + // <<< however does something magical like write to a temporary file or something + // like that internally, which means that this Just Works. + + " PREFIX=$(/usr/libexec/PlistBuddy -c 'Print ApplicationIdentifierPrefix:0'" + + " /dev/stdin <<< \"${PLIST}\") && " + + " echo ${PREFIX} > " + teamPrefixFile.getExecPathString()) + .addInput(attributes.provisioningProfile()) + .addOutput(teamPrefixFile) + .build(ruleContext)); + } + + private ApplicationSupport registerExtractEntitlementsAction(Artifact entitlements) { + // See Apple Glossary (http://goo.gl/EkhXOb) + // An Application Identifier is constructed as: TeamID.BundleID + // TeamID is extracted from the provisioning profile. + // BundleID consists of a reverse-DNS string to identify the app, where the last component + // is the application name, and is specified as an attribute. + + ruleContext.registerAction(ObjcActionsBuilder.spawnOnDarwinActionBuilder() + .setMnemonic("ExtractIosEntitlements") + .setProgressMessage("Extracting entitlements: " + ruleContext.getLabel()) + .setExecutable(new PathFragment("/bin/bash")) + .addArgument("-c") + .addArgument("set -e && " + + "PLIST=$(" + + extractPlistCommand(attributes.provisioningProfile()) + ") && " + + // We think PlistBuddy uses PRead internally to seek through the file. Or possibly + // mmaps the file. Or something similar. + // + // Pipe FDs do not support PRead or mmap, though. + // + // <<< however does something magical like write to a temporary file or something + // like that internally, which means that this Just Works. + + + "/usr/libexec/PlistBuddy -x -c 'Print Entitlements' /dev/stdin <<< \"${PLIST}\" " + + "> " + entitlements.getExecPathString()) + .addInput(attributes.provisioningProfile()) + .addOutput(entitlements) + .build(ruleContext)); + + return this; + } + + private void registerEntitlementsVariableSubstitutionAction(Artifact in, Artifact out, + Artifact prefix) { + String escapedBundleId = ShellUtils.shellEscape(attributes.bundleId()); + ruleContext.registerAction(new SpawnAction.Builder() + .setMnemonic("SubstituteIosEntitlements") + .setExecutable(new PathFragment("/bin/bash")) + .addArgument("-c") + .addArgument("set -e && " + + "PREFIX=\"$(cat " + prefix.getExecPathString() + ")\" && " + + "sed " + in.getExecPathString() + " " + // Replace .* from default entitlements file with bundle ID where suitable. + + "-e \"s#${PREFIX}\\.\\*#${PREFIX}." + escapedBundleId + "#g\" " + + // Replace some variables that people put in their own entitlements files + + "-e \"s#\\$(AppIdentifierPrefix)#${PREFIX}.#g\" " + + "-e \"s#\\$(CFBundleIdentifier)#" + escapedBundleId + "#g\" " + + + "> " + out.getExecPathString()) + .addInput(in) + .addInput(prefix) + .addOutput(out) + .build(ruleContext)); + } + + + private String extractPlistCommand(Artifact provisioningProfile) { + return "security cms -D -i " + ShellUtils.shellEscape(provisioningProfile.getExecPathString()); + } + + private String codesignCommand( + Artifact provisioningProfile, Artifact entitlements, String appDir) { + String fingerprintCommand = + "/usr/libexec/PlistBuddy -c 'Print DeveloperCertificates:0' /dev/stdin <<< " + + "$(" + extractPlistCommand(provisioningProfile) + ") | " + + "openssl x509 -inform DER -noout -fingerprint | " + + "cut -d= -f2 | sed -e 's#:##g'"; + return String.format( + "/usr/bin/codesign --force --sign $(%s) --entitlements %s %s", + fingerprintCommand, + entitlements.getExecPathString(), + appDir); + } + + /** + * Logic to access attributes required by application support. Attributes are required and + * guaranteed to return a value or throw unless they are annotated with {@link Nullable} in which + * case they can return {@code null} if no value is defined. + */ + private static class Attributes { + private final RuleContext ruleContext; + + private Attributes(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + @Nullable + String appIcon() { + return stringAttribute("app_icon"); + } + + @Nullable + String launchImage() { + return stringAttribute("launch_image"); + } + + @Nullable + Artifact provisioningProfile() { + return ruleContext.getPrerequisiteArtifact("provisioning_profile", Mode.TARGET); + } + + /** + * Returns the value of the {@code families} attribute in a form that is more useful than a list + * of strings. Returns an empty set for any invalid {@code families} attribute value, including + * an empty list. + */ + Set<TargetDeviceFamily> families() { + List<String> rawFamilies = ruleContext.attributes().get("families", Type.STRING_LIST); + try { + return TargetDeviceFamily.fromNamesInRule(rawFamilies); + } catch (InvalidFamilyNameException | RepeatedFamilyNameException e) { + return ImmutableSet.of(); + } + } + + @Nullable + Artifact entitlements() { + return ruleContext.getPrerequisiteArtifact("entitlements", Mode.TARGET); + } + + NestedSet<? extends Artifact> dependentLinkedBinaries() { + if (ruleContext.attributes().getAttributeDefinition("binary") == null) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + return ruleContext.getPrerequisite("binary", Mode.TARGET, ObjcProvider.class) + .get(ObjcProvider.LINKED_BINARY); + } + + FilesToRunProvider bundleMergeExecutable() { + return checkNotNull(ruleContext.getExecutablePrerequisite("$bundlemerge", Mode.HOST)); + } + + String bundleId() { + return checkNotNull(stringAttribute("bundle_id")); + } + + @Nullable + private String stringAttribute(String attribute) { + String value = ruleContext.attributes().get(attribute, Type.STRING); + return value.isEmpty() ? null : value; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ArtifactListAttribute.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ArtifactListAttribute.java new file mode 100644 index 0000000..d6c993b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ArtifactListAttribute.java
@@ -0,0 +1,45 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; + +import java.util.Locale; + +/** + * Attributes containing one or more labels. + */ +public enum ArtifactListAttribute { + BUNDLE_IMPORTS; + + public String attrName() { + return name().toLowerCase(Locale.US); + } + + /** + * The artifacts specified by this attribute on the given rule. Returns an empty sequence if the + * attribute is omitted or not available on the rule type. + */ + public Iterable<Artifact> get(RuleContext context) { + if (context.attributes().getAttributeDefinition(attrName()) == null) { + return ImmutableList.of(); + } else { + return context.getPrerequisiteArtifacts(attrName(), Mode.TARGET).list(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/BundleMergeControlBytes.java b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleMergeControlBytes.java new file mode 100644 index 0000000..695dffc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleMergeControlBytes.java
@@ -0,0 +1,121 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.NESTED_BUNDLE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCDATAMODEL; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.io.ByteSource; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos; +import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.Control; +import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.MergeZip; +import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.VariableSubstitution; +import com.google.devtools.build.xcode.common.TargetDeviceFamily; + +import java.io.InputStream; +import java.util.Map; + +/** + * A byte source that can be used to generate a control file for the tool: + * {@code //java/com/google/devtools/build/xcode/bundlemerge}. Note that this generates the control + * proto and bytes on-the-fly rather than eagerly. This is to prevent a copy of the bundle files and + * .xcdatamodels from being stored for each {@code objc_binary} (or any bundle) being built. + */ +final class BundleMergeControlBytes extends ByteSource { + private final Bundling rootBundling; + private final Artifact mergedIpa; + private final ObjcConfiguration objcConfiguration; + private final ImmutableSet<TargetDeviceFamily> families; + + public BundleMergeControlBytes( + Bundling rootBundling, Artifact mergedIpa, ObjcConfiguration objcConfiguration, + ImmutableSet<TargetDeviceFamily> families) { + this.rootBundling = Preconditions.checkNotNull(rootBundling); + this.mergedIpa = Preconditions.checkNotNull(mergedIpa); + this.objcConfiguration = Preconditions.checkNotNull(objcConfiguration); + this.families = Preconditions.checkNotNull(families); + } + + @Override + public InputStream openStream() { + return control("Payload/", "Payload/", rootBundling) + .toByteString() + .newInput(); + } + + private Control control(String mergeZipPrefix, String bundleDirPrefix, Bundling bundling) { + ObjcProvider objcProvider = bundling.getObjcProvider(); + String bundleDir = bundleDirPrefix + bundling.getBundleDir(); + mergeZipPrefix += bundling.getBundleDir() + "/"; + + BundleMergeProtos.Control.Builder control = BundleMergeProtos.Control.newBuilder() + .addAllBundleFile(BundleableFile.toBundleFiles(bundling.getExtraBundleFiles())) + .addAllBundleFile(BundleableFile.toBundleFiles(objcProvider.get(BUNDLE_FILE))) + .addAllSourcePlistFile(Artifact.toExecPaths( + bundling.getInfoplistMerging().getPlistWithEverything().asSet())) + // TODO(bazel-team): Add rule attribute for specifying targeted device family + .setMinimumOsVersion(objcConfiguration.getMinimumOs()) + .setSdkVersion(objcConfiguration.getIosSdkVersion()) + .setPlatform(objcConfiguration.getPlatform().name()) + .setBundleRoot(bundleDir); + + for (Artifact mergeZip : bundling.getMergeZips()) { + control.addMergeZip(MergeZip.newBuilder() + .setEntryNamePrefix(mergeZipPrefix) + .setSourcePath(mergeZip.getExecPathString()) + .build()); + } + + for (Xcdatamodel datamodel : objcProvider.get(XCDATAMODEL)) { + control.addMergeZip(MergeZip.newBuilder() + .setEntryNamePrefix(mergeZipPrefix) + .setSourcePath(datamodel.getOutputZip().getExecPathString()) + .build()); + } + for (TargetDeviceFamily targetDeviceFamily : families) { + control.addTargetDeviceFamily(targetDeviceFamily.name()); + } + + Map<String, String> variableSubstitutions = bundling.variableSubstitutions(); + for (String variable : variableSubstitutions.keySet()) { + control.addVariableSubstitution(VariableSubstitution.newBuilder() + .setName(variable) + .setValue(variableSubstitutions.get(variable)) + .build()); + } + + control.setOutFile(mergedIpa.getExecPathString()); + + for (Artifact linkedBinary : bundling.getCombinedArchitectureBinary().asSet()) { + control + .addBundleFile(BundleMergeProtos.BundleFile.newBuilder() + .setSourceFile(linkedBinary.getExecPathString()) + .setBundlePath(bundling.getName()) + .setExternalFileAttribute(BundleableFile.EXECUTABLE_EXTERNAL_FILE_ATTRIBUTE) + .build()) + .setExecutableName(bundling.getName()); + } + + for (Bundling nestedBundling : bundling.getObjcProvider().get(NESTED_BUNDLE)) { + control.addNestedBundle(control(mergeZipPrefix, "", nestedBundling)); + } + + return control.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/BundleSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleSupport.java new file mode 100644 index 0000000..5434ff7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleSupport.java
@@ -0,0 +1,184 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraActoolArgs; +import com.google.devtools.build.lib.rules.objc.XcodeProvider.Builder; +import com.google.devtools.build.xcode.common.TargetDeviceFamily; + +import java.util.Set; + +/** + * Support for generating iOS bundles which contain metadata (a plist file), assets, resources and + * optionally a binary: registers actions that assemble resources and merge plists, provides data + * to providers and validates bundle-related attributes. + * + * <p>Methods on this class can be called in any order without impacting the result. + */ +final class BundleSupport { + + @VisibleForTesting + static final String NO_INFOPLIST_ERROR = "An infoplist must be specified either in the " + + "'infoplist' attribute or via the 'options' attribute, but none was found"; + + private final RuleContext ruleContext; + private final Set<TargetDeviceFamily> targetDeviceFamilies; + private final ExtraActoolArgs extraActoolArgs; + private final Bundling bundling; + + /** + * Returns merging instructions for a bundle's {@code Info.plist}. + * + * @param ruleContext context this bundle is constructed in + * @param objcProvider provider containing all dependencies' information as well as some of this + * rule's + * @param optionsProvider provider containing options and plist settings for this rule and its + * dependencies + */ + static InfoplistMerging infoPlistMerging(RuleContext ruleContext, + ObjcProvider objcProvider, OptionsProvider optionsProvider) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + + return new InfoplistMerging.Builder(ruleContext) + .setIntermediateArtifacts(intermediateArtifacts) + .setInputPlists(NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(optionsProvider.getInfoplists()) + .addAll(actoolPartialInfoplist(ruleContext, objcProvider).asSet()) + .build()) + .setPlmerge(ruleContext.getExecutablePrerequisite("$plmerge", Mode.HOST)) + .build(); + } + + /** + * Creates a new bundle support with no special {@code actool} arguments. + * + * @param ruleContext context this bundle is constructed in + * @param targetDeviceFamilies device families used in asset catalogue construction + * @param bundling bundle information as configured for this rule + */ + public BundleSupport( + RuleContext ruleContext, Set<TargetDeviceFamily> targetDeviceFamilies, Bundling bundling) { + this(ruleContext, targetDeviceFamilies, bundling, new ExtraActoolArgs()); + } + + /** + * Creates a new bundle support. + * + * @param ruleContext context this bundle is constructed in + * @param targetDeviceFamilies device families used in asset catalogue construction + * @param bundling bundle information as configured for this rule + * @param extraActoolArgs any additional parameters to be used for invoking {@code actool} + */ + public BundleSupport(RuleContext ruleContext, Set<TargetDeviceFamily> targetDeviceFamilies, + Bundling bundling, ExtraActoolArgs extraActoolArgs) { + this.ruleContext = ruleContext; + this.targetDeviceFamilies = targetDeviceFamilies; + this.extraActoolArgs = extraActoolArgs; + this.bundling = bundling; + } + + /** + * Registers actions required for constructing this bundle, namely merging all involved {@code + * Info.plist} files and generating asset catalogues. + * + * @param objcProvider source of information from this rule's attributes and its dependencies + * + * @return this bundle support + */ + BundleSupport registerActions(ObjcProvider objcProvider) { + registerMergeInfoplistAction(); + registerActoolActionIfNecessary(objcProvider); + + return this; + } + + /** + * Adds any Xcode settings related to this bundle to the given provider builder. + * + * @return this bundle support + */ + BundleSupport addXcodeSettings(Builder xcodeProviderBuilder) { + xcodeProviderBuilder.setInfoplistMerging(bundling.getInfoplistMerging()); + return this; + } + + /** + * Validates any rule attributes and dependencies related to this bundle. + * + * @return this bundle support + */ + BundleSupport validateAttributes() { + if (bundling.getInfoplistMerging().getInputPlists().isEmpty()) { + ruleContext.ruleError(NO_INFOPLIST_ERROR); + } + return this; + } + + private void registerMergeInfoplistAction() { + // TODO(bazel-team): Move action implementation from InfoplistMerging to this class. + ruleContext.registerAction(bundling.getInfoplistMerging().getMergeAction()); + } + + private void registerActoolActionIfNecessary(ObjcProvider objcProvider) { + Optional<Artifact> actoolzipOutput = bundling.getActoolzipOutput(); + if (!actoolzipOutput.isPresent()) { + return; + } + + ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext); + + Artifact actoolPartialInfoplist = actoolPartialInfoplist(ruleContext, objcProvider).get(); + actionsBuilder.registerActoolzipAction( + new ObjcRuleClasses.Tools(ruleContext), + objcProvider, + actoolzipOutput.get(), + new ObjcActionsBuilder.ExtraActoolOutputs(actoolPartialInfoplist), + new ExtraActoolArgs( + new ImmutableList.Builder<String>() + .addAll(extraActoolArgs) + .add("--output-partial-info-plist", actoolPartialInfoplist.getExecPathString()) + .build()), + targetDeviceFamilies); + } + + /** + * Returns the artifact that is a plist file generated by an invocation of {@code actool} or + * {@link Optional#absent()} if no asset catalogues are present in this target and its + * dependencies. + * + * <p>All invocations of {@code actool} generate this kind of plist file, which contains metadata + * about the {@code app_icon} and {@code launch_image} if supplied. If neither an app icon or a + * launch image was supplied, the plist file generated is empty. + */ + private static Optional<Artifact> actoolPartialInfoplist( + RuleContext ruleContext, ObjcProvider objcProvider) { + if (objcProvider.hasAssetCatalogs()) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + return Optional.of(intermediateArtifacts.actoolPartialInfoplist()); + } else { + return Optional.absent(); + } + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/BundleableFile.java b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleableFile.java new file mode 100644 index 0000000..eea7bf0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/BundleableFile.java
@@ -0,0 +1,149 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ArtifactListAttribute.BUNDLE_IMPORTS; +import static com.google.devtools.build.lib.rules.objc.ObjcCommon.BUNDLE_CONTAINER_TYPE; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.bundlemerge.proto.BundleMergeProtos.BundleFile; +import com.google.devtools.build.xcode.util.Value; + +/** + * Represents a file which is processed to another file and bundled. It contains the + * {@code Artifact} corresponding to the original file as well as the {@code Artifact} for the file + * converted to its bundled form. Examples of files that fit this pattern are .strings and .xib + * files. + */ +public final class BundleableFile extends Value<BundleableFile> { + static final int EXECUTABLE_EXTERNAL_FILE_ATTRIBUTE = 0100755 << 16; + static final int DEFAULT_EXTERNAL_FILE_ATTRIBUTE = 0100644 << 16; + + private final Artifact bundled; + private final String bundlePath; + private final int zipExternalFileAttribute; + + /** + * Creates an instance whose {@code zipExternalFileAttribute} value is + * {@link #DEFAULT_EXTERNAL_FILE_ATTRIBUTE}. + */ + BundleableFile(Artifact bundled, String bundlePath) { + this(bundled, bundlePath, DEFAULT_EXTERNAL_FILE_ATTRIBUTE); + } + + /** + * @param bundled the {@link Artifact} whose data is placed in the bundle + * @param bundlePath the path of the file in the bundle + * @param the external file attribute of the file in the central directory of the bundle (zip + * file). The lower 16 bits contain the MS-DOS file attributes. The upper 16 bits contain the + * Unix file attributes, for instance 0100755 (octal) for a regular file with permissions + * {@code rwxr-xr-x}. + */ + BundleableFile(Artifact bundled, String bundlePath, int zipExternalFileAttribute) { + super(new ImmutableMap.Builder<String, Object>() + .put("bundled", bundled) + .put("bundlePath", bundlePath) + .put("zipExternalFileAttribute", zipExternalFileAttribute) + .build()); + this.bundled = bundled; + this.bundlePath = bundlePath; + this.zipExternalFileAttribute = zipExternalFileAttribute; + } + + static String bundlePath(PathFragment path) { + String containingDir = path.getParentDirectory().getBaseName(); + return (containingDir.endsWith(".lproj") ? (containingDir + "/") : "") + path.getBaseName(); + } + + /** + * Given a sequence of non-compiled resource files, returns a sequence of the same length of + * instances of this class. Non-compiled resource files are resources which are not processed + * before placing them in the final bundle. This is different from (for example) {@code .strings} + * and {@code .xib} files, which must be converted to binary plist form or compiled. + * + * @param files a sequence of artifacts corresponding to non-compiled resource files + */ + public static Iterable<BundleableFile> nonCompiledResourceFiles(Iterable<Artifact> files) { + ImmutableList.Builder<BundleableFile> result = new ImmutableList.Builder<>(); + for (Artifact file : files) { + result.add(new BundleableFile(file, bundlePath(file.getExecPath()))); + } + return result.build(); + } + + /** + * Returns an instance for every file in a bundle directory. + * <p> + * This uses the parent-most container matching {@code *.bundle} as the bundle root. + * TODO(bazel-team): add something like an import_root attribute to specify this explicitly, which + * will be helpful if a bundle that appears to be nested needs to be imported alone. + */ + public static Iterable<BundleableFile> bundleImportsFromRule(RuleContext context) { + ImmutableList.Builder<BundleableFile> result = new ImmutableList.Builder<>(); + for (Artifact artifact : BUNDLE_IMPORTS.get(context)) { + for (PathFragment container : + ObjcCommon.farthestContainerMatching(BUNDLE_CONTAINER_TYPE, artifact).asSet()) { + // TODO(bazel-team): Figure out if we need to remove symbols of architectures we aren't + // building for from the binary in the bundle. + result.add(new BundleableFile( + artifact, + // The path from the artifact's container (including the container), to the artifact + // itself. For instance, if artifact is foo/bar.bundle/baz, then this value + // is bar.bundle/baz. + artifact.getExecPath().relativeTo(container.getParentDirectory()).getSafePathString())); + } + } + return result.build(); + } + + /** + * The artifact that is ultimately bundled. + */ + public Artifact getBundled() { + return bundled; + } + + /** + * Returns bundle files for each given strings file. These are used to merge the strings files to + * the final application bundle. + */ + public static Iterable<BundleFile> toBundleFiles(Iterable<BundleableFile> files) { + ImmutableList.Builder<BundleFile> result = new ImmutableList.Builder<>(); + for (BundleableFile file : files) { + result.add(BundleFile.newBuilder() + .setBundlePath(file.bundlePath) + .setSourceFile(file.bundled.getExecPathString()) + .setExternalFileAttribute(file.zipExternalFileAttribute) + .build()); + } + return result.build(); + } + + /** + * Returns the artifacts for the bundled files. These can be used, for instance, as the input + * files to add to the bundlemerge action for a bundle that contains all the given files. + */ + public static Iterable<Artifact> toArtifacts(Iterable<BundleableFile> files) { + ImmutableList.Builder<Artifact> result = new ImmutableList.Builder<>(); + for (BundleableFile file : files) { + result.add(file.bundled); + } + return result.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/Bundling.java b/src/main/java/com/google/devtools/build/lib/rules/objc/Bundling.java new file mode 100644 index 0000000..484c553 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/Bundling.java
@@ -0,0 +1,254 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.ASSET_CATALOG; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.MERGE_ZIP; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.NESTED_BUNDLE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCDATAMODEL; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.xcode.util.Value; + +import java.util.Map; + +/** + * Contains information regarding the creation of an iOS bundle. + */ +@Immutable +final class Bundling extends Value<Bundling> { + static final class Builder { + private String name; + private String bundleDirSuffix; + private ImmutableList<BundleableFile> extraBundleFiles = ImmutableList.of(); + private ObjcProvider objcProvider; + private InfoplistMerging infoplistMerging; + private IntermediateArtifacts intermediateArtifacts; + + public Builder setName(String name) { + this.name = name; + return this; + } + + public Builder setBundleDirSuffix(String bundleDirSuffix) { + this.bundleDirSuffix = bundleDirSuffix; + return this; + } + + public Builder setExtraBundleFiles(ImmutableList<BundleableFile> extraBundleFiles) { + this.extraBundleFiles = extraBundleFiles; + return this; + } + + public Builder setObjcProvider(ObjcProvider objcProvider) { + this.objcProvider = objcProvider; + return this; + } + + public Builder setInfoplistMerging(InfoplistMerging infoplistMerging) { + this.infoplistMerging = infoplistMerging; + return this; + } + + public Builder setIntermediateArtifacts(IntermediateArtifacts intermediateArtifacts) { + this.intermediateArtifacts = intermediateArtifacts; + return this; + } + + private static NestedSet<Artifact> nestedBundleContentArtifacts(Iterable<Bundling> bundles) { + NestedSetBuilder<Artifact> artifacts = NestedSetBuilder.stableOrder(); + for (Bundling bundle : bundles) { + artifacts.addTransitive(bundle.getBundleContentArtifacts()); + } + return artifacts.build(); + } + + public Bundling build() { + Preconditions.checkNotNull(intermediateArtifacts, "intermediateArtifacts"); + + Optional<Artifact> actoolzipOutput = Optional.absent(); + if (!Iterables.isEmpty(objcProvider.get(ASSET_CATALOG))) { + actoolzipOutput = Optional.of(intermediateArtifacts.actoolzipOutput()); + } + + Optional<Artifact> combinedArchitectureBinary = Optional.absent(); + if (!Iterables.isEmpty(objcProvider.get(LIBRARY)) + || !Iterables.isEmpty(objcProvider.get(IMPORTED_LIBRARY))) { + combinedArchitectureBinary = + Optional.of(intermediateArtifacts.combinedArchitectureBinary(bundleDirSuffix)); + } + + NestedSet<Artifact> mergeZips = NestedSetBuilder.<Artifact>stableOrder() + .addAll(actoolzipOutput.asSet()) + .addTransitive(objcProvider.get(MERGE_ZIP)) + .build(); + NestedSet<Artifact> bundleContentArtifacts = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(nestedBundleContentArtifacts(objcProvider.get(NESTED_BUNDLE))) + .addAll(combinedArchitectureBinary.asSet()) + .addAll(infoplistMerging.getPlistWithEverything().asSet()) + .addTransitive(mergeZips) + .addAll(BundleableFile.toArtifacts(extraBundleFiles)) + .addAll(BundleableFile.toArtifacts(objcProvider.get(BUNDLE_FILE))) + .addAll(Xcdatamodel.outputZips(objcProvider.get(XCDATAMODEL))) + .build(); + + return new Bundling(name, bundleDirSuffix, combinedArchitectureBinary, extraBundleFiles, + objcProvider, infoplistMerging, actoolzipOutput, bundleContentArtifacts, mergeZips); + } + } + + private final String name; + private final String bundleDirSuffix; + private final Optional<Artifact> combinedArchitectureBinary; + private final ImmutableList<BundleableFile> extraBundleFiles; + private final ObjcProvider objcProvider; + private final InfoplistMerging infoplistMerging; + private final Optional<Artifact> actoolzipOutput; + private final NestedSet<Artifact> bundleContentArtifacts; + private final NestedSet<Artifact> mergeZips; + + private Bundling( + String name, + String bundleDirSuffix, + Optional<Artifact> combinedArchitectureBinary, + ImmutableList<BundleableFile> extraBundleFiles, + ObjcProvider objcProvider, + InfoplistMerging infoplistMerging, + Optional<Artifact> actoolzipOutput, + NestedSet<Artifact> bundleContentArtifacts, + NestedSet<Artifact> mergeZips) { + super(new ImmutableMap.Builder<String, Object>() + .put("name", name) + .put("bundleDirSuffix", bundleDirSuffix) + .put("combinedArchitectureBinary", combinedArchitectureBinary) + .put("extraBundleFiles", extraBundleFiles) + .put("objcProvider", objcProvider) + .put("infoplistMerging", infoplistMerging) + .put("actoolzipOutput", actoolzipOutput) + .put("bundleContentArtifacts", bundleContentArtifacts) + .put("mergeZips", mergeZips) + .build()); + this.name = name; + this.bundleDirSuffix = bundleDirSuffix; + this.combinedArchitectureBinary = combinedArchitectureBinary; + this.extraBundleFiles = extraBundleFiles; + this.objcProvider = objcProvider; + this.infoplistMerging = infoplistMerging; + this.actoolzipOutput = actoolzipOutput; + this.bundleContentArtifacts = bundleContentArtifacts; + this.mergeZips = mergeZips; + } + + /** + * The bundle directory. For apps, {@code "Payload/" + bundleDir} is the directory in the bundle + * zip archive in which every file is found including the linked binary, nested bundles, and + * everything returned by {@link #getExtraBundleFiles()}. In an application bundle, for instance, + * this function returns {@code "(name).app"}. + */ + public String getBundleDir() { + return name + bundleDirSuffix; + } + + /** + * The name of the bundle, from which the bundle root and the path of the linked binary in the + * bundle archive are derived. + */ + public String getName() { + return name; + } + + /** + * An {@link Optional} with the linked binary artifact, or {@link Optional#absent()} if it is + * empty and should not be included in the bundle. + */ + public Optional<Artifact> getCombinedArchitectureBinary() { + return combinedArchitectureBinary; + } + + /** + * Extra bundle files to include in the bundle which are not automatically deduced by the contents + * of the provider. These files are placed under the bundle root (possibly nested, of course, + * depending on the bundle path of the files). + */ + public ImmutableList<BundleableFile> getExtraBundleFiles() { + return extraBundleFiles; + } + + /** + * The {@link ObjcProvider} for this bundle. + */ + public ObjcProvider getObjcProvider() { + return objcProvider; + } + + /** + * Information on the Info.plist and its merge inputs for this bundle. Note that an infoplist is + * only included in the bundle if it has one or more merge inputs. + */ + public InfoplistMerging getInfoplistMerging() { + return infoplistMerging; + } + + /** + * The location of the actoolzip output for this bundle. This is non-absent only included in the + * bundle if there is at least one asset catalog artifact supplied by + * {@link ObjcProvider#ASSET_CATALOG}. + */ + public Optional<Artifact> getActoolzipOutput() { + return actoolzipOutput; + } + + /** + * Returns all zip files whose contents should be merged into this bundle under the main bundle + * directory. For instance, if a merge zip contains files a/b and c/d, then the resulting bundling + * would have additional files at: + * <ul> + * <li>{bundleDir}/a/b + * <li>{bundleDir}/c/d + * </ul> + */ + public NestedSet<Artifact> getMergeZips() { + return mergeZips; + } + + /** + * Returns the variable substitutions that should be used when merging the plist info file of + * this bundle. + */ + public Map<String, String> variableSubstitutions() { + return ImmutableMap.of( + "EXECUTABLE_NAME", name, + "BUNDLE_NAME", name + bundleDirSuffix, + "PRODUCT_NAME", name); + } + + /** + * Returns the artifacts that are required to generate this bundle. + */ + public NestedSet<Artifact> getBundleContentArtifacts() { + return bundleContentArtifacts; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationArtifacts.java b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationArtifacts.java new file mode 100644 index 0000000..5aac139 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationArtifacts.java
@@ -0,0 +1,98 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; + +/** + * Artifacts related to compilation. Any rule containing compilable sources will create an instance + * of this class. + */ +final class CompilationArtifacts { + static class Builder { + private Iterable<Artifact> srcs = ImmutableList.of(); + private Iterable<Artifact> nonArcSrcs = ImmutableList.of(); + private Optional<Artifact> pchFile; + private IntermediateArtifacts intermediateArtifacts; + + Builder addSrcs(Iterable<Artifact> srcs) { + this.srcs = Iterables.concat(this.srcs, srcs); + return this; + } + + Builder addNonArcSrcs(Iterable<Artifact> nonArcSrcs) { + this.nonArcSrcs = Iterables.concat(this.nonArcSrcs, nonArcSrcs); + return this; + } + + Builder setPchFile(Optional<Artifact> pchFile) { + Preconditions.checkState(this.pchFile == null, + "pchFile is already set to: %s", this.pchFile); + this.pchFile = Preconditions.checkNotNull(pchFile); + return this; + } + + Builder setIntermediateArtifacts(IntermediateArtifacts intermediateArtifacts) { + Preconditions.checkState(this.intermediateArtifacts == null, + "intermediateArtifacts is already set to: %s", this.intermediateArtifacts); + this.intermediateArtifacts = intermediateArtifacts; + return this; + } + + CompilationArtifacts build() { + Optional<Artifact> archive = Optional.absent(); + if (!Iterables.isEmpty(srcs) || !Iterables.isEmpty(nonArcSrcs)) { + archive = Optional.of(intermediateArtifacts.archive()); + } + return new CompilationArtifacts(srcs, nonArcSrcs, archive, pchFile); + } + } + + private final Iterable<Artifact> srcs; + private final Iterable<Artifact> nonArcSrcs; + private final Optional<Artifact> archive; + private final Optional<Artifact> pchFile; + + private CompilationArtifacts( + Iterable<Artifact> srcs, + Iterable<Artifact> nonArcSrcs, + Optional<Artifact> archive, + Optional<Artifact> pchFile) { + this.srcs = Preconditions.checkNotNull(srcs); + this.nonArcSrcs = Preconditions.checkNotNull(nonArcSrcs); + this.archive = Preconditions.checkNotNull(archive); + this.pchFile = Preconditions.checkNotNull(pchFile); + } + + public Iterable<Artifact> getSrcs() { + return srcs; + } + + public Iterable<Artifact> getNonArcSrcs() { + return nonArcSrcs; + } + + public Optional<Artifact> getArchive() { + return archive; + } + + public Optional<Artifact> getPchFile() { + return pchFile; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java new file mode 100644 index 0000000..1e23798 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java
@@ -0,0 +1,235 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.NON_ARC_SRCS_TYPE; +import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.SRCS_TYPE; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkArgs; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkInputs; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.CompilationAttributes; +import com.google.devtools.build.lib.rules.objc.XcodeProvider.Builder; +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Support for rules that compile sources. Provides ways to determine files that should be output, + * registering Xcode settings and generating the various actions that might be needed for + * compilation. + * + * <p>Methods on this class can be called in any order without impacting the result. + */ +final class CompilationSupport { + + @VisibleForTesting + static final String ABSOLUTE_INCLUDES_PATH_FORMAT = + "The path '%s' is absolute, but only relative paths are allowed."; + + /** + * Returns information about the given rule's compilation artifacts. + */ + // TODO(bazel-team): Remove this information from ObjcCommon and move it internal to this class. + static CompilationArtifacts compilationArtifacts(RuleContext ruleContext) { + return new CompilationArtifacts.Builder() + .addSrcs(ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET) + .errorsForNonMatching(SRCS_TYPE) + .list()) + .addNonArcSrcs(ruleContext.getPrerequisiteArtifacts("non_arc_srcs", Mode.TARGET) + .errorsForNonMatching(NON_ARC_SRCS_TYPE) + .list()) + .setIntermediateArtifacts(ObjcRuleClasses.intermediateArtifacts(ruleContext)) + .setPchFile(Optional.fromNullable(ruleContext.getPrerequisiteArtifact("pch", Mode.TARGET))) + .build(); + } + + private final RuleContext ruleContext; + private final CompilationAttributes attributes; + + /** + * Creates a new compilation support for the given rule. + */ + CompilationSupport(RuleContext ruleContext) { + this.ruleContext = ruleContext; + this.attributes = new CompilationAttributes(ruleContext); + } + + /** + * Registers all actions necessary to compile this rule's sources and archive them. + * + * @param common common information about this rule and its dependencies + * @param optionsProvider option and plist information about this rule and its dependencies + * + * @return this compilation support + */ + CompilationSupport registerCompileAndArchiveActions( + ObjcCommon common, OptionsProvider optionsProvider) { + if (common.getCompilationArtifacts().isPresent()) { + ObjcRuleClasses.actionsBuilder(ruleContext).registerCompileAndArchiveActions( + common.getCompilationArtifacts().get(), common.getObjcProvider(), optionsProvider); + } + return this; + } + + /** + * Registers any actions necessary to link this rule and its dependencies. Debug symbols are + * generated if {@link ObjcConfiguration#generateDebugSymbols()} is set. + * + * @param objcProvider common information about this rule's attributes and its dependencies + * @param extraLinkArgs any additional arguments to pass to the linker + * @param extraLinkInputs any additional input artifacts to pass to the link action + * + * @return this compilation support + */ + CompilationSupport registerLinkActions(ObjcProvider objcProvider, ExtraLinkArgs extraLinkArgs, + ExtraLinkInputs extraLinkInputs) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + Optional<Artifact> dsymBundle; + if (ObjcRuleClasses.objcConfiguration(ruleContext).generateDebugSymbols()) { + registerDsymActions(); + dsymBundle = Optional.of(intermediateArtifacts.dsymBundle()); + } else { + dsymBundle = Optional.absent(); + } + + ObjcRuleClasses.actionsBuilder(ruleContext).registerLinkAction( + intermediateArtifacts.singleArchitectureBinary(), objcProvider, extraLinkArgs, + extraLinkInputs, dsymBundle); + return this; + } + + /** + * Registers actions that compile and archive j2Objc dependencies of this rule. + * + * @param optionsProvider option and plist information about this rule and its dependencies + * @param objcProvider common information about this rule's attributes and its dependencies + * + * @return this compilation support + */ + CompilationSupport registerJ2ObjcCompileAndArchiveActions( + OptionsProvider optionsProvider, ObjcProvider objcProvider) { + for (J2ObjcSource j2ObjcSource : ObjcRuleClasses.j2ObjcSrcsProvider(ruleContext).getSrcs()) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.j2objcIntermediateArtifacts(ruleContext, j2ObjcSource); + CompilationArtifacts compilationArtifact = new CompilationArtifacts.Builder() + .addNonArcSrcs(j2ObjcSource.getObjcSrcs()) + .setIntermediateArtifacts(intermediateArtifacts) + .setPchFile(Optional.<Artifact>absent()) + .build(); + ObjcActionsBuilder actionBuilder = new ObjcActionsBuilder( + ruleContext, + intermediateArtifacts, + ObjcRuleClasses.objcConfiguration(ruleContext), + ruleContext.getConfiguration(), + ruleContext); + actionBuilder + .registerCompileAndArchiveActions(compilationArtifact, objcProvider, optionsProvider); + } + + return this; + } + + /** + * Sets compilation-related Xcode project information on the given provider builder. + * + * @param common common information about this rule's attributes and its dependencies + * @param optionsProvider option and plist information about this rule and its dependencies + * @return this compilation support + */ + CompilationSupport addXcodeSettings(Builder xcodeProviderBuilder, + ObjcCommon common, OptionsProvider optionsProvider) { + ObjcConfiguration objcConfiguration = ObjcRuleClasses.objcConfiguration(ruleContext); + for (CompilationArtifacts artifacts : common.getCompilationArtifacts().asSet()) { + xcodeProviderBuilder.setCompilationArtifacts(artifacts); + } + xcodeProviderBuilder + .addHeaders(attributes.hdrs()) + .addUserHeaderSearchPaths(ObjcCommon.userHeaderSearchPaths(ruleContext.getConfiguration())) + .addHeaderSearchPaths("$(WORKSPACE_ROOT)", attributes.headerSearchPaths()) + .addHeaderSearchPaths("$(SDKROOT)/usr/include", attributes.sdkIncludes()) + .addCompilationModeCopts(objcConfiguration.getCoptsForCompilationMode()) + .addCopts(objcConfiguration.getCopts()) + .addCopts(optionsProvider.getCopts()); + return this; + } + + /** + * Validates compilation-related attributes on this rule. + * + * @return this compilation support + */ + CompilationSupport validateAttributes() { + for (PathFragment absoluteInclude : + Iterables.filter(attributes.includes(), PathFragment.IS_ABSOLUTE)) { + ruleContext.attributeError( + "includes", String.format(ABSOLUTE_INCLUDES_PATH_FORMAT, absoluteInclude)); + } + + return this; + } + + private CompilationSupport registerDsymActions() { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + Artifact dsymBundle = intermediateArtifacts.dsymBundle(); + Artifact debugSymbolFile = intermediateArtifacts.dsymSymbol(); + ruleContext.registerAction(new SpawnAction.Builder() + .setMnemonic("UnzipDsym") + .setProgressMessage("Unzipping dSYM file: " + ruleContext.getLabel()) + .setExecutable(new PathFragment("/usr/bin/unzip")) + .addInput(dsymBundle) + .setCommandLine(CustomCommandLine.builder() + .add(dsymBundle.getExecPathString()) + .add("-d") + .add(stripSuffix(dsymBundle.getExecPathString(), + IntermediateArtifacts.TMP_DSYM_BUNDLE_SUFFIX) + ".app.dSYM") + .build()) + .addOutput(intermediateArtifacts.dsymPlist()) + .addOutput(debugSymbolFile) + .build(ruleContext)); + + Artifact dumpsyms = ruleContext.getPrerequisiteArtifact("$dumpsyms", Mode.HOST); + Artifact breakpadFile = intermediateArtifacts.breakpadSym(); + ruleContext.registerAction(new SpawnAction.Builder() + .setMnemonic("GenBreakpad") + .setProgressMessage("Generating breakpad file: " + ruleContext.getLabel()) + .setShellCommand(ImmutableList.of("/bin/bash", "-c")) + .setExecutionInfo(ImmutableMap.of(ExecutionRequirements.REQUIRES_DARWIN, "")) + .addInput(dumpsyms) + .addInput(debugSymbolFile) + .addArgument(String.format("%s %s > %s", + ShellUtils.shellEscape(dumpsyms.getExecPathString()), + ShellUtils.shellEscape(debugSymbolFile.getExecPathString()), + ShellUtils.shellEscape(breakpadFile.getExecPathString()))) + .addOutput(breakpadFile) + .build(ruleContext)); + return this; + } + + private String stripSuffix(String str, String suffix) { + // TODO(bazel-team): Throw instead of returning null? + return str.endsWith(suffix) ? str.substring(0, str.length() - suffix.length()) : null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/CompiledResourceFile.java b/src/main/java/com/google/devtools/build/lib/rules/objc/CompiledResourceFile.java new file mode 100644 index 0000000..cd84710 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/CompiledResourceFile.java
@@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; + +/** + * Represents a strings file. + */ +public class CompiledResourceFile { + private final Artifact original; + private final BundleableFile bundled; + + private CompiledResourceFile(Artifact original, BundleableFile bundled) { + this.original = Preconditions.checkNotNull(original); + this.bundled = Preconditions.checkNotNull(bundled); + } + + /** + * The checked-in version of the bundled file. + */ + public Artifact getOriginal() { + return original; + } + + public BundleableFile getBundled() { + return bundled; + } + + public static final Function<CompiledResourceFile, BundleableFile> TO_BUNDLED = + new Function<CompiledResourceFile, BundleableFile>() { + @Override + public BundleableFile apply(CompiledResourceFile input) { + return input.bundled; + } + }; + + /** + * Given a sequence of artifacts corresponding to {@code .strings} files, returns a sequence of + * the same length of instances of this class. The value returned by {@link #getBundled()} of each + * instance will be the plist file in binary form. + */ + public static Iterable<CompiledResourceFile> fromStringsFiles( + IntermediateArtifacts intermediateArtifacts, Iterable<Artifact> strings) { + ImmutableList.Builder<CompiledResourceFile> result = new ImmutableList.Builder<>(); + for (Artifact originalFile : strings) { + Artifact binaryFile = intermediateArtifacts.convertedStringsFile(originalFile); + result.add(new CompiledResourceFile( + originalFile, + new BundleableFile(binaryFile, BundleableFile.bundlePath(originalFile.getExecPath())))); + } + return result.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ExecutionRequirements.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ExecutionRequirements.java new file mode 100644 index 0000000..2257136 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ExecutionRequirements.java
@@ -0,0 +1,23 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +/** + * Strings used to express requirements on action execution environments. + */ +public class ExecutionRequirements { + /** If an action would not successfully run other than on Darwin. */ + public static final String REQUIRES_DARWIN = "requires-darwin"; +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/InfoplistMerging.java b/src/main/java/com/google/devtools/build/lib/rules/objc/InfoplistMerging.java new file mode 100644 index 0000000..58369ad --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/InfoplistMerging.java
@@ -0,0 +1,142 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.xcode.util.Interspersing; + +/** + * Supplies information regarding Infoplist merging for a particular binary. This includes: + * <ul> + * <li>the Info.plist which contains the fields from every source. If there is only one source + * plist, this is that plist. + * <li>the action to merge all the Infoplists into a single one. This is present even if there is + * only one Infoplist, to prevent a Bazel error when an Artifact does not have a generating + * action. + * </ul> + */ +class InfoplistMerging { + static class Builder { + private final ActionConstructionContext context; + private NestedSet<Artifact> inputPlists; + private FilesToRunProvider plmerge; + private IntermediateArtifacts intermediateArtifacts; + + public Builder(ActionConstructionContext context) { + this.context = Preconditions.checkNotNull(context); + } + + public Builder setInputPlists(NestedSet<Artifact> inputPlists) { + Preconditions.checkState(this.inputPlists == null); + this.inputPlists = inputPlists; + return this; + } + + public Builder setPlmerge(FilesToRunProvider plmerge) { + Preconditions.checkState(this.plmerge == null); + this.plmerge = plmerge; + return this; + } + + public Builder setIntermediateArtifacts(IntermediateArtifacts intermediateArtifacts) { + this.intermediateArtifacts = intermediateArtifacts; + return this; + } + + /** + * This static factory method prevents retention of the outer {@link Builder} class reference by + * the anonymous {@link CommandLine} instance. + */ + private static CommandLine mergeCommandLine( + final NestedSet<Artifact> inputPlists, final Artifact mergedInfoplist) { + return new CommandLine() { + @Override + public Iterable<String> arguments() { + return new ImmutableList.Builder<String>() + .addAll(Interspersing.beforeEach( + "--source_file", Artifact.toExecPaths(inputPlists))) + .add("--out_file", mergedInfoplist.getExecPathString()) + .build(); + } + }; + } + + public InfoplistMerging build() { + Preconditions.checkNotNull(intermediateArtifacts, "intermediateArtifacts"); + + Optional<Artifact> plistWithEverything = Optional.absent(); + Action[] mergeActions = new Action[0]; + + int inputs = Iterables.size(inputPlists); + if (inputs == 1) { + plistWithEverything = Optional.of(Iterables.getOnlyElement(inputPlists)); + } else if (inputs > 1) { + Artifact merged = intermediateArtifacts.mergedInfoplist(); + + plistWithEverything = Optional.of(merged); + mergeActions = new SpawnAction.Builder() + .setMnemonic("MergeInfoPlistFiles") + .setExecutable(plmerge) + .setCommandLine(mergeCommandLine(inputPlists, merged)) + .addTransitiveInputs(inputPlists) + .addOutput(merged) + .build(context); + } + + return new InfoplistMerging(plistWithEverything, mergeActions, inputPlists); + } + } + + private final Optional<Artifact> plistWithEverything; + private final Action[] mergeActions; + private final NestedSet<Artifact> inputPlists; + + private InfoplistMerging(Optional<Artifact> plistWithEverything, Action[] mergeActions, + NestedSet<Artifact> inputPlists) { + this.plistWithEverything = plistWithEverything; + this.mergeActions = mergeActions; + this.inputPlists = inputPlists; + } + + /** + * Creates action to merge multiple Info.plist files of a binary into a single Info.plist. No + * action is necessary if there is only one source. + */ + public Action[] getMergeAction() { + return mergeActions; + } + + /** + * An {@link Optional} with the merged infoplist, or {@link Optional#absent()} if there are no + * merge inputs and it should not be included in the bundle. + */ + public Optional<Artifact> getPlistWithEverything() { + return plistWithEverything; + } + + public NestedSet<Artifact> getInputPlists() { + return inputPlists; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IntermediateArtifacts.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IntermediateArtifacts.java new file mode 100644 index 0000000..b84bf03 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IntermediateArtifacts.java
@@ -0,0 +1,220 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Factory class for generating artifacts which are used as intermediate output. + */ +// TODO(bazel-team): This should really be named DerivedArtifacts as it contains methods for +// final as well as intermediate artifacts. +final class IntermediateArtifacts { + + /** + * Extension used on the temporary dsym bundle location. Must end in {@code .dSYM} for dsymutil + * to generate a plist file. + */ + static final String TMP_DSYM_BUNDLE_SUFFIX = ".temp.app.dSYM"; + + private final AnalysisEnvironment analysisEnvironment; + private final Root binDirectory; + private final Label ownerLabel; + private final String archiveFileNameSuffix; + + IntermediateArtifacts( + AnalysisEnvironment analysisEnvironment, Root binDirectory, Label ownerLabel, + String archiveFileNameSuffix) { + this.analysisEnvironment = Preconditions.checkNotNull(analysisEnvironment); + this.binDirectory = Preconditions.checkNotNull(binDirectory); + this.ownerLabel = Preconditions.checkNotNull(ownerLabel); + this.archiveFileNameSuffix = Preconditions.checkNotNull(archiveFileNameSuffix); + } + + /** + * Returns a derived artifact in the bin directory obtained by appending some extension to the end + * of the given {@link PathFragment}. + */ + private Artifact appendExtension(PathFragment original, String extension) { + return analysisEnvironment.getDerivedArtifact( + FileSystemUtils.appendExtension(original, extension), binDirectory); + } + + /** + * Returns a derived artifact in the bin directory obtained by appending some extension to the end + * of the {@link PathFragment} corresponding to the owner {@link Label}. + */ + private Artifact appendExtension(String extension) { + return appendExtension(ownerLabel.toPathFragment(), extension); + } + + /** + * The output of using {@code actooloribtoolzip} to run {@code actool} for a given bundle which is + * merged under the {@code .app} or {@code .bundle} directory root. + */ + public Artifact actoolzipOutput() { + return appendExtension(".actool.zip"); + } + + /** + * Output of the partial infoplist generated by {@code actool} when given the + * {@code --output-partial-info-plist [path]} flag. + */ + public Artifact actoolPartialInfoplist() { + return appendExtension(".actool-PartialInfo.plist"); + } + + /** + * The Info.plist file for a bundle which is comprised of more than one originating plist file. + * This is not needed for a bundle which has no source Info.plist files, or only one Info.plist + * file, since no merging occurs in that case. + */ + public Artifact mergedInfoplist() { + return appendExtension("-MergedInfo.plist"); + } + + /** + * The .objlist file, which contains a list of paths of object files to archive and is read by + * libtool in the archive action. + */ + public Artifact objList() { + return appendExtension(".objlist"); + } + + /** + * The artifact which is the binary (or library) which is comprised of one or more .a files linked + * together. + */ + public Artifact singleArchitectureBinary() { + return appendExtension("_bin"); + } + + /** + * Lipo binary generated by combining one or more linked binaries. This binary is the one included + * in generated bundles and invoked as entry point to the application. + * + * @param bundleDirSuffix suffix of the bundle containing this binary + */ + public Artifact combinedArchitectureBinary(String bundleDirSuffix) { + String baseName = ownerLabel.toPathFragment().getBaseName(); + return appendExtension(bundleDirSuffix + "/" + baseName); + } + + /** + * The {@code .a} file which contains all the compiled sources for a rule. + */ + public Artifact archive() { + PathFragment labelPath = ownerLabel.toPathFragment(); + PathFragment rootRelative = labelPath + .getParentDirectory() + .getRelative(String.format("lib%s%s.a", labelPath.getBaseName(), archiveFileNameSuffix)); + return analysisEnvironment.getDerivedArtifact(rootRelative, binDirectory); + } + + /** + * The debug symbol bundle file which contains debug symbols generated by dsymutil. + */ + public Artifact dsymBundle() { + return appendExtension(TMP_DSYM_BUNDLE_SUFFIX); + } + + /** + * The artifact for the .o file that should be generated when compiling the {@code source} + * artifact. + */ + public Artifact objFile(Artifact source) { + return analysisEnvironment.getDerivedArtifact( + FileSystemUtils.replaceExtension( + AnalysisUtils.getUniqueDirectory(ownerLabel, new PathFragment("_objs")) + .getRelative(source.getRootRelativePath()), + ".o"), + binDirectory); + } + + /** + * Returns the artifact corresponding to the pbxproj control file, which specifies the information + * required to generate the Xcode project file. + */ + public Artifact pbxprojControlArtifact() { + return appendExtension(".xcodeproj-control"); + } + + /** + * The artifact which contains the zipped-up results of compiling the storyboard. This is merged + * into the final bundle under the {@code .app} or {@code .bundle} directory root. + */ + public Artifact compiledStoryboardZip(Artifact input) { + return appendExtension("/" + BundleableFile.bundlePath(input.getExecPath()) + ".zip"); + } + + /** + * Returns the artifact which is the output of building an entire xcdatamodel[d] made of artifacts + * specified by a single rule. + * + * @param containerDir the containing *.xcdatamodeld or *.xcdatamodel directory + * @return the artifact for the zipped up compilation results. + */ + public Artifact compiledMomZipArtifact(PathFragment containerDir) { + return appendExtension( + "/" + FileSystemUtils.replaceExtension(containerDir, ".zip").getBaseName()); + } + + /** + * Returns the compiled (i.e. converted to binary plist format) artifact corresponding to the + * given {@code .strings} file. + */ + public Artifact convertedStringsFile(Artifact originalFile) { + return appendExtension(originalFile.getExecPath(), ".binary"); + } + + /** + * Returns the artifact corresponding to the zipped-up compiled form of the given {@code .xib} + * file. + */ + public Artifact compiledXibFileZip(Artifact originalFile) { + return analysisEnvironment.getDerivedArtifact( + FileSystemUtils.replaceExtension(originalFile.getExecPath(), ".nib.zip"), + binDirectory); + } + + /** + * Debug symbol plist generated for a linked binary. + */ + public Artifact dsymPlist() { + return appendExtension(".app.dSYM/Contents/Info.plist"); + } + + /** + * Debug symbol file generated for a linked binary. + */ + public Artifact dsymSymbol() { + return appendExtension( + String.format(".app.dSYM/Contents/Resources/DWARF/%s_bin", ownerLabel.getName())); + } + + /** + * Breakpad debug symbol representation. + */ + public Artifact breakpadSym() { + return appendExtension(".breakpad"); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosApplicationRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosApplicationRule.java new file mode 100644 index 0000000..b81d9b8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosApplicationRule.java
@@ -0,0 +1,117 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.STRING; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.xcode.common.TargetDeviceFamily; + +/** + * Rule definition for ios_application. + */ +@BlazeRule(name = "$ios_application", + ancestors = { BaseRuleClasses.BaseRule.class, + ObjcRuleClasses.ObjcBaseResourcesRule.class, + ObjcRuleClasses.ObjcHasInfoplistRule.class, + ObjcRuleClasses.ObjcHasEntitlementsRule.class }, + type = RuleClassType.ABSTRACT) // TODO(bazel-team): Add factory once this becomes a real rule. +public class IosApplicationRule implements RuleDefinition { + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(app_icon) --> + The name of the application icon, which should be in one of the asset + catalogs of this target or a (transitive) dependency. In a new project, + this is initialized to "AppIcon" by Xcode. + <p> + If the application icon is not in an asset catalog, do not use this + attribute. Instead, add a CFBundleIcons entry to the Info.plist file. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("app_icon", STRING)) + /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(launch_image) --> + The name of the launch image, which should be in one of the asset + catalogs of this target or a (transitive) dependency. In a new project, + this is initialized to "LaunchImage" by Xcode. + <p> + If the launch image is not in an asset catalog, do not use this + attribute. Instead, add an appropriately-named image resource to the + bundle. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("launch_image", STRING)) + /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(bundle_id) --> + The bundle ID (reverse-DNS path followed by app name) of the binary. If none is specified, a + junk value will be used. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("bundle_id", STRING) + .value(new Attribute.ComputedDefault() { + @Override + public Object getDefault(AttributeMap rule) { + // For tests and similar, we don't want to force people to explicitly specify + // throw-away data. + return "example." + rule.getName(); + } + })) + /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(families) --> + The device families to which this binary is targeted. This is known as + the <code>TARGETED_DEVICE_FAMILY</code> build setting in Xcode project + files. It is a list of one or more of the strings <code>"iphone"</code> + and <code>"ipad"</code>. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("families", STRING_LIST) + .value(ImmutableList.of(TargetDeviceFamily.IPHONE.getNameInRule()))) + /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(provisioning_profile) --> + The provisioning profile (.mobileprovision file) to use when bundling + the application. + <p> + This is only used for non-simulator builds. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("provisioning_profile", LABEL) + .value(env.getLabel("//tools/objc:default_provisioning_profile")) + .allowedFileTypes(FileType.of(".mobileprovision"))) + // TODO(bazel-team): Consider ways to trim dependencies so that changes to deps of these + // tools don't trigger all objc_* targets. Right now we check-in deploy jars, but we + // need a less painful and error-prone way. + /* <!-- #BLAZE_RULE($ios_application).ATTRIBUTE(binary) --> + The binary target included in the final bundle. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("binary", LABEL) + .allowedRuleClasses("objc_binary") + .allowedFileTypes() + .mandatory() + .direct_compile_time_input()) + .add(attr("$bundlemerge", LABEL).cfg(HOST).exec() + .value(env.getLabel("//tools/objc:bundlemerge"))) + .add(attr("$dumpsyms", LABEL).cfg(HOST).exec() + .value(env.getLabel("//tools/objc:dump_syms"))) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosDevice.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDevice.java new file mode 100644 index 0000000..2f01633 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDevice.java
@@ -0,0 +1,42 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * Implementation for the "ios_device" rule. + */ +public final class IosDevice implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext context) throws InterruptedException { + IosDeviceProvider provider = new IosDeviceProvider.Builder() + .setType(context.attributes().get("type", STRING)) + .setIosVersion(context.attributes().get("ios_version", STRING)) + .setLocale(context.attributes().get("locale", STRING)) + .build(); + + return new RuleConfiguredTargetBuilder(context) + .add(RunfilesProvider.class, RunfilesProvider.EMPTY) + .add(IosDeviceProvider.class, provider) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceProvider.java new file mode 100644 index 0000000..6f01e11 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceProvider.java
@@ -0,0 +1,73 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Provider that describes a simulator device. + */ +@Immutable +public final class IosDeviceProvider implements TransitiveInfoProvider { + /** A builder of {@link IosDeviceProvider}s. */ + public static final class Builder { + private String type; + private String iosVersion; + private String locale; + + public Builder setType(String type) { + this.type = type; + return this; + } + + public Builder setIosVersion(String iosVersion) { + this.iosVersion = iosVersion; + return this; + } + + public Builder setLocale(String locale) { + this.locale = locale; + return this; + } + + public IosDeviceProvider build() { + return new IosDeviceProvider(this); + } + } + + private final String type; + private final String iosVersion; + private final String locale; + + private IosDeviceProvider(Builder builder) { + this.type = Preconditions.checkNotNull(builder.type); + this.iosVersion = Preconditions.checkNotNull(builder.iosVersion); + this.locale = Preconditions.checkNotNull(builder.locale); + } + + public String getType() { + return type; + } + + public String getIosVersion() { + return iosVersion; + } + + public String getLocale() { + return locale; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceRule.java new file mode 100644 index 0000000..e028e70 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosDeviceRule.java
@@ -0,0 +1,67 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for ios_device. + */ +@BlazeRule(name = "ios_device", + factoryClass = IosDevice.class, + ancestors = { BaseRuleClasses.RuleBase.class }) +public final class IosDeviceRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE(ios_device).ATTRIBUTE(ios_version) --> + The operating system version of the device. This corresponds to the + <code>simctl</code> runtime. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("ios_version", STRING) + .mandatory()) + /* <!-- #BLAZE_RULE(ios_device).ATTRIBUTE(type) --> + The hardware type. This corresponds to the <code>simctl</code> device + type. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("type", STRING) + .mandatory()) + .add(attr("locale", STRING) + .undocumented("this is not yet supported by any test runner") + .value("en")) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = ios_device, TYPE = BINARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule defines an iOS device profile that defines a simulator against +which to run tests</p>. + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosSdkCommands.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosSdkCommands.java new file mode 100644 index 0000000..0a9fb7b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosSdkCommands.java
@@ -0,0 +1,155 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_DIR; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.common.Platform; +import com.google.devtools.build.xcode.util.Interspersing; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.XcodeprojBuildSetting; + +import java.util.List; + +/** + * Utility code for use when generating iOS SDK commands. + */ +public class IosSdkCommands { + public static final String DEVELOPER_DIR = "/Applications/Xcode.app/Contents/Developer"; + public static final String BIN_DIR = + DEVELOPER_DIR + "/Toolchains/XcodeDefault.xctoolchain/usr/bin"; + public static final String ACTOOL_PATH = DEVELOPER_DIR + "/usr/bin/actool"; + public static final String IBTOOL_PATH = DEVELOPER_DIR + "/usr/bin/ibtool"; + public static final String MOMC_PATH = DEVELOPER_DIR + "/usr/bin/momc"; + + // There is a handy reference to many clang warning flags at + // http://nshipster.com/clang-diagnostics/ + // There is also a useful narrative for many Xcode settings at + // http://www.xs-labs.com/en/blog/2011/02/04/xcode-build-settings/ + @VisibleForTesting + static final ImmutableMap<String, String> DEFAULT_WARNINGS = + new ImmutableMap.Builder<String, String>() + .put("GCC_WARN_64_TO_32_BIT_CONVERSION", "-Wshorten-64-to-32") + .put("CLANG_WARN_BOOL_CONVERSION", "-Wbool-conversion") + .put("CLANG_WARN_CONSTANT_CONVERSION", "-Wconstant-conversion") + // Double-underscores are intentional - thanks Xcode. + .put("CLANG_WARN__DUPLICATE_METHOD_MATCH", "-Wduplicate-method-match") + .put("CLANG_WARN_EMPTY_BODY", "-Wempty-body") + .put("CLANG_WARN_ENUM_CONVERSION", "-Wenum-conversion") + .put("CLANG_WARN_INT_CONVERSION", "-Wint-conversion") + .put("CLANG_WARN_UNREACHABLE_CODE", "-Wunreachable-code") + .put("GCC_WARN_ABOUT_RETURN_TYPE", "-Wmismatched-return-types") + .put("GCC_WARN_UNDECLARED_SELECTOR", "-Wundeclared-selector") + .put("GCC_WARN_UNINITIALIZED_AUTOS", "-Wuninitialized") + .put("GCC_WARN_UNUSED_FUNCTION", "-Wunused-function") + .put("GCC_WARN_UNUSED_VARIABLE", "-Wunused-variable") + .build(); + + static final ImmutableList<String> DEFAULT_LINKER_FLAGS = ImmutableList.of("-ObjC"); + + private IosSdkCommands() { + throw new UnsupportedOperationException("static-only"); + } + + private static String platformDir(ObjcConfiguration configuration) { + return DEVELOPER_DIR + "/Platforms/" + configuration.getPlatform().getNameInPlist() + + ".platform"; + } + + public static String sdkDir(ObjcConfiguration configuration) { + return platformDir(configuration) + "/Developer/SDKs/" + + configuration.getPlatform().getNameInPlist() + configuration.getIosSdkVersion() + ".sdk"; + } + + public static String frameworkDir(ObjcConfiguration configuration) { + return platformDir(configuration) + "/Developer/Library/Frameworks"; + } + + private static Iterable<PathFragment> uniqueParentDirectories(Iterable<PathFragment> paths) { + ImmutableSet.Builder<PathFragment> parents = new ImmutableSet.Builder<>(); + for (PathFragment path : paths) { + parents.add(path.getParentDirectory()); + } + return parents.build(); + } + + public static List<String> commonLinkAndCompileArgsForClang( + ObjcProvider provider, ObjcConfiguration configuration) { + ImmutableList.Builder<String> builder = new ImmutableList.Builder<>(); + if (configuration.getPlatform() == Platform.SIMULATOR) { + builder.add("-mios-simulator-version-min=" + configuration.getMinimumOs()); + } else { + builder.add("-miphoneos-version-min=" + configuration.getMinimumOs()); + } + + if (configuration.generateDebugSymbols()) { + builder.add("-g"); + } + + return builder + .add("-arch", configuration.getIosCpu()) + .add("-isysroot", sdkDir(configuration)) + // TODO(bazel-team): Pass framework search paths to Xcodegen. + .add("-F", sdkDir(configuration) + "/Developer/Library/Frameworks") + // As of sdk8.1, XCTest is in a base Framework dir + .add("-F", frameworkDir(configuration)) + // Add custom (non-SDK) framework search paths. For each framework foo/bar.framework, + // include "foo" as a search path. + .addAll(Interspersing.beforeEach( + "-F", + PathFragment.safePathStrings(uniqueParentDirectories(provider.get(FRAMEWORK_DIR))))) + .build(); + } + + public static Iterable<String> compileArgsForClang(ObjcConfiguration configuration) { + return Iterables.concat( + DEFAULT_WARNINGS.values(), + platformSpecificCompileArgsForClang(configuration) + ); + } + + private static List<String> platformSpecificCompileArgsForClang(ObjcConfiguration configuration) { + switch (configuration.getPlatform()) { + case DEVICE: + return ImmutableList.of(); + case SIMULATOR: + // These are added by Xcode when building, because the simulator is built on OSX + // frameworks so we aim compile to match the OSX objc runtime. + return ImmutableList.of( + "-fexceptions", + "-fasm-blocks", + "-fobjc-abi-version=2", + "-fobjc-legacy-dispatch"); + default: + throw new AssertionError(); + } + } + + public static Iterable<? extends XcodeprojBuildSetting> defaultWarningsForXcode() { + return Iterables.transform(DEFAULT_WARNINGS.keySet(), + new Function<String, XcodeprojBuildSetting>() { + @Override + public XcodeprojBuildSetting apply(String key) { + return XcodeprojBuildSetting.newBuilder().setName(key).setValue("YES").build(); + } + }); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IosTest.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IosTest.java new file mode 100644 index 0000000..9e9ddc4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IosTest.java
@@ -0,0 +1,163 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.objc.ApplicationSupport.LinkedBinary; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkArgs; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkInputs; + +import java.util.ArrayList; +import java.util.List; + +/** + * Contains information needed to create a {@link RuleConfiguredTarget} and invoke test runners + * for some instantiation of this rule. + */ +// TODO(bazel-team): Extract a TestSupport class that takes on most of the logic in this class. +public abstract class IosTest implements RuleConfiguredTargetFactory { + private static final ImmutableList<SdkFramework> AUTOMATIC_SDK_FRAMEWORKS_FOR_XCTEST = + ImmutableList.of(new SdkFramework("XCTest")); + + public static final String TARGET_DEVICE = "target_device"; + public static final String IS_XCTEST = "xctest"; + public static final String XCTEST_APP = "xctest_app"; + + @VisibleForTesting + public static final String REQUIRES_SOURCE_ERROR = + "ios_test requires at least one source file in srcs or non_arc_srcs"; + + /** + * Creates a target, including registering actions, just as {@link #create(RuleContext)} does. + * The difference between {@link #create(RuleContext)} and this method is that this method does + * only what is needed to support tests on the environment besides generate the Xcodeproj file + * and build the app and test {@code .ipa}s. The {@link #create(RuleContext)} method delegates + * to this method. + */ + protected abstract ConfiguredTarget create(RuleContext ruleContext, ObjcCommon common, + XcodeProvider xcodeProvider, NestedSet<Artifact> filesToBuild) throws InterruptedException; + + @Override + public final ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + ObjcCommon common = common(ruleContext); + OptionsProvider optionsProvider = optionsProvider(ruleContext); + + if (!common.getCompilationArtifacts().get().getArchive().isPresent()) { + ruleContext.ruleError(REQUIRES_SOURCE_ERROR); + } + + XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder(); + NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(common.getStoryboards().getOutputZips()) + .addAll(Xcdatamodel.outputZips(common.getDatamodels())); + + XcodeProductType productType; + ExtraLinkArgs extraLinkArgs; + ExtraLinkInputs extraLinkInputs; + if (!isXcTest(ruleContext)) { + productType = XcodeProductType.APPLICATION; + extraLinkArgs = new ExtraLinkArgs(); + extraLinkInputs = new ExtraLinkInputs(); + } else { + productType = XcodeProductType.UNIT_TEST; + XcodeProvider appIpaXcodeProvider = + ruleContext.getPrerequisite(XCTEST_APP, Mode.TARGET, XcodeProvider.class); + xcodeProviderBuilder + .setTestHost(appIpaXcodeProvider) + .setProductType(productType); + + Artifact bundleLoader = xcTestAppProvider(ruleContext).getBundleLoader(); + + // -bundle causes this binary to be linked as a bundle and not require an entry point + // (i.e. main()) + // -bundle_loader causes the code in this test to have access to the symbols in the test rig, + // or more specifically, the flag causes ld to consider the given binary when checking for + // missing symbols. + extraLinkArgs = new ExtraLinkArgs( + "-bundle", + "-bundle_loader", bundleLoader.getExecPathString()); + extraLinkInputs = new ExtraLinkInputs(bundleLoader); + } + + new CompilationSupport(ruleContext) + .registerLinkActions( + common.getObjcProvider(), extraLinkArgs, extraLinkInputs) + .registerJ2ObjcCompileAndArchiveActions(optionsProvider, common.getObjcProvider()) + .registerCompileAndArchiveActions(common, optionsProvider) + .addXcodeSettings(xcodeProviderBuilder, common, optionsProvider) + .validateAttributes(); + + new ApplicationSupport( + ruleContext, common.getObjcProvider(), optionsProvider, LinkedBinary.LOCAL_AND_DEPENDENCIES) + .registerActions() + .addXcodeSettings(xcodeProviderBuilder) + .addFilesToBuild(filesToBuild) + .validateAttributes(); + + new ResourceSupport(ruleContext) + .registerActions(common.getStoryboards()) + .validateAttributes() + .addXcodeSettings(xcodeProviderBuilder); + + new XcodeSupport(ruleContext) + .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), productType) + .addDependencies(xcodeProviderBuilder) + .addFilesToBuild(filesToBuild) + .registerActions(xcodeProviderBuilder.build()); + + return create(ruleContext, common, xcodeProviderBuilder.build(), filesToBuild.build()); + } + + protected static boolean isXcTest(RuleContext ruleContext) { + return ruleContext.attributes().get(IS_XCTEST, Type.BOOLEAN); + } + + private OptionsProvider optionsProvider(RuleContext ruleContext) { + return new OptionsProvider.Builder() + .addCopts(ruleContext.getTokenizedStringListAttr("copts")) + .addInfoplists(ruleContext.getPrerequisiteArtifacts("infoplist", Mode.TARGET).list()) + .addTransitive(Optional.fromNullable( + ruleContext.getPrerequisite("options", Mode.TARGET, OptionsProvider.class))) + .build(); + } + + /** Returns the {@link XcTestAppProvider} of the {@code xctest_app} attribute. */ + private static XcTestAppProvider xcTestAppProvider(RuleContext ruleContext) { + return ruleContext.getPrerequisite(XCTEST_APP, Mode.TARGET, XcTestAppProvider.class); + } + + private ObjcCommon common(RuleContext ruleContext) { + ImmutableList<SdkFramework> extraSdkFrameworks = isXcTest(ruleContext) + ? AUTOMATIC_SDK_FRAMEWORKS_FOR_XCTEST : ImmutableList.<SdkFramework>of(); + List<ObjcProvider> extraDepObjcProviders = new ArrayList<>(); + if (isXcTest(ruleContext)) { + extraDepObjcProviders.add(xcTestAppProvider(ruleContext).getObjcProvider()); + } + + return ObjcLibrary.common(ruleContext, extraSdkFrameworks, /*alwayslink=*/false, + new ObjcLibrary.ExtraImportLibraries(), new ObjcLibrary.Defines(), extraDepObjcProviders); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/IterableWrapper.java b/src/main/java/com/google/devtools/build/lib/rules/objc/IterableWrapper.java new file mode 100644 index 0000000..4bd7b0c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/IterableWrapper.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.Iterator; + +/** + * Base class for tiny container types that encapsulate an iterable. + */ +abstract class IterableWrapper<E> implements Iterable<E> { + private final Iterable<E> contents; + + IterableWrapper(Iterable<E> contents) { + this.contents = Preconditions.checkNotNull(contents); + } + + IterableWrapper(E... contents) { + this.contents = ImmutableList.copyOf(contents); + } + + @Override + public Iterator<E> iterator() { + return contents.iterator(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcHeaderMappingFileProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcHeaderMappingFileProvider.java new file mode 100644 index 0000000..93ff980 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcHeaderMappingFileProvider.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * This provider is exported by java_library rules to supply ObjC header to Java type mapping files + * for J2ObjC translation. J2ObjC needs the mapping files to be able to output translated files with + * correct header import paths in the same directories of the Java source files. + */ +@Immutable +public final class J2ObjcHeaderMappingFileProvider implements TransitiveInfoProvider { + private final NestedSet<Artifact> mappingFiles; + + public J2ObjcHeaderMappingFileProvider(NestedSet<Artifact> mappingFiles) { + this.mappingFiles = mappingFiles; + } + + public NestedSet<Artifact> getMappingFiles() { + return mappingFiles; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSource.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSource.java new file mode 100644 index 0000000..81ba292 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSource.java
@@ -0,0 +1,124 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Objects; +import com.google.common.collect.Iterators; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * An object that captures information of ObjC files generated by J2ObjC in a single target. + */ +public class J2ObjcSource { + + /** + * Indicates the type of files from which the ObjC files included in {@link J2ObjcSource} are + * generated. + */ + public enum SourceType { + /** + * Indicates the original file type is java source file. + */ + JAVA, + + /** + * Indicates the original file type is proto file. + */ + PROTO; + } + + private final Label targetLabel; + private final Iterable<Artifact> objcSrcs; + private final Iterable<Artifact> objcHdrs; + private final PathFragment objcFilePath; + private final SourceType sourceType; + + /** + * Constructs a J2ObjcSource containing target information for j2objc transpilation. + * + * @param targetLabel the @{code Label} of the associated target. + * @param objcSrcs the {@code Iterable} containing objc source files generated by J2ObjC + * @param objcHdrs the {@code Iterable} containing objc header files generated by J2ObjC + * @param objcFilePath the {@code PathFragment} under which all the generated objc files are. It + * can be used as header search path for objc compilations. + * @param sourceType the type of files from which the ObjC files are generated. + */ + public J2ObjcSource(Label targetLabel, Iterable<Artifact> objcSrcs, + Iterable<Artifact> objcHdrs, PathFragment objcFilePath, SourceType sourceType) { + this.targetLabel = targetLabel; + this.objcSrcs = objcSrcs; + this.objcHdrs = objcHdrs; + this.objcFilePath = objcFilePath; + this.sourceType = sourceType; + } + + /** + * Returns the label of the associated target. + */ + public Label getTargetLabel() { + return targetLabel; + } + + /** + * Returns the objc source files generated by J2ObjC. + */ + public Iterable<Artifact> getObjcSrcs() { + return objcSrcs; + } + + /* + * Returns the objc header files generated by J2ObjC + */ + public Iterable<Artifact> getObjcHdrs() { + return objcHdrs; + } + + /** + * Returns the {@code PathFragment} which represents a directory where the generated ObjC files + * reside and which can also be used as header search path in ObjC compilation. + */ + public PathFragment getObjcFilePath() { + return objcFilePath; + } + + /** + * Returns the type of files from which the ObjC files inside this object are generated. + */ + public SourceType getSourceType() { + return sourceType; + } + + @Override + public final boolean equals(Object other) { + if (!(other instanceof J2ObjcSource)) { + return false; + } + + J2ObjcSource that = (J2ObjcSource) other; + return Objects.equal(this.targetLabel, that.targetLabel) + && Iterators.elementsEqual(this.objcSrcs.iterator(), that.objcSrcs.iterator()) + && Iterators.elementsEqual(this.objcHdrs.iterator(), that.objcHdrs.iterator()) + && Objects.equal(this.objcFilePath, that.objcFilePath) + && this.sourceType == that.sourceType; + } + + @Override + public int hashCode() { + return Objects.hashCode(targetLabel, objcSrcs, objcHdrs, objcFilePath, sourceType); + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSrcsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSrcsProvider.java new file mode 100644 index 0000000..1e577c5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSrcsProvider.java
@@ -0,0 +1,45 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * This provider is exported by java_library rules to supply J2ObjC-translated ObjC sources to + * objc_binary for compilation and linking. + */ +@Immutable +public final class J2ObjcSrcsProvider implements TransitiveInfoProvider { + private final NestedSet<J2ObjcSource> srcs; + private final boolean hasProtos; + + public J2ObjcSrcsProvider(NestedSet<J2ObjcSource> srcs, boolean hasProtos) { + this.srcs = srcs; + this.hasProtos = hasProtos; + } + + public NestedSet<J2ObjcSource> getSrcs() { + return srcs; + } + + /** + * Returns whether the translated source files in the provider has proto files. + */ + public boolean hasProtos() { + return hasProtos; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcActionsBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcActionsBuilder.java new file mode 100644 index 0000000..26742d9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcActionsBuilder.java
@@ -0,0 +1,599 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.IosSdkCommands.BIN_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.ASSET_CATALOG; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.DEFINE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FORCE_LOAD_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.Flag.USES_CPP; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.HEADER; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.INCLUDE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_DYLIB; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_FRAMEWORK; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.WEAK_SDK_FRAMEWORK; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCASSETS_DIR; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.io.ByteSource; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionRegistry; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext; +import com.google.devtools.build.lib.analysis.actions.BinaryFileWriteAction; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.util.LazyString; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.common.TargetDeviceFamily; +import com.google.devtools.build.xcode.util.Interspersing; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +/** + * Object that creates actions used by Objective-C rules. + */ +final class ObjcActionsBuilder { + private final ActionConstructionContext context; + private final IntermediateArtifacts intermediateArtifacts; + private final ObjcConfiguration objcConfiguration; + private final BuildConfiguration buildConfiguration; + private final ActionRegistry actionRegistry; + + ObjcActionsBuilder(ActionConstructionContext context, IntermediateArtifacts intermediateArtifacts, + ObjcConfiguration objcConfiguration, BuildConfiguration buildConfiguration, + ActionRegistry actionRegistry) { + this.context = Preconditions.checkNotNull(context); + this.intermediateArtifacts = Preconditions.checkNotNull(intermediateArtifacts); + this.objcConfiguration = Preconditions.checkNotNull(objcConfiguration); + this.buildConfiguration = Preconditions.checkNotNull(buildConfiguration); + this.actionRegistry = Preconditions.checkNotNull(actionRegistry); + } + + /** + * Creates a new spawn action builder that requires a darwin architecture to run. + */ + // TODO(bazel-team): Use everywhere we currently set the execution info manually. + static SpawnAction.Builder spawnOnDarwinActionBuilder() { + return new SpawnAction.Builder() + .setExecutionInfo(ImmutableMap.of(ExecutionRequirements.REQUIRES_DARWIN, "")); + } + + static final PathFragment JAVA = new PathFragment("/usr/bin/java"); + static final PathFragment CLANG = new PathFragment(BIN_DIR + "/clang"); + static final PathFragment CLANG_PLUSPLUS = new PathFragment(BIN_DIR + "/clang++"); + static final PathFragment LIBTOOL = new PathFragment(BIN_DIR + "/libtool"); + static final PathFragment IBTOOL = new PathFragment(IosSdkCommands.IBTOOL_PATH); + static final PathFragment DSYMUTIL = new PathFragment(BIN_DIR + "/dsymutil"); + static final PathFragment LIPO = new PathFragment(BIN_DIR + "/lipo"); + + // TODO(bazel-team): Reference a rule target rather than a jar file when Darwin runfiles work + // better. + private static SpawnAction.Builder spawnJavaOnDarwinActionBuilder(Artifact deployJarArtifact) { + return spawnOnDarwinActionBuilder() + .setExecutable(JAVA) + .addExecutableArguments("-jar", deployJarArtifact.getExecPathString()) + .addInput(deployJarArtifact); + } + + private void registerCompileAction( + Artifact sourceFile, + Artifact objFile, + Optional<Artifact> pchFile, + ObjcProvider objcProvider, + Iterable<String> otherFlags, + OptionsProvider optionsProvider) { + CustomCommandLine.Builder commandLine = new CustomCommandLine.Builder(); + if (ObjcRuleClasses.CPP_SOURCES.matches(sourceFile.getExecPath())) { + commandLine.add("-stdlib=libc++"); + } + commandLine + .add(IosSdkCommands.compileArgsForClang(objcConfiguration)) + .add(IosSdkCommands.commonLinkAndCompileArgsForClang( + objcProvider, objcConfiguration)) + .add(objcConfiguration.getCoptsForCompilationMode()) + .addBeforeEachPath("-iquote", ObjcCommon.userHeaderSearchPaths(buildConfiguration)) + .addBeforeEachExecPath("-include", pchFile.asSet()) + .addBeforeEachPath("-I", objcProvider.get(INCLUDE)) + .add(otherFlags) + .addFormatEach("-D%s", objcProvider.get(DEFINE)) + .add(objcConfiguration.getCopts()) + .add(optionsProvider.getCopts()) + .addExecPath("-c", sourceFile) + .addExecPath("-o", objFile); + + register(spawnOnDarwinActionBuilder() + .setMnemonic("ObjcCompile") + .setExecutable(CLANG) + .setCommandLine(commandLine.build()) + .addInput(sourceFile) + .addOutput(objFile) + .addTransitiveInputs(objcProvider.get(HEADER)) + .addTransitiveInputs(objcProvider.get(FRAMEWORK_FILE)) + .addInputs(pchFile.asSet()) + .build(context)); + } + + private static final ImmutableList<String> ARC_ARGS = ImmutableList.of("-fobjc-arc"); + private static final ImmutableList<String> NON_ARC_ARGS = ImmutableList.of("-fno-objc-arc"); + + /** + * Creates actions to compile each source file individually, and link all the compiled object + * files into a single archive library. + */ + void registerCompileAndArchiveActions(CompilationArtifacts compilationArtifacts, + ObjcProvider objcProvider, OptionsProvider optionsProvider) { + ImmutableList.Builder<Artifact> objFiles = new ImmutableList.Builder<>(); + for (Artifact sourceFile : compilationArtifacts.getSrcs()) { + Artifact objFile = intermediateArtifacts.objFile(sourceFile); + objFiles.add(objFile); + registerCompileAction(sourceFile, objFile, compilationArtifacts.getPchFile(), + objcProvider, ARC_ARGS, optionsProvider); + } + for (Artifact nonArcSourceFile : compilationArtifacts.getNonArcSrcs()) { + Artifact objFile = intermediateArtifacts.objFile(nonArcSourceFile); + objFiles.add(objFile); + registerCompileAction(nonArcSourceFile, objFile, compilationArtifacts.getPchFile(), + objcProvider, NON_ARC_ARGS, optionsProvider); + } + for (Artifact archive : compilationArtifacts.getArchive().asSet()) { + registerAll(archiveActions(context, objFiles.build(), archive, objcConfiguration, + intermediateArtifacts.objList())); + } + } + + private static Iterable<Action> archiveActions( + ActionConstructionContext context, + final Iterable<Artifact> objFiles, + final Artifact archive, + final ObjcConfiguration objcConfiguration, + final Artifact objList) { + + ImmutableList.Builder<Action> actions = new ImmutableList.Builder<>(); + + actions.add(new FileWriteAction( + context.getActionOwner(), objList, joinExecPaths(objFiles), /*makeExecutable=*/ false)); + + actions.add(spawnOnDarwinActionBuilder() + .setMnemonic("ObjcLink") + .setExecutable(LIBTOOL) + .setCommandLine(new CommandLine() { + @Override + public Iterable<String> arguments() { + return new ImmutableList.Builder<String>() + .add("-static") + .add("-filelist").add(objList.getExecPathString()) + .add("-arch_only").add(objcConfiguration.getIosCpu()) + .add("-syslibroot").add(IosSdkCommands.sdkDir(objcConfiguration)) + .add("-o").add(archive.getExecPathString()) + .build(); + } + }) + .addInputs(objFiles) + .addInput(objList) + .addOutput(archive) + .build(context)); + + return actions.build(); + } + + private void register(Action... action) { + actionRegistry.registerAction(action); + } + + private void registerAll(Iterable<? extends Action> actions) { + for (Action action : actions) { + actionRegistry.registerAction(action); + } + } + + private static ByteSource xcodegenControlFileBytes( + final Artifact pbxproj, final XcodeProvider.Project project, final String minimumOs) { + return new ByteSource() { + @Override + public InputStream openStream() { + return XcodeGenProtos.Control.newBuilder() + .setPbxproj(pbxproj.getExecPathString()) + .addAllTarget(project.targets()) + .addBuildSetting(XcodeGenProtos.XcodeprojBuildSetting.newBuilder() + .setName("IPHONEOS_DEPLOYMENT_TARGET") + .setValue(minimumOs) + .build()) + .build() + .toByteString() + .newInput(); + } + }; + } + + /** + * Generates actions needed to create an Xcode project file. + */ + void registerXcodegenActions( + ObjcRuleClasses.Tools baseTools, Artifact pbxproj, XcodeProvider.Project project) { + Artifact controlFile = intermediateArtifacts.pbxprojControlArtifact(); + register(new BinaryFileWriteAction( + context.getActionOwner(), + controlFile, + xcodegenControlFileBytes(pbxproj, project, objcConfiguration.getMinimumOs()), + /*makeExecutable=*/false)); + register(new SpawnAction.Builder() + .setMnemonic("GenerateXcodeproj") + .setExecutable(baseTools.xcodegen()) + .addArgument("--control") + .addInputArgument(controlFile) + .addOutput(pbxproj) + .addTransitiveInputs(project.getInputsToXcodegen()) + .build(context)); + } + + /** + * Creates actions to convert all files specified by the strings attribute into binary format. + */ + private static Iterable<Action> convertStringsActions( + ActionConstructionContext context, + ObjcRuleClasses.Tools baseTools, + StringsFiles stringsFiles) { + ImmutableList.Builder<Action> result = new ImmutableList.Builder<>(); + for (CompiledResourceFile stringsFile : stringsFiles) { + final Artifact original = stringsFile.getOriginal(); + final Artifact bundled = stringsFile.getBundled().getBundled(); + result.add(new SpawnAction.Builder() + .setMnemonic("ConvertStringsPlist") + .setExecutable(baseTools.plmerge()) + .setCommandLine(new CommandLine() { + @Override + public Iterable<String> arguments() { + return ImmutableList.of("--source_file", original.getExecPathString(), + "--out_file", bundled.getExecPathString()); + } + }) + .addInput(original) + .addOutput(bundled) + .build(context)); + } + return result.build(); + } + + private Action[] ibtoolzipAction(ObjcRuleClasses.Tools baseTools, String mnemonic, Artifact input, + Artifact zipOutput, String archiveRoot) { + return spawnJavaOnDarwinActionBuilder(baseTools.actooloribtoolzipDeployJar()) + .setMnemonic(mnemonic) + .setCommandLine(new CustomCommandLine.Builder() + // The next three arguments are positional, i.e. they don't have flags before them. + .addPath(zipOutput.getExecPath()) + .add(archiveRoot) + .addPath(IBTOOL) + + .add("--minimum-deployment-target").add(objcConfiguration.getMinimumOs()) + .addPath(input.getExecPath()) + .build()) + .addOutput(zipOutput) + .addInput(input) + .build(context); + } + + /** + * Creates actions to convert all files specified by the xibs attribute into nib format. + */ + private Iterable<Action> convertXibsActions(ObjcRuleClasses.Tools baseTools, XibFiles xibFiles) { + ImmutableList.Builder<Action> result = new ImmutableList.Builder<>(); + for (Artifact original : xibFiles) { + Artifact zipOutput = intermediateArtifacts.compiledXibFileZip(original); + String archiveRoot = BundleableFile.bundlePath( + FileSystemUtils.replaceExtension(original.getExecPath(), ".nib")); + result.add(ibtoolzipAction(baseTools, "XibCompile", original, zipOutput, archiveRoot)); + } + return result.build(); + } + + /** + * Outputs of an {@code actool} action besides the zip file. + */ + static final class ExtraActoolOutputs extends IterableWrapper<Artifact> { + ExtraActoolOutputs(Artifact... extraActoolOutputs) { + super(extraActoolOutputs); + } + } + + static final class ExtraActoolArgs extends IterableWrapper<String> { + ExtraActoolArgs(Iterable<String> args) { + super(args); + } + + ExtraActoolArgs(String... args) { + super(args); + } + } + + void registerActoolzipAction( + ObjcRuleClasses.Tools tools, + ObjcProvider provider, + Artifact zipOutput, + ExtraActoolOutputs extraActoolOutputs, + ExtraActoolArgs extraActoolArgs, + Set<TargetDeviceFamily> families) { + // TODO(bazel-team): Do not use the deploy jar explicitly here. There is currently a bug where + // we cannot .setExecutable({java_binary target}) and set REQUIRES_DARWIN in the execution info. + // Note that below we set the archive root to the empty string. This means that the generated + // zip file will be rooted at the bundle root, and we have to prepend the bundle root to each + // entry when merging it with the final .ipa file. + register(spawnJavaOnDarwinActionBuilder(tools.actooloribtoolzipDeployJar()) + .setMnemonic("AssetCatalogCompile") + .addTransitiveInputs(provider.get(ASSET_CATALOG)) + .addOutput(zipOutput) + .addOutputs(extraActoolOutputs) + .setCommandLine(actoolzipCommandLine( + objcConfiguration, + provider, + zipOutput, + extraActoolArgs, + ImmutableSet.copyOf(families))) + .build(context)); + } + + private static CommandLine actoolzipCommandLine( + final ObjcConfiguration objcConfiguration, + final ObjcProvider provider, + final Artifact zipOutput, + final ExtraActoolArgs extraActoolArgs, + final ImmutableSet<TargetDeviceFamily> families) { + return new CommandLine() { + @Override + public Iterable<String> arguments() { + ImmutableList.Builder<String> args = new ImmutableList.Builder<String>() + // The next three arguments are positional, i.e. they don't have flags before them. + .add(zipOutput.getExecPathString()) + .add("") // archive root + .add(IosSdkCommands.ACTOOL_PATH) + .add("--platform") + .add(objcConfiguration.getPlatform().getLowerCaseNameInPlist()) + .add("--minimum-deployment-target").add(objcConfiguration.getMinimumOs()); + for (TargetDeviceFamily targetDeviceFamily : families) { + args.add("--target-device").add(targetDeviceFamily.name().toLowerCase(Locale.US)); + } + return args + .addAll(PathFragment.safePathStrings(provider.get(XCASSETS_DIR))) + .addAll(extraActoolArgs) + .build(); + } + }; + } + + void registerIbtoolzipAction(ObjcRuleClasses.Tools tools, Artifact input, Artifact outputZip) { + String archiveRoot = BundleableFile.bundlePath(input.getExecPath()) + "c"; + register(ibtoolzipAction(tools, "StoryboardCompile", input, outputZip, archiveRoot)); + } + + @VisibleForTesting + static Iterable<String> commonMomczipArguments(ObjcConfiguration configuration) { + return ImmutableList.of( + "-XD_MOMC_SDKROOT=" + IosSdkCommands.sdkDir(configuration), + "-XD_MOMC_IOS_TARGET_VERSION=" + configuration.getMinimumOs(), + "-MOMC_PLATFORMS", configuration.getPlatform().getLowerCaseNameInPlist(), + "-XD_MOMC_TARGET_VERSION=10.6"); + } + + private static Iterable<Action> momczipActions(ActionConstructionContext context, + ObjcRuleClasses.Tools baseTools, final ObjcConfiguration objcConfiguration, + Iterable<Xcdatamodel> datamodels) { + ImmutableList.Builder<Action> result = new ImmutableList.Builder<>(); + for (Xcdatamodel datamodel : datamodels) { + final Artifact outputZip = datamodel.getOutputZip(); + final String archiveRoot = datamodel.archiveRootForMomczip(); + final String container = datamodel.getContainer().getSafePathString(); + result.add(spawnJavaOnDarwinActionBuilder(baseTools.momczipDeployJar()) + .setMnemonic("MomCompile") + .addOutput(outputZip) + .addInputs(datamodel.getInputs()) + .setCommandLine(new CommandLine() { + @Override + public Iterable<String> arguments() { + return new ImmutableList.Builder<String>() + .add(outputZip.getExecPathString()) + .add(archiveRoot) + .add(IosSdkCommands.MOMC_PATH) + .addAll(commonMomczipArguments(objcConfiguration)) + .add(container) + .build(); + } + }) + .build(context)); + } + return result.build(); + } + + private static final String FRAMEWORK_SUFFIX = ".framework"; + + /** + * All framework names to pass to the linker using {@code -framework} flags. For a framework in + * the directory foo/bar.framework, the name is "bar". Each framework is found without using the + * full path by means of the framework search paths. The search paths are added by + * {@link IosSdkCommands#commonLinkAndCompileArgsForClang(ObjcProvider, ObjcConfiguration)}). + * + * <p>It's awful that we can't pass the full path to the framework and avoid framework search + * paths, but this is imposed on us by clang. clang does not support passing the full path to the + * framework, so Bazel cannot do it either. + */ + private static Iterable<String> frameworkNames(ObjcProvider provider) { + List<String> names = new ArrayList<>(); + Iterables.addAll(names, SdkFramework.names(provider.get(SDK_FRAMEWORK))); + for (PathFragment frameworkDir : provider.get(FRAMEWORK_DIR)) { + String segment = frameworkDir.getBaseName(); + Preconditions.checkState(segment.endsWith(FRAMEWORK_SUFFIX), + "expect %s to end with %s, but it does not", segment, FRAMEWORK_SUFFIX); + names.add(segment.substring(0, segment.length() - FRAMEWORK_SUFFIX.length())); + } + return names; + } + + static final class ExtraLinkArgs extends IterableWrapper<String> { + ExtraLinkArgs(Iterable<String> args) { + super(args); + } + + ExtraLinkArgs(String... args) { + super(args); + } + } + + static final class ExtraLinkInputs extends IterableWrapper<Artifact> { + ExtraLinkInputs(Iterable<Artifact> inputs) { + super(inputs); + } + + ExtraLinkInputs(Artifact... inputs) { + super(inputs); + } + } + + private static final class LinkCommandLine extends CommandLine { + private static final Joiner commandJoiner = Joiner.on(' '); + private final ObjcProvider objcProvider; + private final ObjcConfiguration objcConfiguration; + private final Artifact linkedBinary; + private final Optional<Artifact> dsymBundle; + private final ExtraLinkArgs extraLinkArgs; + + LinkCommandLine(ObjcConfiguration objcConfiguration, ExtraLinkArgs extraLinkArgs, + ObjcProvider objcProvider, Artifact linkedBinary, Optional<Artifact> dsymBundle) { + this.objcConfiguration = Preconditions.checkNotNull(objcConfiguration); + this.extraLinkArgs = Preconditions.checkNotNull(extraLinkArgs); + this.objcProvider = Preconditions.checkNotNull(objcProvider); + this.linkedBinary = Preconditions.checkNotNull(linkedBinary); + this.dsymBundle = Preconditions.checkNotNull(dsymBundle); + } + + Iterable<String> dylibPaths() { + ImmutableList.Builder<String> args = new ImmutableList.Builder<>(); + for (String dylib : objcProvider.get(SDK_DYLIB)) { + args.add(String.format( + "%s/usr/lib/%s.dylib", IosSdkCommands.sdkDir(objcConfiguration), dylib)); + } + return args.build(); + } + + @Override + public Iterable<String> arguments() { + StringBuilder argumentStringBuilder = new StringBuilder(); + + Iterable<String> archiveExecPaths = Artifact.toExecPaths( + Iterables.concat(objcProvider.get(LIBRARY), objcProvider.get(IMPORTED_LIBRARY))); + commandJoiner.appendTo(argumentStringBuilder, new ImmutableList.Builder<String>() + .add(objcProvider.is(USES_CPP) ? CLANG_PLUSPLUS.toString() : CLANG.toString()) + .addAll(objcProvider.is(USES_CPP) + ? ImmutableList.of("-stdlib=libc++") : ImmutableList.<String>of()) + .addAll(IosSdkCommands.commonLinkAndCompileArgsForClang(objcProvider, objcConfiguration)) + .add("-Xlinker", "-objc_abi_version") + .add("-Xlinker", "2") + .add("-fobjc-link-runtime") + .addAll(IosSdkCommands.DEFAULT_LINKER_FLAGS) + .addAll(Interspersing.beforeEach("-framework", frameworkNames(objcProvider))) + .addAll(Interspersing.beforeEach( + "-weak_framework", SdkFramework.names(objcProvider.get(WEAK_SDK_FRAMEWORK)))) + .add("-o", linkedBinary.getExecPathString()) + .addAll(archiveExecPaths) + .addAll(dylibPaths()) + .addAll(extraLinkArgs) + .build()); + + // Call to dsymutil for debug symbol generation must happen in the link action. + // All debug symbol information is encoded in object files inside archive files. To generate + // the debug symbol bundle, dsymutil will look inside the linked binary for the encoded + // absolute paths to archive files, which are only valid in the link action. + for (Artifact justDsymBundle : dsymBundle.asSet()) { + argumentStringBuilder.append(" "); + commandJoiner.appendTo(argumentStringBuilder, new ImmutableList.Builder<String>() + .add("&&") + .add(DSYMUTIL.toString()) + .add(linkedBinary.getExecPathString()) + .add("-o").add(justDsymBundle.getExecPathString()) + .build()); + } + + return ImmutableList.of(argumentStringBuilder.toString()); + } + } + + /** + * Generates an action to link a binary. + */ + void registerLinkAction(Artifact linkedBinary, ObjcProvider objcProvider, + ExtraLinkArgs extraLinkArgs, ExtraLinkInputs extraLinkInputs, Optional<Artifact> dsymBundle) { + extraLinkArgs = new ExtraLinkArgs(Iterables.concat( + Interspersing.beforeEach( + "-force_load", Artifact.toExecPaths(objcProvider.get(FORCE_LOAD_LIBRARY))), + extraLinkArgs)); + register(spawnOnDarwinActionBuilder() + .setMnemonic("ObjcLink") + .setShellCommand(ImmutableList.of("/bin/bash", "-c")) + .setCommandLine( + new LinkCommandLine(objcConfiguration, extraLinkArgs, objcProvider, linkedBinary, + dsymBundle)) + .addOutput(linkedBinary) + .addOutputs(dsymBundle.asSet()) + .addTransitiveInputs(objcProvider.get(LIBRARY)) + .addTransitiveInputs(objcProvider.get(IMPORTED_LIBRARY)) + .addTransitiveInputs(objcProvider.get(FRAMEWORK_FILE)) + .addInputs(extraLinkInputs) + .build(context)); + } + + static final class StringsFiles extends IterableWrapper<CompiledResourceFile> { + StringsFiles(Iterable<CompiledResourceFile> files) { + super(files); + } + } + + /** + * Registers actions for resource conversion that are needed by all rules that inherit from + * {@link ObjcBase}. + */ + void registerResourceActions(ObjcRuleClasses.Tools baseTools, StringsFiles stringsFiles, + XibFiles xibFiles, Iterable<Xcdatamodel> datamodels) { + registerAll(convertStringsActions(context, baseTools, stringsFiles)); + registerAll(convertXibsActions(baseTools, xibFiles)); + registerAll(momczipActions(context, baseTools, objcConfiguration, datamodels)); + } + + static LazyString joinExecPaths(final Iterable<Artifact> artifacts) { + return new LazyString() { + @Override + public String toString() { + return Artifact.joinExecPaths("\n", artifacts); + } + }; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinary.java new file mode 100644 index 0000000..52c897f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinary.java
@@ -0,0 +1,147 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LIBRARY; +import static com.google.devtools.build.lib.rules.objc.XcodeProductType.APPLICATION; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.objc.ApplicationSupport.LinkedBinary; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkArgs; +import com.google.devtools.build.lib.rules.objc.ObjcActionsBuilder.ExtraLinkInputs; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.CompilationAttributes; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.ResourceAttributes; + +/** + * Implementation for the "objc_binary" rule. + */ +public class ObjcBinary implements RuleConfiguredTargetFactory { + + @VisibleForTesting + static final String REQUIRES_AT_LEAST_ONE_LIBRARY_OR_SOURCE_FILE = + "At least one library dependency or source file is required."; + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + ObjcCommon common = common(ruleContext); + OptionsProvider optionsProvider = optionsProvider(ruleContext); + + ObjcProvider objcProvider = common.getObjcProvider(); + if (!hasLibraryOrSources(objcProvider)) { + ruleContext.ruleError(REQUIRES_AT_LEAST_ONE_LIBRARY_OR_SOURCE_FILE); + return null; + } + + XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder(); + NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.stableOrder(); + + new CompilationSupport(ruleContext) + .registerJ2ObjcCompileAndArchiveActions(optionsProvider, common.getObjcProvider()) + .registerCompileAndArchiveActions(common, optionsProvider) + .addXcodeSettings(xcodeProviderBuilder, common, optionsProvider) + .registerLinkActions(common.getObjcProvider(), new ExtraLinkArgs(), new ExtraLinkInputs()) + .validateAttributes(); + + // TODO(bazel-team): Remove once all bundle users are migrated to ios_application. + ApplicationSupport applicationSupport = new ApplicationSupport( + ruleContext, common.getObjcProvider(), optionsProvider, LinkedBinary.LOCAL_AND_DEPENDENCIES) + .registerActions() + .addXcodeSettings(xcodeProviderBuilder) + .addFilesToBuild(filesToBuild) + .validateAttributes(); + + new ResourceSupport(ruleContext) + .registerActions(common.getStoryboards()) + .validateAttributes() + .addXcodeSettings(xcodeProviderBuilder); + + XcodeSupport xcodeSupport = new XcodeSupport(ruleContext) + // TODO(bazel-team): Use LIBRARY_STATIC as parameter instead of APPLICATION once objc_binary + // no longer creates an application bundle + .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), APPLICATION) + .addDependencies(xcodeProviderBuilder) + .addFilesToBuild(filesToBuild); + XcodeProvider xcodeProvider = xcodeProviderBuilder.build(); + xcodeSupport.registerActions(xcodeProvider); + + // TODO(bazel-team): Stop exporting an XcTestAppProvider once objc_binary no longer creates an + // application bundle. + return common.configuredTarget( + filesToBuild.build(), + Optional.of(xcodeProvider), + Optional.<ObjcProvider>absent(), + Optional.of(applicationSupport.xcTestAppProvider()), + Optional.<J2ObjcSrcsProvider>absent()); + } + + private OptionsProvider optionsProvider(RuleContext ruleContext) { + return new OptionsProvider.Builder() + .addCopts(ruleContext.getTokenizedStringListAttr("copts")) + .addInfoplists(ruleContext.getPrerequisiteArtifacts("infoplist", Mode.TARGET).list()) + .addTransitive(Optional.fromNullable( + ruleContext.getPrerequisite("options", Mode.TARGET, OptionsProvider.class))) + .build(); + } + + private boolean hasLibraryOrSources(ObjcProvider objcProvider) { + return !Iterables.isEmpty(objcProvider.get(LIBRARY)) // Includes sources from this target. + || !Iterables.isEmpty(objcProvider.get(IMPORTED_LIBRARY)); + } + + private ObjcCommon common(RuleContext ruleContext) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + CompilationArtifacts compilationArtifacts = + CompilationSupport.compilationArtifacts(ruleContext); + + return new ObjcCommon.Builder(ruleContext) + .setCompilationAttributes(new CompilationAttributes(ruleContext)) + .setResourceAttributes(new ResourceAttributes(ruleContext)) + .setCompilationArtifacts(compilationArtifacts) + .addDepObjcProviders(ruleContext.getPrerequisites("deps", Mode.TARGET, ObjcProvider.class)) + .addDepObjcProviders( + ruleContext.getPrerequisites("bundles", Mode.TARGET, ObjcProvider.class)) + .addNonPropagatedDepObjcProviders( + ruleContext.getPrerequisites("non_propagated_deps", Mode.TARGET, ObjcProvider.class)) + .setIntermediateArtifacts(intermediateArtifacts) + .setAlwayslink(false) + .addExtraImportLibraries(j2ObjcLibraries(ruleContext)) + .setLinkedBinary(intermediateArtifacts.singleArchitectureBinary()) + .build(); + } + + private Iterable<Artifact> j2ObjcLibraries(RuleContext ruleContext) { + J2ObjcSrcsProvider j2ObjcSrcsProvider = ObjcRuleClasses.j2ObjcSrcsProvider(ruleContext); + ImmutableList.Builder<Artifact> j2objcLibraries = ImmutableList.builder(); + + // TODO(bazel-team): Refactor the code to stop flattening the nested set here. + for (J2ObjcSource j2ObjcSource : j2ObjcSrcsProvider.getSrcs()) { + j2objcLibraries.add( + ObjcRuleClasses.j2objcIntermediateArtifacts(ruleContext, j2ObjcSource).archive()); + } + + return j2objcLibraries.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinaryRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinaryRule.java new file mode 100644 index 0000000..c9fc57e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBinaryRule.java
@@ -0,0 +1,62 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for objc_binary. + */ +@BlazeRule(name = "objc_binary", + factoryClass = ObjcBinary.class, + ancestors = { ObjcLibraryRule.class, IosApplicationRule.class }) +public class ObjcBinaryRule implements RuleDefinition { + + @Override + public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) { + return builder + // TODO(bazel-team): Remove bundling functionality (dependency on IosApplicationRule). + /*<!-- #BLAZE_RULE(objc_binary).IMPLICIT_OUTPUTS --> + <ul> + <li><code><var>name</var>.ipa</code>: the application bundle as an <code>.ipa</code> + file</li> + <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: An Xcode project file which + can be used to develop or build on a Mac.</li> + </ul> + <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/ + .setImplicitOutputsFunction( + ImplicitOutputsFunction.fromFunctions(ApplicationSupport.IPA, XcodeSupport.PBXPROJ)) + .removeAttribute("binary") + .removeAttribute("alwayslink") + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_binary, TYPE = BINARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule produces an application bundle by linking one or more Objective-C libraries.</p> + +${IMPLICIT_OUTPUTS} + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundle.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundle.java new file mode 100644 index 0000000..6585cba --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundle.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.collect.nestedset.Order.STABLE_ORDER; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * Implementation for {@code objc_bundle}. + */ +public class ObjcBundle implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + ObjcCommon common = new ObjcCommon.Builder(ruleContext).build(); + + ImmutableList<Artifact> bundleImports = ruleContext + .getPrerequisiteArtifacts("bundle_imports", Mode.TARGET).list(); + Iterable<String> bundleImportErrors = + ObjcCommon.notInContainerErrors(bundleImports, ObjcCommon.BUNDLE_CONTAINER_TYPE); + for (String error : bundleImportErrors) { + ruleContext.attributeError("bundle_imports", error); + } + + return common.configuredTarget( + /*filesToBuild=*/NestedSetBuilder.<Artifact>emptySet(STABLE_ORDER), + Optional.<XcodeProvider>absent(), + Optional.of(common.getObjcProvider()), + Optional.<XcTestAppProvider>absent(), + Optional.<J2ObjcSrcsProvider>absent()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibrary.java new file mode 100644 index 0000000..0e7f6b0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibrary.java
@@ -0,0 +1,103 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.NESTED_BUNDLE; +import static com.google.devtools.build.lib.rules.objc.XcodeProductType.BUNDLE; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.ResourceAttributes; +import com.google.devtools.build.xcode.common.TargetDeviceFamily; + +/** + * Implementation for {@code objc_bundle_library}. + */ +public class ObjcBundleLibrary implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + ObjcCommon common = common(ruleContext); + OptionsProvider optionsProvider = optionsProvider(ruleContext); + + Bundling bundling = bundling(ruleContext, common, optionsProvider); + + XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder(); + NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.stableOrder(); + + // TODO(bazel-team): Figure out if the target device is important, and what to set it to. It may + // have to inherit this from the binary being built. As of this writing, this is only used for + // asset catalogs compilation (actool). + new BundleSupport(ruleContext, ImmutableSet.of(TargetDeviceFamily.IPHONE), bundling) + .registerActions(common.getObjcProvider()) + .addXcodeSettings(xcodeProviderBuilder); + + new ResourceSupport(ruleContext) + .validateAttributes() + .registerActions(common.getStoryboards()) + .addXcodeSettings(xcodeProviderBuilder); + + new XcodeSupport(ruleContext) + .addFilesToBuild(filesToBuild) + .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), BUNDLE) + .registerActions(xcodeProviderBuilder.build()); + + ObjcProvider nestedBundleProvider = new ObjcProvider.Builder() + .add(NESTED_BUNDLE, bundling) + .build(); + + return common.configuredTarget( + filesToBuild.build(), + Optional.of(xcodeProviderBuilder.build()), + Optional.of(nestedBundleProvider), + Optional.<XcTestAppProvider>absent(), + Optional.<J2ObjcSrcsProvider>absent()); + } + + private OptionsProvider optionsProvider(RuleContext ruleContext) { + return new OptionsProvider.Builder() + .addInfoplists(ruleContext.getPrerequisiteArtifacts("infoplist", Mode.TARGET).list()) + .build(); + } + + private Bundling bundling( + RuleContext ruleContext, ObjcCommon common, OptionsProvider optionsProvider) { + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + return new Bundling.Builder() + .setName(ruleContext.getLabel().getName()) + .setBundleDirSuffix(".bundle") + .setObjcProvider(common.getObjcProvider()) + .setInfoplistMerging( + BundleSupport.infoPlistMerging(ruleContext, common.getObjcProvider(), optionsProvider)) + .setIntermediateArtifacts(intermediateArtifacts) + .build(); + } + + private ObjcCommon common(RuleContext ruleContext) { + return new ObjcCommon.Builder(ruleContext) + .setResourceAttributes(new ResourceAttributes(ruleContext)) + .addDepObjcProviders( + ruleContext.getPrerequisites("bundles", Mode.TARGET, ObjcProvider.class)) + .setIntermediateArtifacts(ObjcRuleClasses.intermediateArtifacts(ruleContext)) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibraryRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibraryRule.java new file mode 100644 index 0000000..b9405a6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleLibraryRule.java
@@ -0,0 +1,67 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for objc_bundle_library. + */ +@BlazeRule(name = "objc_bundle_library", + factoryClass = ObjcBundleLibrary.class, + ancestors = { ObjcRuleClasses.ObjcBaseResourcesRule.class, + ObjcRuleClasses.ObjcHasInfoplistRule.class }) +public class ObjcBundleLibraryRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /*<!-- #BLAZE_RULE(objc_bundle_library).IMPLICIT_OUTPUTS --> + <ul> + <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: An Xcode project file which + can be used to develop or build on a Mac.</li> + </ul> + <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/ + .setImplicitOutputsFunction(ImplicitOutputsFunction.fromFunctions(XcodeSupport.PBXPROJ)) + /* <!-- #BLAZE_RULE(objc_bundle_library).ATTRIBUTE(bundles) --> + The list of bundle targets that this target requires to be included in the final bundle. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("bundles", LABEL_LIST) + .direct_compile_time_input() + .allowedRuleClasses("objc_bundle", "objc_bundle_library") + .allowedFileTypes()) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_bundle_library, TYPE = LIBRARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule encapsulates a library which is provided to dependers as a bundle. +A <code>objc_bundle_library</code>'s resources are put in a nested bundle in +the final iOS application. + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleRule.java new file mode 100644 index 0000000..9be0d04 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcBundleRule.java
@@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.util.FileTypeSet; + +/** + * Rule definition for objc_bundle. + */ +@BlazeRule(name = "objc_bundle", + factoryClass = ObjcBundle.class, + ancestors = { BaseRuleClasses.BaseRule.class }) +public class ObjcBundleRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE(objc_bundle).ATTRIBUTE(bundle_imports) --> + The list of files under a <code>.bundle</code> directory which are + provided to Objective-C targets that depend on this target. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("bundle_imports", LABEL_LIST) + .allowedFileTypes(FileTypeSet.ANY_FILE) + .nonEmpty()) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_bundle, TYPE = LIBRARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule encapsulates an already-built bundle. It is defined by a list of +files in one or more <code>.bundle</code> directories. + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommandLineOptions.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommandLineOptions.java new file mode 100644 index 0000000..f85609a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommandLineOptions.java
@@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.xcode.common.BuildOptionsUtil.DEFAULT_OPTIONS_NAME; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.config.FragmentOptions; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.common.options.Option; + +import java.util.List; + +/** + * Command-line options for building Objective-C targets. + */ +public class + ObjcCommandLineOptions extends FragmentOptions { + @Option(name = "ios_sdk_version", + defaultValue = DEFAULT_SDK_VERSION, + category = "undocumented", + help = "Specifies the version of the iOS SDK to use to build iOS applications." + ) + public String iosSdkVersion; + + @VisibleForTesting static final String DEFAULT_SDK_VERSION = "8.1"; + + @Option(name = "ios_simulator_version", + defaultValue = "7.1", + category = "undocumented", + help = "The version of iOS to run on the simulator when running tests. This is ignored if the" + + " ios_test rule specifies the target device.", + deprecationWarning = "This flag is deprecated in favor of the target_device attribute and" + + " will eventually removed.") + public String iosSimulatorVersion; + + @Option(name = "ios_cpu", + defaultValue = "i386", + category = "undocumented", + help = "Specifies to target CPU of iOS compilation.") + public String iosCpu; + + @Option(name = "xcode_options", + defaultValue = DEFAULT_OPTIONS_NAME, + category = "undocumented", + help = "Specifies the name of the build settings to use.") + public String xcodeOptions; + + @Option(name = "objc_generate_debug_symbols", + defaultValue = "false", + category = "undocumented", + help = "Specifies whether to generate debug symbol(.dSYM) file.") + public boolean generateDebugSymbols; + + @Option(name = "objccopt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to Objective C compilation.") + public List<String> copts; + + @Option(name = "ios_minimum_os", + defaultValue = DEFAULT_MINIMUM_IOS, + category = "flags", + help = "Minimum compatible iOS version for target simulators and devices.") + public String iosMinimumOs; + + @VisibleForTesting static final String DEFAULT_MINIMUM_IOS = "7.0"; + + @Override + public void addAllLabels(Multimap<String, Label> labelMap) {} +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommon.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommon.java new file mode 100644 index 0000000..ade86ee --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommon.java
@@ -0,0 +1,620 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.ASSET_CATALOG; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_IMPORT_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.DEFINE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FLAG; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FORCE_LOAD_FOR_XCODEGEN; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FORCE_LOAD_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.Flag.USES_CPP; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.GENERAL_RESOURCE_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.HEADER; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.INCLUDE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.LINKED_BINARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.MERGE_ZIP; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_DYLIB; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_FRAMEWORK; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.WEAK_SDK_FRAMEWORK; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCASSETS_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCDATAMODEL; +import static com.google.devtools.build.lib.vfs.PathFragment.TO_PATH_FRAGMENT; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CcCommon; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.util.Interspersing; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Contains information common to multiple objc_* rules, and provides a unified API for extracting + * and accessing it. + */ +// TODO(bazel-team): Decompose and subsume area-specific logic and data into the various *Support +// classes. Make sure to distinguish rule output (providers, runfiles, ...) from intermediate, +// rule-internal information. Any provider created by a rule should not be read, only published. +public final class ObjcCommon { + /** + * Provides a way to access attributes that are common to all compilation rules that inherit from + * {@link ObjcRuleClasses.ObjcCompilationRule}. + */ + // TODO(bazel-team): Delete and move into support-specific attributes classes once ObjcCommon is + // gone. + static final class CompilationAttributes { + private final RuleContext ruleContext; + private final ObjcSdkFrameworks.Attributes sdkFrameworkAttributes; + + CompilationAttributes(RuleContext ruleContext) { + this.ruleContext = Preconditions.checkNotNull(ruleContext); + this.sdkFrameworkAttributes = new ObjcSdkFrameworks.Attributes(ruleContext); + } + + ImmutableList<Artifact> hdrs() { + return ImmutableList.copyOf(CcCommon.getHeaders(ruleContext)); + } + + Iterable<PathFragment> includes() { + return Iterables.transform( + ruleContext.attributes().get("includes", Type.STRING_LIST), + PathFragment.TO_PATH_FRAGMENT); + } + + Iterable<PathFragment> sdkIncludes() { + return Iterables.transform( + ruleContext.attributes().get("sdk_includes", Type.STRING_LIST), + PathFragment.TO_PATH_FRAGMENT); + } + + /** + * Returns the value of the sdk_frameworks attribute plus frameworks that are included + * automatically. + */ + ImmutableSet<SdkFramework> sdkFrameworks() { + return sdkFrameworkAttributes.sdkFrameworks(); + } + + /** + * Returns the value of the weak_sdk_frameworks attribute. + */ + ImmutableSet<SdkFramework> weakSdkFrameworks() { + return sdkFrameworkAttributes.weakSdkFrameworks(); + } + + /** + * Returns the value of the sdk_dylibs attribute. + */ + ImmutableSet<String> sdkDylibs() { + return sdkFrameworkAttributes.sdkDylibs(); + } + + /** + * Returns the exec paths of all header search paths that should be added to this target and + * dependers on this target, obtained from the {@code includes} attribute. + */ + ImmutableList<PathFragment> headerSearchPaths() { + ImmutableList.Builder<PathFragment> paths = new ImmutableList.Builder<>(); + PathFragment packageFragment = ruleContext.getLabel().getPackageFragment(); + List<PathFragment> rootFragments = ImmutableList.of( + packageFragment, + ruleContext.getConfiguration().getGenfilesFragment().getRelative(packageFragment)); + + Iterable<PathFragment> relativeIncludes = + Iterables.filter(includes(), Predicates.not(PathFragment.IS_ABSOLUTE)); + for (PathFragment include : relativeIncludes) { + for (PathFragment rootFragment : rootFragments) { + paths.add(rootFragment.getRelative(include).normalize()); + } + } + return paths.build(); + } + } + + /** + * Provides a way to access attributes that are common to all resources rules that inherit from + * {@link ObjcRuleClasses.ObjcBaseResourcesRule}. + */ + // TODO(bazel-team): Delete and move into support-specific attributes classes once ObjcCommon is + // gone. + static final class ResourceAttributes { + private final RuleContext ruleContext; + + ResourceAttributes(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + ImmutableList<Artifact> strings() { + return ruleContext.getPrerequisiteArtifacts("strings", Mode.TARGET).list(); + } + + ImmutableList<Artifact> xibs() { + return ruleContext.getPrerequisiteArtifacts("xibs", Mode.TARGET) + .errorsForNonMatching(ObjcRuleClasses.XIB_TYPE) + .list(); + } + + ImmutableList<Artifact> storyboards() { + return ruleContext.getPrerequisiteArtifacts("storyboards", Mode.TARGET).list(); + } + + ImmutableList<Artifact> resources() { + return ruleContext.getPrerequisiteArtifacts("resources", Mode.TARGET).list(); + } + + ImmutableList<Artifact> datamodels() { + return ruleContext.getPrerequisiteArtifacts("datamodels", Mode.TARGET).list(); + } + + ImmutableList<Artifact> assetCatalogs() { + return ruleContext.getPrerequisiteArtifacts("asset_catalogs", Mode.TARGET).list(); + } + } + + static class Builder { + private RuleContext context; + private Optional<CompilationAttributes> compilationAttributes = Optional.absent(); + private Optional<ResourceAttributes> resourceAttributes = Optional.absent(); + private Iterable<SdkFramework> extraSdkFrameworks = ImmutableList.of(); + private Iterable<SdkFramework> extraWeakSdkFrameworks = ImmutableList.of(); + private Iterable<String> extraSdkDylibs = ImmutableList.of(); + private Iterable<Artifact> frameworkImports = ImmutableList.of(); + private Optional<CompilationArtifacts> compilationArtifacts = Optional.absent(); + private Iterable<ObjcProvider> depObjcProviders = ImmutableList.of(); + private Iterable<ObjcProvider> directDepObjcProviders = ImmutableList.of(); + private Iterable<String> defines = ImmutableList.of(); + private Iterable<PathFragment> userHeaderSearchPaths = ImmutableList.of(); + private Iterable<Artifact> headers = ImmutableList.of(); + private IntermediateArtifacts intermediateArtifacts; + private boolean alwayslink; + private Iterable<Artifact> extraImportLibraries = ImmutableList.of(); + private Optional<Artifact> linkedBinary = Optional.absent(); + + Builder(RuleContext context) { + this.context = Preconditions.checkNotNull(context); + } + + public Builder setCompilationAttributes(CompilationAttributes baseCompilationAttributes) { + Preconditions.checkState(!this.compilationAttributes.isPresent(), + "compilationAttributes is already set to: %s", this.compilationAttributes); + this.compilationAttributes = Optional.of(baseCompilationAttributes); + return this; + } + + public Builder setResourceAttributes(ResourceAttributes baseResourceAttributes) { + Preconditions.checkState(!this.resourceAttributes.isPresent(), + "resourceAttributes is already set to: %s", this.resourceAttributes); + this.resourceAttributes = Optional.of(baseResourceAttributes); + return this; + } + + Builder addExtraSdkFrameworks(Iterable<SdkFramework> extraSdkFrameworks) { + this.extraSdkFrameworks = Iterables.concat(this.extraSdkFrameworks, extraSdkFrameworks); + return this; + } + + Builder addExtraWeakSdkFrameworks(Iterable<SdkFramework> extraWeakSdkFrameworks) { + this.extraWeakSdkFrameworks = + Iterables.concat(this.extraWeakSdkFrameworks, extraWeakSdkFrameworks); + return this; + } + + Builder addExtraSdkDylibs(Iterable<String> extraSdkDylibs) { + this.extraSdkDylibs = Iterables.concat(this.extraSdkDylibs, extraSdkDylibs); + return this; + } + + Builder addFrameworkImports(Iterable<Artifact> frameworkImports) { + this.frameworkImports = Iterables.concat(this.frameworkImports, frameworkImports); + return this; + } + + Builder setCompilationArtifacts(CompilationArtifacts compilationArtifacts) { + Preconditions.checkState(!this.compilationArtifacts.isPresent(), + "compilationArtifacts is already set to: %s", this.compilationArtifacts); + this.compilationArtifacts = Optional.of(compilationArtifacts); + return this; + } + + /** + * Add providers which will be exposed both to the declaring rule and to any dependers on the + * declaring rule. + */ + Builder addDepObjcProviders(Iterable<ObjcProvider> depObjcProviders) { + this.depObjcProviders = Iterables.concat(this.depObjcProviders, depObjcProviders); + return this; + } + + /** + * Add providers which will only be used by the declaring rule, and won't be propagated to any + * dependers on the declaring rule. + */ + Builder addNonPropagatedDepObjcProviders(Iterable<ObjcProvider> directDepObjcProviders) { + this.directDepObjcProviders = Iterables.concat( + this.directDepObjcProviders, directDepObjcProviders); + return this; + } + + public Builder addUserHeaderSearchPaths(Iterable<PathFragment> userHeaderSearchPaths) { + this.userHeaderSearchPaths = + Iterables.concat(this.userHeaderSearchPaths, userHeaderSearchPaths); + return this; + } + + public Builder addDefines(Iterable<String> defines) { + this.defines = Iterables.concat(this.defines, defines); + return this; + } + + public Builder addHeaders(Iterable<Artifact> headers) { + this.headers = Iterables.concat(this.headers, headers); + return this; + } + + Builder setIntermediateArtifacts(IntermediateArtifacts intermediateArtifacts) { + this.intermediateArtifacts = intermediateArtifacts; + return this; + } + + Builder setAlwayslink(boolean alwayslink) { + this.alwayslink = alwayslink; + return this; + } + + /** + * Adds additional static libraries to be linked into the final ObjC application bundle. + */ + Builder addExtraImportLibraries(Iterable<Artifact> extraImportLibraries) { + this.extraImportLibraries = Iterables.concat(this.extraImportLibraries, extraImportLibraries); + return this; + } + + /** + * Sets a linked binary generated by this rule to be propagated to dependers. + */ + Builder setLinkedBinary(Artifact linkedBinary) { + this.linkedBinary = Optional.of(linkedBinary); + return this; + } + + ObjcCommon build() { + Iterable<BundleableFile> bundleImports = BundleableFile.bundleImportsFromRule(context); + + ObjcProvider.Builder objcProvider = new ObjcProvider.Builder() + .addAll(IMPORTED_LIBRARY, extraImportLibraries) + .addAll(BUNDLE_FILE, bundleImports) + .addAll(BUNDLE_IMPORT_DIR, + uniqueContainers(BundleableFile.toArtifacts(bundleImports), BUNDLE_CONTAINER_TYPE)) + .addAll(SDK_FRAMEWORK, extraSdkFrameworks) + .addAll(WEAK_SDK_FRAMEWORK, extraWeakSdkFrameworks) + .addAll(SDK_DYLIB, extraSdkDylibs) + .addAll(FRAMEWORK_FILE, frameworkImports) + .addAll(FRAMEWORK_DIR, uniqueContainers(frameworkImports, FRAMEWORK_CONTAINER_TYPE)) + .addAll(INCLUDE, userHeaderSearchPaths) + .addAll(DEFINE, defines) + .addAll(HEADER, headers) + .addTransitiveAndPropagate(depObjcProviders) + .addTransitiveWithoutPropagating(directDepObjcProviders); + + Storyboards storyboards; + Iterable<Xcdatamodel> datamodels; + if (compilationAttributes.isPresent()) { + CompilationAttributes attributes = compilationAttributes.get(); + ObjcConfiguration objcConfiguration = ObjcRuleClasses.objcConfiguration(context); + Iterable<PathFragment> sdkIncludes = Iterables.transform( + Interspersing.prependEach( + IosSdkCommands.sdkDir(objcConfiguration) + "/usr/include/", + PathFragment.safePathStrings(attributes.sdkIncludes())), + TO_PATH_FRAGMENT); + objcProvider + .addAll(HEADER, attributes.hdrs()) + .addAll(INCLUDE, attributes.headerSearchPaths()) + .addAll(INCLUDE, sdkIncludes) + .addAll(SDK_FRAMEWORK, attributes.sdkFrameworks()) + .addAll(WEAK_SDK_FRAMEWORK, attributes.weakSdkFrameworks()) + .addAll(SDK_DYLIB, attributes.sdkDylibs()); + } + + if (resourceAttributes.isPresent()) { + ResourceAttributes attributes = resourceAttributes.get(); + storyboards = Storyboards.fromInputs(attributes.storyboards(), intermediateArtifacts); + datamodels = Xcdatamodels.xcdatamodels(intermediateArtifacts, attributes.datamodels()); + Iterable<CompiledResourceFile> compiledResources = + CompiledResourceFile.fromStringsFiles(intermediateArtifacts, attributes.strings()); + XibFiles xibFiles = new XibFiles(attributes.xibs()); + + objcProvider + .addTransitiveAndPropagate(MERGE_ZIP, storyboards.getOutputZips()) + .addAll(MERGE_ZIP, xibFiles.compiledZips(intermediateArtifacts)) + .addAll(GENERAL_RESOURCE_FILE, storyboards.getInputs()) + .addAll(GENERAL_RESOURCE_FILE, attributes.resources()) + .addAll(GENERAL_RESOURCE_FILE, attributes.strings()) + .addAll(GENERAL_RESOURCE_FILE, attributes.xibs()) + .addAll(BUNDLE_FILE, BundleableFile.nonCompiledResourceFiles(attributes.resources())) + .addAll(BUNDLE_FILE, + Iterables.transform(compiledResources, CompiledResourceFile.TO_BUNDLED)) + .addAll(XCASSETS_DIR, + uniqueContainers(attributes.assetCatalogs(), ASSET_CATALOG_CONTAINER_TYPE)) + .addAll(ASSET_CATALOG, attributes.assetCatalogs()) + .addAll(XCDATAMODEL, datamodels); + } else { + storyboards = Storyboards.empty(); + datamodels = ImmutableList.of(); + } + + for (CompilationArtifacts artifacts : compilationArtifacts.asSet()) { + objcProvider.addAll(LIBRARY, artifacts.getArchive().asSet()); + + boolean usesCpp = false; + for (Artifact sourceFile : + Iterables.concat(artifacts.getSrcs(), artifacts.getNonArcSrcs())) { + usesCpp = usesCpp || ObjcRuleClasses.CPP_SOURCES.matches(sourceFile.getExecPath()); + } + if (usesCpp) { + objcProvider.add(FLAG, USES_CPP); + } + } + + if (alwayslink) { + for (CompilationArtifacts artifacts : compilationArtifacts.asSet()) { + for (Artifact archive : artifacts.getArchive().asSet()) { + objcProvider.add(FORCE_LOAD_LIBRARY, archive); + objcProvider.add(FORCE_LOAD_FOR_XCODEGEN, + "$(BUILT_PRODUCTS_DIR)/" + archive.getExecPath().getBaseName()); + } + } + for (Artifact archive : extraImportLibraries) { + objcProvider.add(FORCE_LOAD_LIBRARY, archive); + objcProvider.add(FORCE_LOAD_FOR_XCODEGEN, + "$(WORKSPACE_ROOT)/" + archive.getExecPath().getSafePathString()); + } + } + + objcProvider.addAll(LINKED_BINARY, linkedBinary.asSet()); + + return new ObjcCommon( + context, objcProvider.build(), storyboards, datamodels, compilationArtifacts); + } + + } + + static final FileType BUNDLE_CONTAINER_TYPE = FileType.of(".bundle"); + + static final FileType ASSET_CATALOG_CONTAINER_TYPE = FileType.of(".xcassets"); + + static final FileType FRAMEWORK_CONTAINER_TYPE = FileType.of(".framework"); + private final RuleContext context; + private final ObjcProvider objcProvider; + private final Storyboards storyboards; + private final Iterable<Xcdatamodel> datamodels; + + private final Optional<CompilationArtifacts> compilationArtifacts; + + private ObjcCommon( + RuleContext context, + ObjcProvider objcProvider, + Storyboards storyboards, + Iterable<Xcdatamodel> datamodels, + Optional<CompilationArtifacts> compilationArtifacts) { + this.context = Preconditions.checkNotNull(context); + this.objcProvider = Preconditions.checkNotNull(objcProvider); + this.storyboards = Preconditions.checkNotNull(storyboards); + this.datamodels = Preconditions.checkNotNull(datamodels); + this.compilationArtifacts = Preconditions.checkNotNull(compilationArtifacts); + } + + public ObjcProvider getObjcProvider() { + return objcProvider; + } + + public Optional<CompilationArtifacts> getCompilationArtifacts() { + return compilationArtifacts; + } + + /** + * Returns all storyboards declared in this rule (not including others in the transitive + * dependency tree). + */ + public Storyboards getStoryboards() { + return storyboards; + } + + /** + * Returns all datamodels declared in this rule (not including others in the transitive + * dependency tree). + */ + public Iterable<Xcdatamodel> getDatamodels() { + return datamodels; + } + + /** + * Returns an {@link Optional} containing the compiled {@code .a} file, or + * {@link Optional#absent()} if this object contains no {@link CompilationArtifacts} or the + * compilation information has no sources. + */ + public Optional<Artifact> getCompiledArchive() { + for (CompilationArtifacts justCompilationArtifacts : compilationArtifacts.asSet()) { + return justCompilationArtifacts.getArchive(); + } + return Optional.absent(); + } + + /** + * Reports any known errors to the {@link RuleContext}. This should be called exactly once for + * a target. + */ + public void reportErrors() { + + // TODO(bazel-team): Report errors for rules that are not actually useful (i.e. objc_library + // without sources or resources, empty objc_bundles) + } + + static ImmutableList<PathFragment> userHeaderSearchPaths(BuildConfiguration configuration) { + return ImmutableList.of( + new PathFragment("."), + configuration.getGenfilesFragment()); + } + + /** + * Returns the first directory in the sequence of parents of the exec path of the given artifact + * that matches {@code type}. For instance, if {@code type} is FileType.of(".foo") and the exec + * path of {@code artifact} is {@code a/b/c/bar.foo/d/e}, then the return value is + * {@code a/b/c/bar.foo}. + */ + static Optional<PathFragment> nearestContainerMatching(FileType type, Artifact artifact) { + PathFragment container = artifact.getExecPath(); + do { + if (type.matches(container)) { + return Optional.of(container); + } + container = container.getParentDirectory(); + } while (container != null); + return Optional.absent(); + } + + /** + * Similar to {@link #nearestContainerMatching(FileType, Artifact)}, but tries matching several + * file types in {@code types}, and returns a path for the first match in the sequence. + */ + static Optional<PathFragment> nearestContainerMatching( + Iterable<FileType> types, Artifact artifact) { + for (FileType type : types) { + for (PathFragment container : nearestContainerMatching(type, artifact).asSet()) { + return Optional.of(container); + } + } + return Optional.absent(); + } + + /** + * Returns all directories matching {@code containerType} that contain the items in + * {@code artifacts}. This function ignores artifacts that are not in any directory matching + * {@code containerType}. + */ + static Iterable<PathFragment> uniqueContainers( + Iterable<Artifact> artifacts, FileType containerType) { + ImmutableSet.Builder<PathFragment> containers = new ImmutableSet.Builder<>(); + for (Artifact artifact : artifacts) { + containers.addAll(ObjcCommon.nearestContainerMatching(containerType, artifact).asSet()); + } + return containers.build(); + } + + /** + * Similar to {@link #nearestContainerMatching(FileType, Artifact)}, but returns the container + * closest to the root that matches the given type. + */ + static Optional<PathFragment> farthestContainerMatching(FileType type, Artifact artifact) { + PathFragment container = artifact.getExecPath(); + Optional<PathFragment> lastMatch = Optional.absent(); + do { + if (type.matches(container)) { + lastMatch = Optional.of(container); + } + container = container.getParentDirectory(); + } while (container != null); + return lastMatch; + } + + static Iterable<String> notInContainerErrors( + Iterable<Artifact> artifacts, FileType containerType) { + return notInContainerErrors(artifacts, ImmutableList.of(containerType)); + } + + @VisibleForTesting + static final String NOT_IN_CONTAINER_ERROR_FORMAT = + "File '%s' is not in a directory of one of these type(s): %s"; + + static Iterable<String> notInContainerErrors( + Iterable<Artifact> artifacts, Iterable<FileType> containerTypes) { + Set<String> errors = new HashSet<>(); + for (Artifact artifact : artifacts) { + boolean inContainer = nearestContainerMatching(containerTypes, artifact).isPresent(); + if (!inContainer) { + errors.add(String.format(NOT_IN_CONTAINER_ERROR_FORMAT, + artifact.getExecPath(), Iterables.toString(containerTypes))); + } + } + return errors; + } + + /** + * @param filesToBuild files to build for this target. These also become the data runfiles. Note + * that this method may add more files to create the complete list of files to build for this + * target. + * @param maybeTargetProvider the provider for this target. + * @param maybeExportedProvider the {@link ObjcProvider} for this target. This should generally be + * present whenever {@code objc_} rules may depend on this target. + * @param maybeJ2ObjcSrcsProvider the {@link J2ObjcSrcsProvider} for this target. + */ + public ConfiguredTarget configuredTarget(NestedSet<Artifact> filesToBuild, + Optional<XcodeProvider> maybeTargetProvider, Optional<ObjcProvider> maybeExportedProvider, + Optional<XcTestAppProvider> maybeXcTestAppProvider, + Optional<J2ObjcSrcsProvider> maybeJ2ObjcSrcsProvider) { + NestedSet<Artifact> allFilesToBuild = NestedSetBuilder.<Artifact>stableOrder() + .addTransitive(filesToBuild) + .addTransitive(storyboards.getOutputZips()) + .addAll(Xcdatamodel.outputZips(datamodels)) + .build(); + + RunfilesProvider runfilesProvider = RunfilesProvider.withData( + new Runfiles.Builder() + .addRunfiles(context, RunfilesProvider.DEFAULT_RUNFILES) + .build(), + new Runfiles.Builder().addTransitiveArtifacts(allFilesToBuild).build()); + + RuleConfiguredTargetBuilder target = new RuleConfiguredTargetBuilder(context) + .setFilesToBuild(allFilesToBuild) + .add(RunfilesProvider.class, runfilesProvider); + for (ObjcProvider exportedProvider : maybeExportedProvider.asSet()) { + target.addProvider(ObjcProvider.class, exportedProvider); + } + for (XcTestAppProvider xcTestAppProvider : maybeXcTestAppProvider.asSet()) { + target.addProvider(XcTestAppProvider.class, xcTestAppProvider); + } + for (XcodeProvider targetProvider : maybeTargetProvider.asSet()) { + target.addProvider(XcodeProvider.class, targetProvider); + } + for (J2ObjcSrcsProvider j2ObjcSrcsProvider : maybeJ2ObjcSrcsProvider.asSet()) { + target.addProvider(J2ObjcSrcsProvider.class, j2ObjcSrcsProvider); + } + return target.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfiguration.java new file mode 100644 index 0000000..3f2e073 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfiguration.java
@@ -0,0 +1,136 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.CompilationMode; +import com.google.devtools.build.xcode.common.Platform; + +import java.util.List; + +/** + * A compiler configuration containing flags required for Objective-C compilation. + */ +public class ObjcConfiguration extends BuildConfiguration.Fragment { + @VisibleForTesting + static final ImmutableList<String> DBG_COPTS = ImmutableList.of("-O0", "-DDEBUG=1", + "-fstack-protector", "-fstack-protector-all", "-D_GLIBCXX_DEBUG_PEDANTIC", "-D_GLIBCXX_DEBUG", + "-D_GLIBCPP_CONCEPT_CHECKS"); + + @VisibleForTesting + static final ImmutableList<String> FASTBUILD_COPTS = ImmutableList.of("-O0", "-DDEBUG=1"); + + @VisibleForTesting + static final ImmutableList<String> OPT_COPTS = + ImmutableList.of("-Os", "-DNDEBUG=1", "-Wno-unused-variable", "-Winit-self", "-Wno-extra"); + + private final String iosSdkVersion; + private final String iosMinimumOs; + private final String iosSimulatorVersion; + private final String iosCpu; + private final String xcodeOptions; + private final boolean generateDebugSymbols; + private final List<String> copts; + private final CompilationMode compilationMode; + + ObjcConfiguration(ObjcCommandLineOptions objcOptions, BuildConfiguration.Options options) { + this.iosSdkVersion = Preconditions.checkNotNull(objcOptions.iosSdkVersion, "iosSdkVersion"); + this.iosMinimumOs = Preconditions.checkNotNull(objcOptions.iosMinimumOs, "iosMinimumOs"); + this.iosSimulatorVersion = + Preconditions.checkNotNull(objcOptions.iosSimulatorVersion, "iosSimulatorVersion"); + this.iosCpu = Preconditions.checkNotNull(objcOptions.iosCpu, "iosCpu"); + this.xcodeOptions = Preconditions.checkNotNull(objcOptions.xcodeOptions, "xcodeOptions"); + this.generateDebugSymbols = objcOptions.generateDebugSymbols; + this.copts = ImmutableList.copyOf(objcOptions.copts); + this.compilationMode = Preconditions.checkNotNull(options.compilationMode, "compilationMode"); + } + + public String getIosSdkVersion() { + return iosSdkVersion; + } + + /** + * Returns the minimum iOS version supported by binaries and libraries. Any dependencies on newer + * iOS version features or libraries will become weak dependencies which are only loaded if the + * runtime OS supports them. + */ + public String getMinimumOs() { + return iosMinimumOs; + } + + public String getIosSimulatorVersion() { + return iosSimulatorVersion; + } + + public String getIosCpu() { + return iosCpu; + } + + public Platform getPlatform() { + return Platform.forArch(getIosCpu()); + } + + public String getXcodeOptions() { + return xcodeOptions; + } + + public boolean generateDebugSymbols() { + return generateDebugSymbols; + } + + /** + * Returns the current compilation mode. + */ + public CompilationMode getCompilationMode() { + return compilationMode; + } + + /** + * Returns the default set of clang options for the current compilation mode. + */ + public List<String> getCoptsForCompilationMode() { + switch (compilationMode) { + case DBG: + return DBG_COPTS; + case FASTBUILD: + return FASTBUILD_COPTS; + case OPT: + return OPT_COPTS; + default: + throw new AssertionError(); + } + } + + /** + * Returns options passed to (Apple) clang when compiling Objective C. These options should be + * applied after any default options but before options specified in the attributes of the rule. + */ + public List<String> getCopts() { + return copts; + } + + @Override + public String getName() { + return "Objective-C"; + } + + @Override + public String cacheKey() { + return iosSdkVersion; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfigurationLoader.java new file mode 100644 index 0000000..19713a3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcConfigurationLoader.java
@@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; + +/** + * A loader that creates ObjcConfiguration instances based on Objective-C configurations and + * command-line options. + */ +public class ObjcConfigurationLoader implements ConfigurationFragmentFactory { + @Override + public ObjcConfiguration create(ConfigurationEnvironment env, BuildOptions buildOptions) + throws InvalidConfigurationException { + return new ObjcConfiguration(buildOptions.get(ObjcCommandLineOptions.class), + buildOptions.get(BuildConfiguration.Options.class)); + } + + @Override + public Class<? extends BuildConfiguration.Fragment> creates() { + return ObjcConfiguration.class; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFramework.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFramework.java new file mode 100644 index 0000000..c6a0037 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFramework.java
@@ -0,0 +1,60 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.collect.nestedset.Order.STABLE_ORDER; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.objc.ObjcSdkFrameworks.Attributes; + +/** + * Implementation for the {@code objc_framework} rule. + */ +public class ObjcFramework implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + Attributes sdkFrameworkAttributes = new Attributes(ruleContext); + + ImmutableList<Artifact> frameworkImports = + ruleContext.getPrerequisiteArtifacts("framework_imports", Mode.TARGET).list(); + ObjcCommon common = new ObjcCommon.Builder(ruleContext) + .addFrameworkImports( + frameworkImports) + .addExtraSdkFrameworks(sdkFrameworkAttributes.sdkFrameworks()) + .addExtraWeakSdkFrameworks(sdkFrameworkAttributes.weakSdkFrameworks()) + .addExtraSdkDylibs(sdkFrameworkAttributes.sdkDylibs()) + .build(); + + Iterable<String> containerErrors = + ObjcCommon.notInContainerErrors(frameworkImports, ObjcCommon.FRAMEWORK_CONTAINER_TYPE); + for (String error : containerErrors) { + ruleContext.attributeError("framework_imports", error); + } + + return common.configuredTarget( + NestedSetBuilder.<Artifact>emptySet(STABLE_ORDER) /* filesToBuild */, + Optional.<XcodeProvider>absent(), + Optional.of(common.getObjcProvider()), + Optional.<XcTestAppProvider>absent(), + Optional.<J2ObjcSrcsProvider>absent()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFrameworkRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFrameworkRule.java new file mode 100644 index 0000000..7fcfdd3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcFrameworkRule.java
@@ -0,0 +1,62 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcSdkFrameworksRule; +import com.google.devtools.build.lib.util.FileTypeSet; + +/** + * Rule definition for objc_framework. + */ +@BlazeRule(name = "objc_framework", + factoryClass = ObjcFramework.class, + ancestors = { BaseRuleClasses.BaseRule.class, ObjcSdkFrameworksRule.class}) +public class ObjcFrameworkRule implements RuleDefinition { + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE(objc_framework).ATTRIBUTE(framework_imports) --> + The list of files under a <code>.framework</code> directory which are + provided to Objective-C targets that depend on this target. + <i>(List of <a href="build-ref.html#labels">labels</a>; required)</i> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("framework_imports", LABEL_LIST) + .allowedFileTypes(FileTypeSet.ANY_FILE) + .mandatory() + .nonEmpty()) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_framework, TYPE = LIBRARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule encapsulates an already-built framework. It is defined by a list +of files in one or more <code>.framework</code> directories. + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImport.java new file mode 100644 index 0000000..70743ed --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImport.java
@@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.XcodeProductType.LIBRARY_STATIC; + +import com.google.common.base.Optional; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.CompilationAttributes; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.ResourceAttributes; + +/** + * Implementation for {@code objc_import}. + */ +public class ObjcImport implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + ObjcCommon common = new ObjcCommon.Builder(ruleContext) + .setCompilationAttributes(new CompilationAttributes(ruleContext)) + .setResourceAttributes(new ResourceAttributes(ruleContext)) + .setIntermediateArtifacts(ObjcRuleClasses.intermediateArtifacts(ruleContext)) + .setAlwayslink(ruleContext.attributes().get("alwayslink", Type.BOOLEAN)) + .addExtraImportLibraries( + ruleContext.getPrerequisiteArtifacts("archives", Mode.TARGET).list()) + .build(); + + XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder(); + NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.stableOrder(); + + new CompilationSupport(ruleContext) + .addXcodeSettings(xcodeProviderBuilder, common, OptionsProvider.DEFAULT) + .validateAttributes(); + + new ResourceSupport(ruleContext) + .registerActions(common.getStoryboards()) + .validateAttributes() + .addXcodeSettings(xcodeProviderBuilder); + + new XcodeSupport(ruleContext) + .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), LIBRARY_STATIC) + .registerActions(xcodeProviderBuilder.build()) + .addFilesToBuild(filesToBuild); + + return common.configuredTarget( + filesToBuild.build(), + Optional.of(xcodeProviderBuilder.build()), + Optional.of(common.getObjcProvider()), + Optional.<XcTestAppProvider>absent(), + Optional.<J2ObjcSrcsProvider>absent()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImportRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImportRule.java new file mode 100644 index 0000000..24e9412 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcImportRule.java
@@ -0,0 +1,81 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcCompilationRule; +import com.google.devtools.build.lib.util.FileType; + +/** + * Rule definition for {@code objc_import}. + */ +@BlazeRule(name = "objc_import", + factoryClass = ObjcImport.class, + ancestors = { ObjcCompilationRule.class }) +public class ObjcImportRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /*<!-- #BLAZE_RULE(objc_import).IMPLICIT_OUTPUTS --> + <ul> + <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: An Xcode project file which + can be used to develop or build on a Mac.</li> + </ul> + <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/ + .setImplicitOutputsFunction(XcodeSupport.PBXPROJ) + /* <!-- #BLAZE_RULE(objc_import).ATTRIBUTE(archives) --> + The list of <code>.a</code> files provided to Objective-C targets that + depend on this target. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("archives", LABEL_LIST) + .mandatory() + .nonEmpty() + .allowedFileTypes(FileType.of(".a"))) + /* <!-- #BLAZE_RULE(objc_import).ATTRIBUTE(alwayslink) --> + If 1, any bundle or binary that depends (directly or indirectly) on this + library will link in all the archive files listed in + <code>archives</code>, even if some contain no symbols referenced by the + binary. + ${SYNOPSIS} + This is useful if your code isn't explicitly called by code in + the binary, e.g., if your code registers to receive some callback + provided by some service. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("alwayslink", BOOLEAN)) + .removeAttribute("deps") + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_import, TYPE = LIBRARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule encapsulates an already-compiled static library in the form of an +<code>.a</code> file. It also allows exporting headers and resources using the same +attributes supported by <code>objc_library</code>.</p> + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibrary.java new file mode 100644 index 0000000..4136ffe --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibrary.java
@@ -0,0 +1,132 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.XcodeProductType.LIBRARY_STATIC; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.CompilationAttributes; +import com.google.devtools.build.lib.rules.objc.ObjcCommon.ResourceAttributes; + +/** + * Implementation for {@code objc_library}. + */ +public class ObjcLibrary implements RuleConfiguredTargetFactory { + + /** + * An {@link IterableWrapper} containing extra library {@link Artifact}s to be linked into the + * final ObjC application bundle. + */ + static final class ExtraImportLibraries extends IterableWrapper<Artifact> { + ExtraImportLibraries(Artifact... extraImportLibraries) { + super(extraImportLibraries); + } + } + + /** + * An {@link IterableWrapper} containing defines as specified in the {@code defines} attribute to + * be applied to this target and all depending targets' compilation actions. + */ + static final class Defines extends IterableWrapper<String> { + Defines(Iterable<String> defines) { + super(defines); + } + + Defines(String... defines) { + super(defines); + } + } + + /** + * Constructs an {@link ObjcCommon} instance based on the attributes of the given rule. The rule + * should inherit from {@link ObjcLibraryRule}.. + */ + static ObjcCommon common(RuleContext ruleContext, Iterable<SdkFramework> extraSdkFrameworks, + boolean alwayslink, ExtraImportLibraries extraImportLibraries, Defines defines, + Iterable<ObjcProvider> extraDepObjcProviders) { + CompilationArtifacts compilationArtifacts = + CompilationSupport.compilationArtifacts(ruleContext); + + return new ObjcCommon.Builder(ruleContext) + .setCompilationAttributes(new CompilationAttributes(ruleContext)) + .setResourceAttributes(new ResourceAttributes(ruleContext)) + .addExtraSdkFrameworks(extraSdkFrameworks) + .addDefines(defines) + .setCompilationArtifacts(compilationArtifacts) + .addDepObjcProviders(ruleContext.getPrerequisites("deps", Mode.TARGET, ObjcProvider.class)) + .addDepObjcProviders( + ruleContext.getPrerequisites("bundles", Mode.TARGET, ObjcProvider.class)) + .addNonPropagatedDepObjcProviders(ruleContext.getPrerequisites("non_propagated_deps", + Mode.TARGET, ObjcProvider.class)) + .setIntermediateArtifacts(ObjcRuleClasses.intermediateArtifacts(ruleContext)) + .setAlwayslink(alwayslink) + .addExtraImportLibraries(extraImportLibraries) + .addDepObjcProviders(extraDepObjcProviders) + .build(); + } + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + ObjcCommon common = common( + ruleContext, ImmutableList.<SdkFramework>of(), + ruleContext.attributes().get("alwayslink", Type.BOOLEAN), new ExtraImportLibraries(), + new Defines(ruleContext.getTokenizedStringListAttr("defines")), + ImmutableList.<ObjcProvider>of()); + OptionsProvider optionsProvider = optionsProvider(ruleContext); + + XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder(); + NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.<Artifact>stableOrder() + .addAll(common.getCompiledArchive().asSet()); + + new CompilationSupport(ruleContext) + .registerCompileAndArchiveActions(common, optionsProvider) + .addXcodeSettings(xcodeProviderBuilder, common, optionsProvider) + .validateAttributes(); + + new ResourceSupport(ruleContext) + .registerActions(common.getStoryboards()) + .validateAttributes() + .addXcodeSettings(xcodeProviderBuilder); + + new XcodeSupport(ruleContext) + .addFilesToBuild(filesToBuild) + .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), LIBRARY_STATIC) + .addDependencies(xcodeProviderBuilder) + .registerActions(xcodeProviderBuilder.build()); + + return common.configuredTarget( + filesToBuild.build(), + Optional.of(xcodeProviderBuilder.build()), + Optional.of(common.getObjcProvider()), + Optional.<XcTestAppProvider>absent(), + Optional.of(ObjcRuleClasses.j2ObjcSrcsProvider(ruleContext))); + } + + private OptionsProvider optionsProvider(RuleContext ruleContext) { + return new OptionsProvider.Builder() + .addCopts(ruleContext.getTokenizedStringListAttr("copts")) + .addTransitive(Optional.fromNullable( + ruleContext.getPrerequisite("options", Mode.TARGET, OptionsProvider.class))) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryRule.java new file mode 100644 index 0000000..f721492 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryRule.java
@@ -0,0 +1,157 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; +import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.NON_ARC_SRCS_TYPE; +import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.SRCS_TYPE; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcCompilationRule; +import com.google.devtools.build.lib.util.FileType; + +/** + * Rule definition for objc_library. + */ +@BlazeRule(name = "objc_library", + factoryClass = ObjcLibrary.class, + ancestors = { ObjcCompilationRule.class, + ObjcRuleClasses.ObjcOptsRule.class }) +public class ObjcLibraryRule implements RuleDefinition { + private static final Iterable<String> ALLOWED_DEPS_RULE_CLASSES = ImmutableSet.of( + "objc_library", + "objc_import", + "objc_bundle", + "objc_framework", + "objc_bundle_library", + "objc_proto_library", + "j2objc_library"); + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /*<!-- #BLAZE_RULE(objc_library).IMPLICIT_OUTPUTS --> + <ul> + <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: An Xcode project file which + can be used to develop or build on a Mac.</li> + </ul> + <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/ + .setImplicitOutputsFunction(XcodeSupport.PBXPROJ) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(srcs) --> + The list of C, C++, Objective-C, and Objective-C++ files that are + processed to create the library target. + ${SYNOPSIS} + These are your checked-in source files, plus any generated files. + These are compiled into .o files with Clang, so headers should not go + here (see the hdrs attribute). + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("srcs", LABEL_LIST) + .direct_compile_time_input() + .allowedFileTypes(SRCS_TYPE)) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(non_arc_srcs) --> + The list of Objective-C files that are processed to create the + library target that DO NOT use ARC. + ${SYNOPSIS} + The files in this attribute are treated very similar to those in the + srcs attribute, but are compiled without ARC enabled. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("non_arc_srcs", LABEL_LIST) + .direct_compile_time_input() + .allowedFileTypes(NON_ARC_SRCS_TYPE)) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(pch) --> + Header file to prepend to every source file being compiled (both arc + and non-arc). Note that the file will not be precompiled - this is + simply a convenience, not a build-speed enhancement. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("pch", LABEL) + .direct_compile_time_input() + .allowedFileTypes(FileType.of(".pch"))) + .add(attr("options", LABEL) + .undocumented("objc_options will probably be removed") + .allowedFileTypes() + .allowedRuleClasses("objc_options")) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(alwayslink) --> + If 1, any bundle or binary that depends (directly or indirectly) on this + library will link in all the object files for the files listed in + <code>srcs</code> and <code>non_arc_srcs</code>, even if some contain no + symbols referenced by the binary. + ${SYNOPSIS} + This is useful if your code isn't explicitly called by code in + the binary, e.g., if your code registers to receive some callback + provided by some service. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("alwayslink", BOOLEAN)) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(deps) --> + The list of targets that are linked together to form the final bundle. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .override(attr("deps", LABEL_LIST) + .direct_compile_time_input() + .allowedRuleClasses(ALLOWED_DEPS_RULE_CLASSES) + .allowedFileTypes()) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(bundles) --> + The list of bundle targets that this target requires to be included in the final bundle. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("bundles", LABEL_LIST) + .direct_compile_time_input() + .allowedRuleClasses("objc_bundle", "objc_bundle_library") + .allowedFileTypes()) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(non_propagated_deps) --> + The list of targets that are required in order to build this target, + but which are not included in the final bundle. + <br /> + This attribute should only rarely be used, and probably only for proto + dependencies. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("non_propagated_deps", LABEL_LIST) + .direct_compile_time_input() + .allowedRuleClasses(ALLOWED_DEPS_RULE_CLASSES) + .allowedFileTypes()) + /* <!-- #BLAZE_RULE(objc_library).ATTRIBUTE(defines) --> + Extra <code>-D</code> flags to pass to the compiler. They should be in + the form <code>KEY=VALUE</code> or simply <code>KEY</code> and are + passed not only the compiler for this target (as <code>copts</code> + are) but also to all <code>objc_</code> dependers of this target. + ${SYNOPSIS} + Subject to <a href="#make_variables">"Make variable"</a> substitution and + <a href="#sh-tokenization">Bourne shell tokenization</a>. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("defines", STRING_LIST)) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_library, TYPE = LIBRARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule produces a static library from the given Objective-C source files.</p> + +${IMPLICIT_OUTPUTS} + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptions.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptions.java new file mode 100644 index 0000000..a7e2b8f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptions.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * Implementation for the {@code objc_options} rule. + */ +public class ObjcOptions implements RuleConfiguredTargetFactory { + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + return new RuleConfiguredTargetBuilder(ruleContext) + .add(RunfilesProvider.class, RunfilesProvider.EMPTY) + .add(OptionsProvider.class, + new OptionsProvider.Builder() + .addCopts(ruleContext.getTokenizedStringListAttr("copts")) + .addInfoplists( + ruleContext.getPrerequisiteArtifacts("infoplists", Mode.TARGET).list()) + .build()) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptionsRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptionsRule.java new file mode 100644 index 0000000..7f26bfe --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcOptionsRule.java
@@ -0,0 +1,67 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.PLIST_TYPE; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses.BaseRule; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcOptsRule; + +/** + * Rule definition for {@code objc_options}. + */ +@BlazeRule(name = "objc_options", + factoryClass = ObjcOptions.class, + ancestors = { BaseRule.class, ObjcOptsRule.class }) +public class ObjcOptionsRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + // TODO(bazel-team): Figure out if we really need objc_options, and if + // we don't, delete it. + .setUndocumented() + /* <!-- #BLAZE_RULE(objc_options).ATTRIBUTE(xcode_name)[DEPRECATED] --> + This attribute is ignored and will be removed. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("xcode_name", Type.STRING)) + /* <!-- #BLAZE_RULE(objc_options).ATTRIBUTE(infoplists) --> + infoplist files to merge with the final binary's infoplist. This + corresponds to a single file <i>appname</i>-Info.plist in Xcode + projects. + <i>(List of <a href="build-ref.html#labels">labels</a>; optional)</i> + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("infoplists", Type.LABEL_LIST) + .allowedFileTypes(PLIST_TYPE)) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_options, TYPE = OTHER, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule provides a nameable set of build settings to use when building +Objective-C targets.</p> + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibrary.java new file mode 100644 index 0000000..647b221 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibrary.java
@@ -0,0 +1,242 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.common.base.CaseFormat.LOWER_UNDERSCORE; +import static com.google.common.base.CaseFormat.UPPER_CAMEL; +import static com.google.devtools.build.lib.rules.objc.XcodeProductType.LIBRARY_STATIC; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.proto.ProtoSourcesProvider; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import javax.annotation.Nullable; + +/** + * Implementation for the "objc_proto_library" rule. + */ +public class ObjcProtoLibrary implements RuleConfiguredTargetFactory { + private static final Function<Artifact, PathFragment> PARENT_PATHFRAGMENT = + new Function<Artifact, PathFragment>() { + @Override + public PathFragment apply(Artifact input) { + return input.getExecPath().getParentDirectory(); + } + }; + + @VisibleForTesting + static final String NO_PROTOS_ERROR = + "no protos to compile - a non-empty deps attribute is required"; + + @Override + public ConfiguredTarget create(final RuleContext ruleContext) throws InterruptedException { + Artifact compileProtos = ruleContext.getPrerequisiteArtifact( + ObjcRuleClasses.ObjcProtoRule.COMPILE_PROTOS_ATTR, Mode.HOST); + Optional<Artifact> optionsFile = Optional.fromNullable( + ruleContext.getPrerequisiteArtifact(ObjcProtoLibraryRule.OPTIONS_FILE_ATTR, Mode.HOST)); + NestedSet<Artifact> protos = NestedSetBuilder.<Artifact>stableOrder() + .addAll(ruleContext.getPrerequisiteArtifacts("deps", Mode.TARGET) + .filter(FileType.of(".proto")) + .list()) + .addTransitive(maybeGetProtoSources(ruleContext)) + .build(); + + if (Iterables.isEmpty(protos)) { + ruleContext.ruleError(NO_PROTOS_ERROR); + } + + ImmutableList<Artifact> libProtobuf = ruleContext + .getPrerequisiteArtifacts(ObjcProtoLibraryRule.LIBPROTOBUF_ATTR, Mode.TARGET) + .list(); + ImmutableList<Artifact> protoSupport = ruleContext + .getPrerequisiteArtifacts(ObjcRuleClasses.ObjcProtoRule.PROTO_SUPPORT_ATTR, Mode.HOST) + .list(); + + // Generate sources in a package-and-rule-scoped directory; adds both the + // package-and-rule-scoped directory and the header-containing-directory to the include path of + // dependers. + PathFragment rootRelativeOutputDir = new PathFragment( + ruleContext.getLabel().getPackageFragment(), + new PathFragment("_generated_protos_" + ruleContext.getLabel().getName())); + PathFragment workspaceRelativeOutputDir = new PathFragment( + ruleContext.getBinOrGenfilesDirectory().getExecPath(), rootRelativeOutputDir); + PathFragment generatedProtoDir = + new PathFragment(workspaceRelativeOutputDir, ruleContext.getLabel().getPackageFragment()); + + boolean outputCpp = + ruleContext.attributes().get(ObjcProtoLibraryRule.OUTPUT_CPP_ATTR, Type.BOOLEAN); + + ImmutableList<Artifact> protoGeneratedSources = outputArtifacts( + ruleContext, rootRelativeOutputDir, protos, FileType.of(".pb." + (outputCpp ? "cc" : "m")), + outputCpp); + ImmutableList<Artifact> protoGeneratedHeaders = outputArtifacts( + ruleContext, rootRelativeOutputDir, protos, FileType.of(".pb.h"), outputCpp); + + Artifact inputFileList = ruleContext.getAnalysisEnvironment().getDerivedArtifact( + AnalysisUtils.getUniqueDirectory(ruleContext.getLabel(), new PathFragment("_protos")) + .getRelative("_proto_input_files"), + ruleContext.getConfiguration().getGenfilesDirectory()); + + ruleContext.registerAction(new FileWriteAction( + ruleContext.getActionOwner(), + inputFileList, + ObjcActionsBuilder.joinExecPaths(protos), + false)); + + CustomCommandLine.Builder commandLineBuilder = new CustomCommandLine.Builder() + .add(compileProtos.getExecPathString()) + .add("--input-file-list").add(inputFileList.getExecPathString()) + .add("--output-dir").add(workspaceRelativeOutputDir.getSafePathString()); + if (optionsFile.isPresent()) { + commandLineBuilder + .add("--compiler-options-path") + .add(optionsFile.get().getExecPathString()); + } + if (outputCpp) { + commandLineBuilder.add("--generate-cpp"); + } + + if (!Iterables.isEmpty(protos)) { + ruleContext.registerAction(new SpawnAction.Builder() + .setMnemonic("GenObjcProtos") + .addInput(compileProtos) + .addInputs(optionsFile.asSet()) + .addInputs(protos) + .addInput(inputFileList) + .addInputs(libProtobuf) + .addInputs(protoSupport) + .addOutputs(Iterables.concat(protoGeneratedSources, protoGeneratedHeaders)) + .setExecutable(new PathFragment("/usr/bin/python")) + .setCommandLine(commandLineBuilder.build()) + .setExecutionInfo(ImmutableMap.of(ExecutionRequirements.REQUIRES_DARWIN, "")) + .build(ruleContext)); + } + + IntermediateArtifacts intermediateArtifacts = + ObjcRuleClasses.intermediateArtifacts(ruleContext); + CompilationArtifacts compilationArtifacts = new CompilationArtifacts.Builder() + .addNonArcSrcs(protoGeneratedSources) + .setIntermediateArtifacts(intermediateArtifacts) + .setPchFile(Optional.<Artifact>absent()) + .build(); + + ImmutableSet<PathFragment> searchPathEntries = new ImmutableSet.Builder<PathFragment>() + .add(workspaceRelativeOutputDir) + .add(generatedProtoDir) + .addAll(Iterables.transform(protoGeneratedHeaders, PARENT_PATHFRAGMENT)) + .build(); + ObjcCommon common = new ObjcCommon.Builder(ruleContext) + .setCompilationArtifacts(compilationArtifacts) + .addUserHeaderSearchPaths(searchPathEntries) + .addDepObjcProviders(ruleContext.getPrerequisites( + ObjcProtoLibraryRule.LIBPROTOBUF_ATTR, Mode.TARGET, ObjcProvider.class)) + .setIntermediateArtifacts(intermediateArtifacts) + .addHeaders(protoGeneratedHeaders) + .addHeaders(protoGeneratedSources) + .build(); + + XcodeProvider xcodeProvider = new XcodeProvider.Builder() + .setLabel(ruleContext.getLabel()) + .addUserHeaderSearchPaths(searchPathEntries) + .addDependencies(ruleContext.getPrerequisites( + ObjcProtoLibraryRule.LIBPROTOBUF_ATTR, Mode.TARGET, XcodeProvider.class)) + .addCopts(ObjcRuleClasses.objcConfiguration(ruleContext).getCopts()) + .setProductType(LIBRARY_STATIC) + .addHeaders(protoGeneratedHeaders) + .setCompilationArtifacts(common.getCompilationArtifacts().get()) + .setObjcProvider(common.getObjcProvider()) + .build(); + + ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext); + actionsBuilder + .registerCompileAndArchiveActions( + compilationArtifacts, common.getObjcProvider(), OptionsProvider.DEFAULT); + actionsBuilder.registerXcodegenActions( + new ObjcRuleClasses.Tools(ruleContext), + ruleContext.getImplicitOutputArtifact(XcodeSupport.PBXPROJ), + XcodeProvider.Project.fromTopLevelTarget(xcodeProvider)); + + return common.configuredTarget( + NestedSetBuilder.<Artifact>stableOrder() + .addAll(common.getCompiledArchive().asSet()) + .addAll(protoGeneratedSources) + .addAll(protoGeneratedHeaders) + .add(ruleContext.getImplicitOutputArtifact(XcodeSupport.PBXPROJ)) + .build(), + Optional.of(xcodeProvider), + Optional.of(common.getObjcProvider()), + Optional.<XcTestAppProvider>absent(), + Optional.<J2ObjcSrcsProvider>absent()); + } + + private NestedSet<Artifact> maybeGetProtoSources(RuleContext ruleContext) { + NestedSetBuilder<Artifact> artifacts = new NestedSetBuilder<>(Order.STABLE_ORDER); + Iterable<ProtoSourcesProvider> providers = + ruleContext.getPrerequisites("deps", Mode.TARGET, ProtoSourcesProvider.class); + for (ProtoSourcesProvider provider : providers) { + artifacts.addTransitive(provider.getTransitiveProtoSources()); + } + return artifacts.build(); + } + + private ImmutableList<Artifact> outputArtifacts(RuleContext ruleContext, + PathFragment rootRelativeOutputDir, Iterable<Artifact> protos, FileType newFileType, + boolean outputCpp) { + ImmutableList.Builder<Artifact> builder = new ImmutableList.Builder<>(); + for (Artifact proto : protos) { + String protoOutputName; + if (outputCpp) { + protoOutputName = proto.getFilename(); + } else { + String lowerUnderscoreBaseName = proto.getFilename().replace('-', '_').toLowerCase(); + protoOutputName = LOWER_UNDERSCORE.to(UPPER_CAMEL, lowerUnderscoreBaseName); + } + PathFragment rawFragment = new PathFragment( + rootRelativeOutputDir, + proto.getExecPath().getParentDirectory(), + new PathFragment(protoOutputName)); + @Nullable PathFragment outputFile = FileSystemUtils.replaceExtension( + rawFragment, + newFileType.getExtensions().get(0), + ".proto"); + if (outputFile != null) { + builder.add(ruleContext.getAnalysisEnvironment().getDerivedArtifact( + outputFile, ruleContext.getBinOrGenfilesDirectory())); + } + } + return builder.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibraryRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibraryRule.java new file mode 100644 index 0000000..a25f96e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProtoLibraryRule.java
@@ -0,0 +1,80 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for objc_proto_library. + * + * This is a temporary rule until it is better known how to support proto_library rules. + */ +@BlazeRule(name = "objc_proto_library", + factoryClass = ObjcProtoLibrary.class, + ancestors = { BaseRuleClasses.RuleBase.class, ObjcRuleClasses.ObjcProtoRule.class }) +public class ObjcProtoLibraryRule implements RuleDefinition { + static final String OPTIONS_FILE_ATTR = "options_file"; + static final String OUTPUT_CPP_ATTR = "output_cpp"; + static final String LIBPROTOBUF_ATTR = "$lib_protobuf"; + + @Override + public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE(objc_proto_library).ATTRIBUTE(deps) --> + The directly depended upon proto_library rules. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .override(attr("deps", LABEL_LIST) + .allowedRuleClasses("proto_library", "filegroup") + .legacyAllowAnyFileType()) + /* <!-- #BLAZE_RULE(objc_proto_library).ATTRIBUTE(options_file) --> + Optional options file to apply to protos which affects compilation (e.g. class + whitelist/blacklist settings). + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr(OPTIONS_FILE_ATTR, LABEL).legacyAllowAnyFileType().singleArtifact().cfg(HOST)) + /* <!-- #BLAZE_RULE(objc_proto_library).ATTRIBUTE(output_cpp) --> + If true, output C++ rather than ObjC. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr(OUTPUT_CPP_ATTR, BOOLEAN).value(false)) + // TODO(bazel-team): Use //external:objc_proto_lib when bind() support is a little better + .add(attr(LIBPROTOBUF_ATTR, LABEL).allowedRuleClasses("objc_library") + .value(env.getLabel( + "//googlemac/ThirdParty/ProtocolBuffers2/objectivec:ProtocolBuffers_lib"))) + .add(attr("$xcodegen", LABEL).cfg(HOST).exec() + .value(env.getLabel("//tools/objc:xcodegen"))) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_proto_library, TYPE = LIBRARY, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule produces a static library from the given proto_library dependencies, after applying an +options file.</p> + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProvider.java new file mode 100644 index 0000000..c48710e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcProvider.java
@@ -0,0 +1,313 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.collect.nestedset.Order.LINK_ORDER; +import static com.google.devtools.build.lib.collect.nestedset.Order.STABLE_ORDER; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.TargetControl; + +import java.util.HashMap; +import java.util.Map; + +/** + * A provider that provides all compiling and linking information in the transitive closure of its + * deps that are needed for building Objective-C rules. + */ +@Immutable +public final class ObjcProvider implements TransitiveInfoProvider { + /** + * Represents one of the things this provider can provide transitively. Things are provided as + * {@link NestedSet}s of type E. + */ + public static class Key<E> { + private final Order order; + + private Key(Order order) { + this.order = Preconditions.checkNotNull(order); + } + } + + public static final Key<Artifact> LIBRARY = new Key<>(LINK_ORDER); + public static final Key<Artifact> IMPORTED_LIBRARY = new Key<>(LINK_ORDER); + + /** + * Single-architecture linked binaries to be combined for the final multi-architecture binary. + */ + public static final Key<Artifact> LINKED_BINARY = new Key<>(STABLE_ORDER); + + /** + * Indicates which libraries to load with {@code -force_load}. This is a subset of the union of + * the {@link #LIBRARY} and {@link #IMPORTED_LIBRARY} sets. + */ + public static final Key<Artifact> FORCE_LOAD_LIBRARY = new Key<>(LINK_ORDER); + + /** + * Libraries to pass with -force_load flags when setting the linkopts in Xcodegen. This is needed + * in addition to {@link #FORCE_LOAD_LIBRARY} because that one, contains a mixture of import + * archives (which are not built by Xcode) and built-from-source library archives (which are built + * by Xcode). Archives that are built by Xcode are placed directly under + * {@code BUILT_PRODUCTS_DIR} while those not built by Xcode appear somewhere in the Bazel + * workspace under {@code WORKSPACE_ROOT}. + */ + public static final Key<String> FORCE_LOAD_FOR_XCODEGEN = new Key<>(LINK_ORDER); + + public static final Key<Artifact> HEADER = new Key<>(STABLE_ORDER); + + /** + * Include search paths specified with {@code -I} on the command line. Also known as header search + * paths (and distinct from <em>user</em> header search paths). + */ + public static final Key<PathFragment> INCLUDE = new Key<>(LINK_ORDER); + + /** + * Key for values in {@code defines} attributes. These are passed as {@code -D} flags to all + * invocations of the compiler for this target and all depending targets. + */ + public static final Key<String> DEFINE = new Key<>(STABLE_ORDER); + + public static final Key<Artifact> ASSET_CATALOG = new Key<>(STABLE_ORDER); + + /** + * Added to {@link TargetControl#getGeneralResourceFileList()} when running Xcodegen. + */ + public static final Key<Artifact> GENERAL_RESOURCE_FILE = new Key<>(STABLE_ORDER); + + /** + * Exec paths of {@code .bundle} directories corresponding to imported bundles to link. + * These are passed to Xcodegen. + */ + public static final Key<PathFragment> BUNDLE_IMPORT_DIR = new Key<>(STABLE_ORDER); + + /** + * Files that are plopped into the final bundle at some arbitrary bundle path. Note that these are + * not passed to Xcodegen, and these don't include information about where the file originated + * from. + */ + public static final Key<BundleableFile> BUNDLE_FILE = new Key<>(STABLE_ORDER); + + public static final Key<PathFragment> XCASSETS_DIR = new Key<>(STABLE_ORDER); + public static final Key<String> SDK_DYLIB = new Key<>(STABLE_ORDER); + public static final Key<SdkFramework> SDK_FRAMEWORK = new Key<>(STABLE_ORDER); + public static final Key<SdkFramework> WEAK_SDK_FRAMEWORK = new Key<>(STABLE_ORDER); + public static final Key<Xcdatamodel> XCDATAMODEL = new Key<>(STABLE_ORDER); + public static final Key<Flag> FLAG = new Key<>(STABLE_ORDER); + + /** + * Merge zips to include in the bundle. The entries of these zip files are included in the final + * bundle with the same path. The entries in the merge zips should not include the bundle root + * path (e.g. {@code Foo.app}). + */ + public static final Key<Artifact> MERGE_ZIP = new Key<>(STABLE_ORDER); + + /** + * Exec paths of {@code .framework} directories corresponding to frameworks to link. These cause + * -F arguments (framework search paths) to be added to each compile action, and -framework (link + * framework) arguments to be added to each link action. + */ + public static final Key<PathFragment> FRAMEWORK_DIR = new Key<>(LINK_ORDER); + + /** + * Files in {@code .framework} directories that should be included as inputs when compiling and + * linking. + */ + public static final Key<Artifact> FRAMEWORK_FILE = new Key<>(STABLE_ORDER); + + /** + * Bundles which should be linked in as a nested bundle to the final application. + */ + public static final Key<Bundling> NESTED_BUNDLE = new Key<>(STABLE_ORDER); + + /** + * Artifact containing information on debug symbols + */ + public static final Key<Artifact> DEBUG_SYMBOLS = new Key<>(STABLE_ORDER); + + /** + * Flags that apply to a transitive build dependency tree. Each item in the enum corresponds to a + * flag. If the item is included in the key {@link #FLAG}, then the flag is considered set. + */ + public enum Flag { + /** + * Indicates that C++ (or Objective-C++) is used in any source file. This affects how the linker + * is invoked. + */ + USES_CPP; + } + + private final ImmutableMap<Key<?>, NestedSet<?>> items; + + // Items which should be passed to direct dependers, but not transitive dependers. + private final ImmutableMap<Key<?>, NestedSet<?>> nonPropagatedItems; + + private ObjcProvider( + ImmutableMap<Key<?>, NestedSet<?>> items, + ImmutableMap<Key<?>, NestedSet<?>> nonPropagatedItems) { + this.items = Preconditions.checkNotNull(items); + this.nonPropagatedItems = Preconditions.checkNotNull(nonPropagatedItems); + } + + /** + * All artifacts, bundleable files, etc. of the type specified by {@code key}. + */ + @SuppressWarnings("unchecked") + public <E> NestedSet<E> get(Key<E> key) { + Preconditions.checkNotNull(key); + NestedSetBuilder<E> builder = new NestedSetBuilder<>(key.order); + if (nonPropagatedItems.containsKey(key)) { + builder.addTransitive((NestedSet<E>) nonPropagatedItems.get(key)); + } + if (items.containsKey(key)) { + builder.addTransitive((NestedSet<E>) items.get(key)); + } + return builder.build(); + } + + /** + * Indicates whether {@code flag} is set on this provider. + */ + public boolean is(Flag flag) { + return Iterables.contains(get(FLAG), flag); + } + + /** + * Indicates whether this provider has any asset catalogs. This is true whenever some target in + * its transitive dependency tree specifies a non-empty {@code asset_catalogs} attribute. + */ + public boolean hasAssetCatalogs() { + return !get(XCASSETS_DIR).isEmpty(); + } + + /** + * A builder for this context with an API that is optimized for collecting information from + * several transitive dependencies. + */ + public static final class Builder { + private final Map<Key<?>, NestedSetBuilder<?>> items = new HashMap<>(); + private final Map<Key<?>, NestedSetBuilder<?>> nonPropagatedItems = new HashMap<>(); + + private static void maybeAddEmptyBuilder(Map<Key<?>, NestedSetBuilder<?>> set, Key<?> key) { + if (!set.containsKey(key)) { + set.put(key, new NestedSetBuilder<>(key.order)); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private void uncheckedAddAll(Key key, Iterable toAdd) { + maybeAddEmptyBuilder(items, key); + items.get(key).addAll(toAdd); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private void uncheckedAddTransitive(Key key, NestedSet toAdd, boolean propagate) { + Map<Key<?>, NestedSetBuilder<?>> set = propagate ? items : nonPropagatedItems; + maybeAddEmptyBuilder(set, key); + set.get(key).addTransitive(toAdd); + } + + /** + * Adds elements in items, and propagate them to any (transitive) dependers on this + * ObjcProvider. + */ + public <E> Builder addTransitiveAndPropagate(Key<E> key, NestedSet<E> items) { + uncheckedAddTransitive(key, items, true); + return this; + } + + /** + * Add all elements from provider, and propagate them to any (transitive) dependers on this + * ObjcProvider. + */ + public Builder addTransitiveAndPropagate(ObjcProvider provider) { + for (Map.Entry<Key<?>, NestedSet<?>> typeEntry : provider.items.entrySet()) { + uncheckedAddTransitive(typeEntry.getKey(), typeEntry.getValue(), true); + } + return this; + } + + /** + * Add all elements from a single key of the given provider, and propagate them to any + * (transitive) dependers on this ObjcProvider. + */ + public <E> Builder addTransitiveAndPropagate(Key<E> key, ObjcProvider provider) { + addTransitiveAndPropagate(key, provider.get(key)); + return this; + } + + /** + * Add all elements from providers, and propagate them to any (transitive) dependers on this + * ObjcProvider. + */ + public Builder addTransitiveAndPropagate(Iterable<ObjcProvider> providers) { + for (ObjcProvider provider : providers) { + addTransitiveAndPropagate(provider); + } + return this; + } + + /** + * Add elements from providers, but don't propagate them to any dependers on this ObjcProvider. + * These elements will be exposed to {@link #get(Key)} calls, but not to any ObjcProviders + * which add this provider to themself. + */ + public Builder addTransitiveWithoutPropagating(Iterable<ObjcProvider> providers) { + for (ObjcProvider provider : providers) { + for (Map.Entry<Key<?>, NestedSet<?>> typeEntry : provider.items.entrySet()) { + uncheckedAddTransitive(typeEntry.getKey(), typeEntry.getValue(), false); + } + } + return this; + } + + /** + * Add element, and propagate it to any (transitive) dependers on this ObjcProvider. + */ + public <E> Builder add(Key<E> key, E toAdd) { + uncheckedAddAll(key, ImmutableList.of(toAdd)); + return this; + } + + /** + * Add elements in toAdd, and propagate them to any (transitive) dependers on this ObjcProvider. + */ + public <E> Builder addAll(Key<E> key, Iterable<? extends E> toAdd) { + uncheckedAddAll(key, toAdd); + return this; + } + + public ObjcProvider build() { + ImmutableMap.Builder<Key<?>, NestedSet<?>> propagated = new ImmutableMap.Builder<>(); + for (Map.Entry<Key<?>, NestedSetBuilder<?>> typeEntry : items.entrySet()) { + propagated.put(typeEntry.getKey(), typeEntry.getValue().build()); + } + ImmutableMap.Builder<Key<?>, NestedSet<?>> nonPropagated = new ImmutableMap.Builder<>(); + for (Map.Entry<Key<?>, NestedSetBuilder<?>> typeEntry : nonPropagatedItems.entrySet()) { + nonPropagated.put(typeEntry.getKey(), typeEntry.getValue().build()); + } + return new ObjcProvider(propagated.build(), nonPropagated.build()); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java new file mode 100644 index 0000000..ad1e4dc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java
@@ -0,0 +1,531 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BaseRuleClasses.BaseRule; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Shared utility code for Objective-C rules. + */ +public class ObjcRuleClasses { + + private ObjcRuleClasses() { + throw new UnsupportedOperationException("static-only"); + } + + /** + * Returns a derived Artifact by appending a String to a root-relative path. This is similar to + * {@link RuleContext#getRelatedArtifact(PathFragment, String)}, except the existing extension is + * not removed. + */ + static Artifact artifactByAppendingToRootRelativePath( + RuleContext ruleContext, PathFragment path, String suffix) { + return ruleContext.getAnalysisEnvironment().getDerivedArtifact( + path.replaceName(path.getBaseName() + suffix), + ruleContext.getBinOrGenfilesDirectory()); + } + + static IntermediateArtifacts intermediateArtifacts(RuleContext ruleContext) { + return new IntermediateArtifacts( + ruleContext.getAnalysisEnvironment(), ruleContext.getBinOrGenfilesDirectory(), + ruleContext.getLabel(), /*archiveFileNameSuffix=*/""); + } + + /** + * Returns a {@link IntermediateArtifacts} to be used to compile and link the ObjC source files + * in {@code j2ObjcSource}. + */ + static IntermediateArtifacts j2objcIntermediateArtifacts(RuleContext ruleContext, + J2ObjcSource j2ObjcSource) { + // We need to append "_j2objc" to the name of the generated archive file to distinguish it from + // the C/C++ archive file created by proto_library targets with attribute cc_api_version + // specified. + return new IntermediateArtifacts( + ruleContext.getAnalysisEnvironment(), + ruleContext.getConfiguration().getBinDirectory(), + j2ObjcSource.getTargetLabel(), + /*archiveFileNameSuffix=*/"_j2objc"); + } + + /** + * Returns a {@link J2ObjcSrcsProvider} with J2ObjC-generated ObjC file information from the + * current rule, and from rules that can be reached transitively through the "deps" attribute. + * + * @param ruleContext the rule context of the current rule + * @param currentSource J2ObjC-generated ObjC file information from the current rule to contribute + * to the returned provider + * @return a {@link J2ObjcSrcsProvider} containing {@code currentSources} and source information + * from the transitive closure. + */ + public static J2ObjcSrcsProvider j2ObjcSrcsProvider(RuleContext ruleContext, + J2ObjcSource currentSource) { + return j2ObjcSrcsProvider(ruleContext, Optional.of(currentSource)); + } + + /** + * Returns a {@link J2ObjcSrcsProvider} with J2ObjC-generated ObjC file information from rules + * that can be reached transitively through the "deps" attribute. + * + * @param ruleContext the rule context of the current rule + * @return a {@link J2ObjcSrcsProvider} containing source information from the transitive closure. + */ + public static J2ObjcSrcsProvider j2ObjcSrcsProvider(RuleContext ruleContext) { + return j2ObjcSrcsProvider(ruleContext, Optional.<J2ObjcSource>absent()); + } + + private static J2ObjcSrcsProvider j2ObjcSrcsProvider(RuleContext ruleContext, + Optional<J2ObjcSource> currentSource) { + NestedSetBuilder<J2ObjcSource> builder = NestedSetBuilder.stableOrder(); + builder.addAll(currentSource.asSet()); + boolean hasProtos = currentSource.isPresent() + && currentSource.get().getSourceType() == J2ObjcSource.SourceType.PROTO; + + for (J2ObjcSrcsProvider provider : + ruleContext.getPrerequisites("deps", Mode.TARGET, J2ObjcSrcsProvider.class)) { + builder.addTransitive(provider.getSrcs()); + hasProtos |= provider.hasProtos(); + } + + return new J2ObjcSrcsProvider(builder.build(), hasProtos); + } + + public static Artifact artifactByAppendingToBaseName(RuleContext context, String suffix) { + return artifactByAppendingToRootRelativePath( + context, context.getLabel().toPathFragment(), suffix); + } + + static ObjcActionsBuilder actionsBuilder(RuleContext ruleContext) { + return new ObjcActionsBuilder( + ruleContext, + intermediateArtifacts(ruleContext), + ObjcRuleClasses.objcConfiguration(ruleContext), + ruleContext.getConfiguration(), + ruleContext); + } + + public static ObjcConfiguration objcConfiguration(RuleContext ruleContext) { + return ruleContext.getFragment(ObjcConfiguration.class); + } + + @VisibleForTesting + static final Iterable<SdkFramework> AUTOMATIC_SDK_FRAMEWORKS = ImmutableList.of( + new SdkFramework("Foundation"), new SdkFramework("UIKit")); + + /** + * Attributes for {@code objc_*} rules that have compiler (and in the future, possibly linker) + * options + */ + @BlazeRule(name = "$objc_opts_rule", + type = RuleClassType.ABSTRACT) + public static class ObjcOptsRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE($objc_opts_rule).ATTRIBUTE(copts) --> + Extra flags to pass to the compiler. + ${SYNOPSIS} + Subject to <a href="#make_variables">"Make variable"</a> substitution and + <a href="#sh-tokenization">Bourne shell tokenization</a>. + These flags will only apply to this target, and not those upon which + it depends, or those which depend on it. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("copts", STRING_LIST)) + .build(); + } + } + + /** + * Attributes for {@code objc_*} rules that can link in SDK frameworks. + */ + @BlazeRule(name = "$objc_sdk_frameworks_rule", + type = RuleClassType.ABSTRACT) + public static class ObjcSdkFrameworksRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE($objc_sdk_frameworks_rule).ATTRIBUTE(sdk_frameworks) --> + Names of SDK frameworks to link with. For instance, "XCTest" or + "Cocoa". "UIKit" and "Foundation" are always included and do not mean + anything if you include them. + When linking a library, only those frameworks named in that library's + sdk_frameworks attribute are linked in. When linking a binary, all + SDK frameworks named in that binary's transitive dependency graph are + used. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("sdk_frameworks", STRING_LIST)) + /* <!-- #BLAZE_RULE($objc_sdk_frameworks_rule).ATTRIBUTE(weak_sdk_frameworks) --> + Names of SDK frameworks to weakly link with. For instance, + "MediaAccessibility". In difference to regularly linked SDK + frameworks, symbols from weakly linked frameworks do not cause an + error if they are not present at runtime. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("weak_sdk_frameworks", STRING_LIST)) + /* <!-- #BLAZE_RULE($objc_sdk_frameworks_rule).ATTRIBUTE(sdk_dylibs) --> + Names of SDK .dylib libraries to link with. For instance, "libz" or + "libarchive". "libc++" is included automatically if the binary has + any C++ or Objective-C++ sources in its dependency tree. When linking + a binary, all libraries named in that binary's transitive dependency + graph are used. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("sdk_dylibs", STRING_LIST)) + .build(); + } + } + + /** + * Iff a file matches this type, it is considered to use C++. + */ + static final FileType CPP_SOURCES = FileType.of(".cc", ".cpp", ".mm", ".cxx", ".C"); + + private static final FileType NON_CPP_SOURCES = FileType.of(".m", ".c"); + + static final FileTypeSet SRCS_TYPE = FileTypeSet.of(NON_CPP_SOURCES, CPP_SOURCES); + + static final FileTypeSet NON_ARC_SRCS_TYPE = FileTypeSet.of(FileType.of(".m", ".mm")); + + static final FileTypeSet PLIST_TYPE = FileTypeSet.of(FileType.of(".plist")); + + static final FileTypeSet STORYBOARD_TYPE = FileTypeSet.of(FileType.of(".storyboard")); + + static final FileType XIB_TYPE = FileType.of(".xib"); + + /** + * Common attributes for {@code objc_*} rules that allow the definition of resources such as + * storyboards. + */ + @BlazeRule(name = "$objc_base_resources_rule", + type = RuleClassType.ABSTRACT, + ancestors = { BaseRule.class }) + public static class ObjcBaseResourcesRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(strings) --> + Files which are plists of strings, often localizable. These files + are converted to binary plists (if they are not already) and placed + in the bundle root of the final package. If this file's immediate + containing directory is named *.lproj (e.g. en.lproj, Base.lproj), it + will be placed under a directory of that name in the final bundle. + This allows for localizable strings. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("strings", LABEL_LIST).legacyAllowAnyFileType() + .direct_compile_time_input()) + /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(xibs) --> + Files which are .xib resources, possibly localizable. These files are + compiled to .nib files and placed the bundle root of the final + package. If this file's immediate containing directory is named + *.lproj (e.g. en.lproj, Base.lproj), it will be placed under a + directory of that name in the final bundle. This allows for + localizable UI. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("xibs", LABEL_LIST) + .direct_compile_time_input() + .allowedFileTypes(XIB_TYPE)) + /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(storyboards) --> + Files which are .storyboard resources, possibly localizable. These + files are compiled to .storyboardc directories, which are placed in + the bundle root of the final package. If the storyboards's immediate + containing directory is named *.lproj (e.g. en.lproj, Base.lproj), it + will be placed under a directory of that name in the final bundle. + This allows for localizable UI. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("storyboards", LABEL_LIST) + .allowedFileTypes(STORYBOARD_TYPE)) + /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(resources) --> + Files to include in the final application bundle. They are not + processed or compiled in any way besides the processing done by the + rules that actually generate them. These files are placed in the root + of the bundle (e.g. Payload/foo.app/...) in most cases. However, if + they appear to be localized (i.e. are contained in a directory called + *.lproj), they will be placed in a directory of the same name in the + app bundle. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("resources", LABEL_LIST).legacyAllowAnyFileType().direct_compile_time_input()) + /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(datamodels) --> + Files that comprise the data models of the final linked binary. + Each file must have a containing directory named *.xcdatamodel, which + is usually contained by another *.xcdatamodeld (note the added d) + directory. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("datamodels", LABEL_LIST).legacyAllowAnyFileType() + .direct_compile_time_input()) + /* <!-- #BLAZE_RULE($objc_base_resources_rule).ATTRIBUTE(asset_catalogs) --> + Files that comprise the asset catalogs of the final linked binary. + Each file must have a containing directory named *.xcassets. This + containing directory becomes the root of one of the asset catalogs + linked with any binary that depends directly or indirectly on this + target. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("asset_catalogs", LABEL_LIST).legacyAllowAnyFileType() + .direct_compile_time_input()) + .add(attr("$xcodegen", LABEL).cfg(HOST).exec() + .value(env.getLabel("//tools/objc:xcodegen"))) + .add(attr("$plmerge", LABEL).cfg(HOST).exec() + .value(env.getLabel("//tools/objc:plmerge"))) + .add(attr("$momczip_deploy", LABEL).cfg(HOST) + .value(env.getLabel("//tools/objc:momczip_deploy.jar"))) + .add(attr("$actooloribtoolzip_deploy", LABEL).cfg(HOST) + .value(env.getLabel("//tools/objc:actooloribtoolzip_deploy.jar"))) + .build(); + } + } + + /** + * Common attributes for {@code objc_*} rules that contain compilable content. + */ + @BlazeRule(name = "$objc_compilation_rule", + type = RuleClassType.ABSTRACT, + ancestors = { BaseRuleClasses.RuleBase.class, ObjcSdkFrameworksRule.class, + ObjcBaseResourcesRule.class }) + public static class ObjcCompilationRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE($objc_compilation_rule).ATTRIBUTE(hdrs) --> + The list of Objective-C files that are included as headers by source + files in this rule or by users of this library. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("hdrs", LABEL_LIST) + .direct_compile_time_input() + .allowedFileTypes(FileTypeSet.ANY_FILE)) + /* <!-- #BLAZE_RULE($objc_compilation_rule).ATTRIBUTE(includes) --> + List of <code>#include/#import</code> search paths to add to this target + and all depending targets. This is to support third party and + open-sourced libraries that do not specify the entire workspace path in + their <code>#import/#include</code> statements. + <p> + The paths are interpreted relative to the package directory, and the + genfiles and bin roots (e.g. <code>blaze-genfiles/pkg/includedir</code> + and <code>blaze-out/pkg/includedir</code>) are included in addition to the + actual client root. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("includes", Type.STRING_LIST)) + /* <!-- #BLAZE_RULE($objc_compilation_rule).ATTRIBUTE(sdk_includes) --> + List of <code>#include/#import</code> search paths to add to this target + and all depending targets, where each path is relative to + <code>$(SDKROOT)/usr/include</code>. + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("sdk_includes", Type.STRING_LIST)) + .build(); + } + } + + /** + * Common attributes for rules that uses ObjC proto compiler. + */ + @BlazeRule(name = "$objc_proto_rule", + type = RuleClassType.ABSTRACT) + public static class ObjcProtoRule implements RuleDefinition { + + /** + * A Predicate that returns true if the ObjC proto compiler and its support deps are needed by + * the current rule. + * + * <p>For proto_library rules, this will return true if they have a j2objc_api_version + * attribute, and it is greater than 0. For other rules, this will return true by default. + */ + public static final Predicate<AttributeMap> USE_PROTO_COMPILER = new Predicate<AttributeMap>() { + @Override + public boolean apply(AttributeMap rule) { + return rule.getAttributeDefinition("j2objc_api_version") == null + || rule.get("j2objc_api_version", Type.INTEGER) != 0; + } + }; + + public static final String COMPILE_PROTOS_ATTR = "$googlemac_proto_compiler"; + public static final String PROTO_SUPPORT_ATTR = "$googlemac_proto_compiler_support"; + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .add(attr(COMPILE_PROTOS_ATTR, LABEL) + .allowedFileTypes(FileType.of(".py")) + .cfg(HOST) + .singleArtifact() + .condition(USE_PROTO_COMPILER) + .value(env.getLabel("//tools/objc:compile_protos"))) + .add(attr(PROTO_SUPPORT_ATTR, LABEL) + .legacyAllowAnyFileType() + .cfg(HOST) + .condition(USE_PROTO_COMPILER) + .value(env.getLabel("//tools/objc:proto_support"))) + .build(); + } + } + + /** + * Base rule definition for iOS test rules. + */ + @BlazeRule(name = "$ios_test_base_rule", + type = RuleClassType.ABSTRACT, + ancestors = { ObjcBinaryRule.class }) + public static class IosTestBaseRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE($ios_test_base_rule).ATTRIBUTE(target_device) --> + The device against which to run the test. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr(IosTest.TARGET_DEVICE, LABEL) + .allowedFileTypes() + .allowedRuleClasses("ios_device")) + /* <!-- #BLAZE_RULE($ios_test_base_rule).ATTRIBUTE(xctest) --> + Whether this target contains tests using the XCTest testing framework. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr(IosTest.IS_XCTEST, BOOLEAN)) + /* <!-- #BLAZE_RULE($ios_test_base_rule).ATTRIBUTE(xctest_app) --> + A <code>objc_binary</code> target that contains the app bundle to test against in XCTest. + This attribute is only valid if <code>xctest</code> is true. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr(IosTest.XCTEST_APP, LABEL) + .value(new Attribute.ComputedDefault(IosTest.IS_XCTEST) { + @Override + public Object getDefault(AttributeMap rule) { + return rule.get(IosTest.IS_XCTEST, Type.BOOLEAN) + ? env.getLabel("//tools/objc:xctest_app") + : null; + } + }) + .allowedFileTypes() + .allowedRuleClasses("objc_binary")) + .override(attr("infoplist", LABEL) + .value(new Attribute.ComputedDefault(IosTest.IS_XCTEST) { + @Override + public Object getDefault(AttributeMap rule) { + return rule.get(IosTest.IS_XCTEST, Type.BOOLEAN) + ? env.getLabel("//tools/objc:xctest_infoplist") + : null; + } + }) + .allowedFileTypes(PLIST_TYPE)) + .build(); + } + } + + /** + * Abstract rule type with the {@code infoplist} attribute. + */ + @BlazeRule(name = "$objc_has_infoplist_rule", + type = RuleClassType.ABSTRACT) + public static class ObjcHasInfoplistRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE($objc_has_infoplist_rule).ATTRIBUTE(infoplist) --> + The infoplist file. This corresponds to <i>appname</i>-Info.plist in Xcode projects. + ${SYNOPSIS} + Blaze will perform variable substitution on the plist file for the following values: + <ul> + <li><code>${EXECUTABLE_NAME}</code>: The name of the executable generated and included + in the bundle by blaze, which can be used as the value for + <code>CFBundleExecutable</code> within the plist. + <li><code>${BUNDLE_NAME}</code>: This target's name and bundle suffix (.bundle or .app) + in the form<code><var>name</var></code>.<code>suffix</code>. + <li><code>${PRODUCT_NAME}</code>: This target's name. + </ul> + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("infoplist", LABEL) + .allowedFileTypes(PLIST_TYPE)) + .build(); + } + } + + /** + * Abstract rule type with the {@code entitlements} attribute. + */ + @BlazeRule(name = "$objc_has_entitlements_rule", + type = RuleClassType.ABSTRACT) + public static class ObjcHasEntitlementsRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, final RuleDefinitionEnvironment env) { + return builder + /* <!-- #BLAZE_RULE($objc_has_entitlements_rule).ATTRIBUTE(entitlements) --> + The entitlements file required for device builds of this application. See + <a href="https://developer.apple.com/library/mac/documentation/Miscellaneous/Reference/EntitlementKeyReference/Chapters/AboutEntitlements.html">the apple documentation</a> + for more information. If absent, the default entitlements from the + provisioning profile will be used. + <p> + The following variables are substituted as per + <a href="https://developer.apple.com/library/ios/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html">their definitions in Apple's documentation</a>: + $(AppIdentifierPrefix) and $(CFBundleIdentifier). + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .add(attr("entitlements", LABEL).legacyAllowAnyFileType()) + .build(); + } + } + + /** + * Object that supplies tools used by all rules which have the helper tools common to most rule + * implementations. + */ + static final class Tools { + private final RuleContext ruleContext; + + Tools(RuleContext ruleContext) { + this.ruleContext = Preconditions.checkNotNull(ruleContext); + } + + Artifact actooloribtoolzipDeployJar() { + return ruleContext.getPrerequisiteArtifact("$actooloribtoolzip_deploy", Mode.HOST); + } + + Artifact momczipDeployJar() { + return ruleContext.getPrerequisiteArtifact("$momczip_deploy", Mode.HOST); + } + + FilesToRunProvider xcodegen() { + return ruleContext.getExecutablePrerequisite("$xcodegen", Mode.HOST); + } + + FilesToRunProvider plmerge() { + return ruleContext.getExecutablePrerequisite("$plmerge", Mode.HOST); + } + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcSdkFrameworks.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcSdkFrameworks.java new file mode 100644 index 0000000..2ff3a7c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcSdkFrameworks.java
@@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.objc.ObjcRuleClasses.ObjcSdkFrameworksRule; + +/** + * Common logic for rules that inherit from {@link ObjcSdkFrameworksRule}. + */ +public class ObjcSdkFrameworks { + + /** + * Class that handles extraction and processing of attributes common to inheritors of {@link + * ObjcSdkFrameworksRule}. + */ + public static class Attributes { + + private final RuleContext ruleContext; + + public Attributes(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + /** + * Returns the SDK frameworks defined on the rule's {@code sdk_frameworks} attribute as well as + * base frameworks defined in {@link ObjcRuleClasses#AUTOMATIC_SDK_FRAMEWORKS}. + */ + ImmutableSet<SdkFramework> sdkFrameworks() { + ImmutableSet.Builder<SdkFramework> result = new ImmutableSet.Builder<>(); + result.addAll(ObjcRuleClasses.AUTOMATIC_SDK_FRAMEWORKS); + for (String explicit : ruleContext.attributes().get("sdk_frameworks", Type.STRING_LIST)) { + result.add(new SdkFramework(explicit)); + } + return result.build(); + } + + /** + * Returns all SDK frameworks defined on the rule's {@code weak_sdk_frameworks} attribute. + */ + ImmutableSet<SdkFramework> weakSdkFrameworks() { + ImmutableSet.Builder<SdkFramework> result = new ImmutableSet.Builder<>(); + for (String frameworkName : + ruleContext.attributes().get("weak_sdk_frameworks", Type.STRING_LIST)) { + result.add(new SdkFramework(frameworkName)); + } + return result.build(); + } + + /** + * Returns all SDK dylibs defined on the rule's {@code sdk_dylibs} attribute. + */ + ImmutableSet<String> sdkDylibs() { + return ImmutableSet.copyOf(ruleContext.attributes().get("sdk_dylibs", Type.STRING_LIST)); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeproj.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeproj.java new file mode 100644 index 0000000..e167f89 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeproj.java
@@ -0,0 +1,47 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * Implementation for {@code objc_xcodeproj}. + */ +public class ObjcXcodeproj implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + XcodeProvider.Project project = XcodeProvider.Project.fromTopLevelTargets( + ruleContext.getPrerequisites("deps", Mode.TARGET, XcodeProvider.class)); + Artifact pbxproj = ruleContext.getImplicitOutputArtifact(XcodeSupport.PBXPROJ); + + ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext); + actionsBuilder.registerXcodegenActions( + new ObjcRuleClasses.Tools(ruleContext), pbxproj, project); + + return new RuleConfiguredTargetBuilder(ruleContext) + .setFilesToBuild(NestedSetBuilder.create(Order.STABLE_ORDER, pbxproj)) + .addProvider(RunfilesProvider.class, RunfilesProvider.EMPTY) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeprojRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeprojRule.java new file mode 100644 index 0000000..0a6c4ba --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcXcodeprojRule.java
@@ -0,0 +1,78 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; + +/** + * Rule definition for {@code objc_xcodeproj}. + */ +@BlazeRule(name = "objc_xcodeproj", + factoryClass = ObjcXcodeproj.class, + ancestors = { BaseRuleClasses.RuleBase.class }) +public class ObjcXcodeprojRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + /*<!-- #BLAZE_RULE(objc_xcodeproj).IMPLICIT_OUTPUTS --> + <ul> + <li><code><var>name</var>.xcodeproj/project.pbxproj</code>: A combined Xcode project file + containing all the included targets which can be used to develop or build on a Mac.</li> + </ul> + <!-- #END_BLAZE_RULE.IMPLICIT_OUTPUTS -->*/ + .setImplicitOutputsFunction(XcodeSupport.PBXPROJ) + /* <!-- #BLAZE_RULE(objc_xcodeproj).ATTRIBUTE(deps) --> + The list of targets to include in the combined Xcode project file. + ${SYNOPSIS} + <!-- #END_BLAZE_RULE.ATTRIBUTE -->*/ + .override(builder.copy("deps") + .nonEmpty() + .allowedRuleClasses( + "objc_binary", + "ios_test", + "objc_bundle_library", + "objc_import", + "objc_library")) + .override(attr("testonly", BOOLEAN) + .nonconfigurable("Must support test deps.") + .value(true)) + .add(attr("$xcodegen", LABEL) + .cfg(HOST) + .exec() + .value(env.getLabel("//tools/objc:xcodegen"))) + .build(); + } +} + +/*<!-- #BLAZE_RULE (NAME = objc_xcodeproj, TYPE = OTHER, FAMILY = Objective-C) --> + +${ATTRIBUTE_SIGNATURE} + +<p>This rule combines build information about several objc targets (and all their transitive +dependencies) into a single Xcode project file, for use in developing on a Mac.</p> + +${ATTRIBUTE_DEFINITION} + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/OptionsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/OptionsProvider.java new file mode 100644 index 0000000..f87c96a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/OptionsProvider.java
@@ -0,0 +1,87 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.xcode.util.Value; + +/** + * Provides information contained in a {@code objc_options} target. + */ +@Immutable +final class OptionsProvider + extends Value<OptionsProvider> + implements TransitiveInfoProvider { + static final class Builder { + private Iterable<String> copts = ImmutableList.of(); + private final NestedSetBuilder<Artifact> infoplists = NestedSetBuilder.stableOrder(); + + /** + * Adds copts to the end of the copts sequence. + */ + public Builder addCopts(Iterable<String> copts) { + this.copts = Iterables.concat(this.copts, copts); + return this; + } + + public Builder addInfoplists(Iterable<Artifact> infoplists) { + this.infoplists.addAll(infoplists); + return this; + } + + /** + * Adds infoplists and copts from the given provider, if present. copts are added to the end of + * the sequence. + */ + public Builder addTransitive(Optional<OptionsProvider> maybeProvider) { + for (OptionsProvider provider : maybeProvider.asSet()) { + this.copts = Iterables.concat(this.copts, provider.copts); + this.infoplists.addTransitive(provider.infoplists); + } + return this; + } + + public OptionsProvider build() { + return new OptionsProvider(ImmutableList.copyOf(copts), infoplists.build()); + } + } + + public static final OptionsProvider DEFAULT = new Builder().build(); + + private final ImmutableList<String> copts; + private final NestedSet<Artifact> infoplists; + + private OptionsProvider(ImmutableList<String> copts, NestedSet<Artifact> infoplists) { + super(copts, infoplists); + this.copts = Preconditions.checkNotNull(copts); + this.infoplists = Preconditions.checkNotNull(infoplists); + } + + public ImmutableList<String> getCopts() { + return copts; + } + + public NestedSet<Artifact> getInfoplists() { + return infoplists; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ResourceSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ResourceSupport.java new file mode 100644 index 0000000..d1a717c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ResourceSupport.java
@@ -0,0 +1,123 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; + +/** + * Support for resource processing on Objc rules. + * + * <p>Methods on this class can be called in any order without impacting the result. + */ +final class ResourceSupport { + private final RuleContext ruleContext; + private final Attributes attributes; + private final IntermediateArtifacts intermediateArtifacts; + private final Iterable<Xcdatamodel> datamodels; + + /** + * Creates a new resource support for the given context. + */ + ResourceSupport(RuleContext ruleContext) { + this.ruleContext = ruleContext; + this.attributes = new Attributes(ruleContext); + this.intermediateArtifacts = ObjcRuleClasses.intermediateArtifacts(ruleContext); + this.datamodels = Xcdatamodels.xcdatamodels(intermediateArtifacts, attributes.datamodels()); + } + + /** + * Registers resource generating actions (strings, storyboards, ...). + * + * @param storyboards storyboards defined by this rule + * + * @return this resource support + */ + ResourceSupport registerActions(Storyboards storyboards) { + ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext); + + ObjcRuleClasses.Tools tools = new ObjcRuleClasses.Tools(ruleContext); + actionsBuilder.registerResourceActions( + tools, + new ObjcActionsBuilder.StringsFiles( + CompiledResourceFile.fromStringsFiles(intermediateArtifacts, attributes.strings())), + new XibFiles(attributes.xibs()), + datamodels); + for (Artifact storyboardInput : storyboards.getInputs()) { + actionsBuilder.registerIbtoolzipAction( + tools, storyboardInput, intermediateArtifacts.compiledStoryboardZip(storyboardInput)); + } + return this; + } + + /** + * Adds common xcode settings to the given provider builder. + * + * @return this resource support + */ + ResourceSupport addXcodeSettings(XcodeProvider.Builder xcodeProviderBuilder) { + xcodeProviderBuilder.addInputsToXcodegen(Xcdatamodel.inputsToXcodegen(datamodels)); + return this; + } + + /** + * Validates resource attributes on this rule. + * + * @return this resource support + */ + ResourceSupport validateAttributes() { + Iterable<String> assetCatalogErrors = ObjcCommon.notInContainerErrors( + attributes.assetCatalogs(), ObjcCommon.ASSET_CATALOG_CONTAINER_TYPE); + for (String error : assetCatalogErrors) { + ruleContext.attributeError("asset_catalogs", error); + } + + Iterable<String> dataModelErrors = + ObjcCommon.notInContainerErrors(attributes.datamodels(), Xcdatamodels.CONTAINER_TYPES); + for (String error : dataModelErrors) { + ruleContext.attributeError("datamodels", error); + } + + return this; + } + + private static class Attributes { + private final RuleContext ruleContext; + + Attributes(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + ImmutableList<Artifact> datamodels() { + return ruleContext.getPrerequisiteArtifacts("datamodels", Mode.TARGET).list(); + } + + ImmutableList<Artifact> xibs() { + return ruleContext.getPrerequisiteArtifacts("xibs", Mode.TARGET) + .errorsForNonMatching(ObjcRuleClasses.XIB_TYPE) + .list(); + } + + ImmutableList<Artifact> strings() { + return ruleContext.getPrerequisiteArtifacts("strings", Mode.TARGET).list(); + } + + ImmutableList<Artifact> assetCatalogs() { + return ruleContext.getPrerequisiteArtifacts("asset_catalogs", Mode.TARGET).list(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/SdkFramework.java b/src/main/java/com/google/devtools/build/lib/rules/objc/SdkFramework.java new file mode 100644 index 0000000..c692fcd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/SdkFramework.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.xcode.util.Value; + +/** + * Represents the name of an SDK framework. + * <p> + * Besides being a glorified String, this class prevents you from adding framework names to an + * argument list without explicitly specifying how to prefix them. + */ +final class SdkFramework extends Value<SdkFramework> { + private final String name; + + public SdkFramework(String name) { + super(name); + this.name = name; + } + + public String getName() { + return name; + } + + /** + * Returns an iterable which contains the name of each given framework in the same order. + */ + static Iterable<String> names(Iterable<SdkFramework> frameworks) { + ImmutableList.Builder<String> result = new ImmutableList.Builder<>(); + for (SdkFramework framework : frameworks) { + result.add(framework.getName()); + } + return result.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/Storyboards.java b/src/main/java/com/google/devtools/build/lib/rules/objc/Storyboards.java new file mode 100644 index 0000000..204c22d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/Storyboards.java
@@ -0,0 +1,76 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; + +/** + * Contains information about storyboards for a single target. This does not include information + * about the transitive closure. A storyboard: + * <ul> + * <li>Is a single file with an extension of {@code .storyboard} in its uncompiled, checked-in + * form. + * <li>Can be in a localized {@code .lproj} directory, including {@code Base.lproj}. + * <li>Compiles with {@code ibtool} to a directory with extension {@code .storyboardc} (note the + * added "c") + * </ul> + * + * <p>The {@link NestedSet}s stored in this class are only one level deep, and do not include the + * storyboards in the transitive closure. This is to facilitate structural sharing between copies + * of the sequences - the output zips can be added transitively to the inputs of the merge bundle + * action, as well as to the files to build set, and only one instance of the sequence exists for + * each set. + */ +final class Storyboards { + private final NestedSet<Artifact> outputZips; + private final NestedSet<Artifact> inputs; + + private Storyboards(NestedSet<Artifact> outputZips, NestedSet<Artifact> inputs) { + this.outputZips = outputZips; + this.inputs = inputs; + } + + public NestedSet<Artifact> getOutputZips() { + return outputZips; + } + + public NestedSet<Artifact> getInputs() { + return inputs; + } + + static Storyboards empty() { + return new Storyboards( + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER), + NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER)); + } + + /** + * Generates a set of new instances given the raw storyboard inputs. + * @param inputs the {@code .storyboard} files. + * @param intermediateArtifacts the object used to determine the output zip {@link Artifact}s. + */ + static Storyboards fromInputs( + Iterable<Artifact> inputs, IntermediateArtifacts intermediateArtifacts) { + NestedSetBuilder<Artifact> outputZips = NestedSetBuilder.stableOrder(); + for (Artifact input : inputs) { + outputZips.add(intermediateArtifacts.compiledStoryboardZip(input)); + } + return new Storyboards(outputZips.build(), NestedSetBuilder.wrap(Order.STABLE_ORDER, inputs)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XcTestAppProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XcTestAppProvider.java new file mode 100644 index 0000000..1fc5d5d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XcTestAppProvider.java
@@ -0,0 +1,57 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Supplies information needed when a dependency serves as an {@code xctest_app}. + */ +@Immutable +final class XcTestAppProvider implements TransitiveInfoProvider { + private final Artifact bundleLoader; + private final Artifact ipa; + private final ObjcProvider objcProvider; + + XcTestAppProvider(Artifact bundleLoader, Artifact ipa, ObjcProvider objcProvider) { + this.bundleLoader = Preconditions.checkNotNull(bundleLoader); + this.ipa = Preconditions.checkNotNull(ipa); + this.objcProvider = Preconditions.checkNotNull(objcProvider); + } + + /** + * The bundle loader, which corresponds to the test app's binary. + */ + public Artifact getBundleLoader() { + return bundleLoader; + } + + public Artifact getIpa() { + return ipa; + } + + /** + * An {@link ObjcProvider} that should be included by any test target that uses this app as its + * {@code xctest_app}. This is <strong>not</strong> a typical {@link ObjcProvider} - it has + * certain linker-releated keys omitted, such as {@link ObjcProvider#LIBRARY}, since XcTests have + * access to symbols in their test rig without linking them into the main test binary. + */ + public ObjcProvider getObjcProvider() { + return objcProvider; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodel.java b/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodel.java new file mode 100644 index 0000000..5b29435 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodel.java
@@ -0,0 +1,136 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.util.Value; + +/** + * Represents an .xcdatamodel[d] directory - knowing all {@code Artifact}s contained therein - and + * the .zip file that it is compiled to which should be merged with the final application bundle. + * <p> + * An .xcdatamodel (here and below note that lack or presence of a d) directory contains the schema + * for a managed object, or a managed object model. It typically has two files: {@code layout} and + * {@code contents}, although this detail isn't addressed in Bazel code. Directories of this + * sort are compiled into a single .mom file. If the .xcdatamodel directory is inside a + * .xcdatamodeld directory, then the .mom file is placed inside a .momd directory. The .momd + * directory or .mom file is placed in the bundle root of the final bundle. + * <p> + * An .xcdatamodeld directory contains several .xcdatamodel directories, each corresponding to a + * different version. In addition the .xcdatamodeld directory contains a {@code .xccurrentversion} + * file which identifies the current version. (this file is also not handled explicitly by Bazel + * code). + * <p> + * When processing artifacts referenced by a {@code datamodels} attribute, we must determine if it + * is in a .xcdatamodeld directory or only a .xcdatamodel directory. We also must group the + * artifacts by their container, the container being an .xcdatamodeld directory if possible, and a + * .xcdatamodel directory otherwise. Every container is compiled with a single invocation of the + * Managed Object Model Compiler (momc) and corresponds to exactly one instance of this class. We + * invoke momc indirectly through the momczip tool (part of Bazel) which runs momc and zips the + * output. The files in this zip are placed in the bundle root of the final application, not unlike + * the zips generated by {@code actooloribtoolzip}. + */ +class Xcdatamodel extends Value<Xcdatamodel> { + private final Artifact outputZip; + private final ImmutableSet<Artifact> inputs; + private final PathFragment container; + + Xcdatamodel(Artifact outputZip, ImmutableSet<Artifact> inputs, PathFragment container) { + super(ImmutableMap.of( + "outputZip", outputZip, + "inputs", inputs, + "container", container)); + this.outputZip = outputZip; + this.inputs = inputs; + this.container = container; + } + + /** + * Returns the files that should be supplied to Xcodegen when generating a project that includes + * all of the given xcdatamodels. + */ + public static Iterable<Artifact> inputsToXcodegen(Iterable<Xcdatamodel> datamodels) { + ImmutableSet.Builder<Artifact> inputs = new ImmutableSet.Builder<>(); + for (Xcdatamodel datamodel : datamodels) { + for (Artifact generalInput : datamodel.inputs) { + if (generalInput.getExecPath().getBaseName().equals(".xccurrentversion")) { + inputs.add(generalInput); + } + } + } + return inputs.build(); + } + + public Artifact getOutputZip() { + return outputZip; + } + + /** + * Returns every known file in the container. This is every input file that is processed by momc. + */ + public ImmutableSet<Artifact> getInputs() { + return inputs; + } + + public PathFragment getContainer() { + return container; + } + + /** + * The ARCHIVE_ROOT passed to momczip. The archive root is the name of the .mom file + * unversioned object models, and the name of the .momd directory for versioned object models. + */ + public String archiveRootForMomczip() { + return name() + (container.getBaseName().endsWith(".xcdatamodeld") ? ".momd" : ".mom"); + } + + /** + * The name of the data model. This is the name of the container without the extension. For + * instance, if the container is "foo/Information.xcdatamodel" or "bar/Information.xcdatamodeld", + * then the name is "Information". + */ + public String name() { + String baseContainerName = container.getBaseName(); + int lastDot = baseContainerName.lastIndexOf('.'); + return baseContainerName.substring(0, lastDot); + } + + public static Iterable<Artifact> outputZips(Iterable<Xcdatamodel> models) { + return Iterables.transform(models, new Function<Xcdatamodel, Artifact>() { + @Override + public Artifact apply(Xcdatamodel model) { + return model.getOutputZip(); + } + }); + } + + /** + * Returns a sequence of all unique *.xcdatamodel directories that contain all the artifacts of + * the given models. Note that this does not return any *.xcdatamodeld directories. + */ + static Iterable<PathFragment> xcdatamodelDirs(Iterable<Xcdatamodel> models) { + ImmutableSet.Builder<PathFragment> result = new ImmutableSet.Builder<>(); + for (Xcdatamodel model : models) { + result.addAll(ObjcCommon.uniqueContainers(model.getInputs(), FileType.of(".xcdatamodel"))); + } + return result.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodels.java b/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodels.java new file mode 100644 index 0000000..32d48aa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/Xcdatamodels.java
@@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.Map; + +/** + * Utility code for getting information specific to xcdatamodels for a single rule. + */ +class Xcdatamodels { + private Xcdatamodels() {} + + static final ImmutableList<FileType> CONTAINER_TYPES = + ImmutableList.of(FileType.of(".xcdatamodeld"), FileType.of(".xcdatamodel")); + + static Iterable<Xcdatamodel> xcdatamodels( + IntermediateArtifacts intermediateArtifacts, Iterable<Artifact> xcdatamodels) { + ImmutableSet.Builder<Xcdatamodel> result = new ImmutableSet.Builder<>(); + Multimap<PathFragment, Artifact> artifactsByContainer = byContainer(xcdatamodels); + + for (Map.Entry<PathFragment, Collection<Artifact>> modelDirEntry : + artifactsByContainer.asMap().entrySet()) { + PathFragment container = modelDirEntry.getKey(); + Artifact outputZip = intermediateArtifacts.compiledMomZipArtifact(container); + result.add( + new Xcdatamodel(outputZip, ImmutableSet.copyOf(modelDirEntry.getValue()), container)); + } + + return result.build(); + } + + + /** + * Arrange a sequence of artifacts into entries of a multimap by their nearest container + * directory, preferring {@code .xcdatamodeld} over {@code .xcdatamodel}. + * If an artifact is not inside any containing directory, then it is not present in the returned + * map. + */ + static Multimap<PathFragment, Artifact> byContainer(Iterable<Artifact> artifacts) { + ImmutableSetMultimap.Builder<PathFragment, Artifact> result = + new ImmutableSetMultimap.Builder<>(); + for (Artifact artifact : artifacts) { + for (PathFragment modelDir : + ObjcCommon.nearestContainerMatching(CONTAINER_TYPES, artifact).asSet()) { + result.put(modelDir, artifact); + } + } + return result.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProductType.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProductType.java new file mode 100644 index 0000000..1a68206 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProductType.java
@@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +/** + * Possible values that {@code objc_*} rules care about for what Xcode project files refer to as + * "product type." + */ +enum XcodeProductType { + LIBRARY_STATIC("com.apple.product-type.library.static"), + BUNDLE("com.apple.product-type.bundle"), + APPLICATION("com.apple.product-type.application"), + UNIT_TEST("com.apple.product-type.bundle.unit-test"), + EXTENSION("com.apple.product-type.app-extension"); + + private final String identifier; + + XcodeProductType(String identifier) { + this.identifier = identifier; + } + + /** + * Returns the string used to identify this product type in the {@code productType} field of + * {@code PBXNativeTarget} objects in Xcode project files. + */ + public String getIdentifier() { + return identifier; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProvider.java new file mode 100644 index 0000000..b244cd9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeProvider.java
@@ -0,0 +1,452 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.BUNDLE_IMPORT_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.DEFINE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FORCE_LOAD_FOR_XCODEGEN; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.FRAMEWORK_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.GENERAL_RESOURCE_FILE; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.IMPORTED_LIBRARY; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_DYLIB; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.SDK_FRAMEWORK; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.WEAK_SDK_FRAMEWORK; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCASSETS_DIR; +import static com.google.devtools.build.lib.rules.objc.ObjcProvider.XCDATAMODEL; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.objc.ObjcProvider.Flag; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.xcode.util.Interspersing; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.DependencyControl; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.TargetControl; +import com.google.devtools.build.xcode.xcodegen.proto.XcodeGenProtos.XcodeprojBuildSetting; + +import java.util.EnumSet; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Provider which provides transitive dependency information that is specific to Xcodegen. In + * particular, it provides a sequence of targets which can be used to create a self-contained + * {@code .xcodeproj} file. + */ +@Immutable +public final class XcodeProvider implements TransitiveInfoProvider { + /** + * A builder for instances of {@link XcodeProvider}. + */ + public static final class Builder { + private Label label; + private final NestedSetBuilder<String> userHeaderSearchPaths = NestedSetBuilder.stableOrder(); + private final NestedSetBuilder<String> headerSearchPaths = NestedSetBuilder.stableOrder(); + private Optional<InfoplistMerging> infoplistMerging = Optional.absent(); + private final NestedSetBuilder<XcodeProvider> dependencies = NestedSetBuilder.stableOrder(); + private final ImmutableList.Builder<XcodeprojBuildSetting> xcodeprojBuildSettings = + new ImmutableList.Builder<>(); + private final ImmutableList.Builder<String> copts = new ImmutableList.Builder<>(); + private final ImmutableList.Builder<String> compilationModeCopts = + new ImmutableList.Builder<>(); + private XcodeProductType productType; + private final ImmutableList.Builder<Artifact> headers = new ImmutableList.Builder<>(); + private Optional<CompilationArtifacts> compilationArtifacts = Optional.absent(); + private ObjcProvider objcProvider; + private Optional<XcodeProvider> testHost = Optional.absent(); + private final NestedSetBuilder<Artifact> inputsToXcodegen = NestedSetBuilder.stableOrder(); + + /** + * Sets the label of the build target which corresponds to this Xcode target. + */ + public Builder setLabel(Label label) { + this.label = label; + return this; + } + + /** + * Adds user header search paths for this target. + */ + public Builder addUserHeaderSearchPaths(Iterable<PathFragment> userHeaderSearchPaths) { + this.userHeaderSearchPaths.addAll(rootEach("$(WORKSPACE_ROOT)", userHeaderSearchPaths)); + return this; + } + + /** + * Adds header search paths for this target. Each path is interpreted relative to the given + * root, such as {@code "$(WORKSPACE_ROOT)"}. + */ + public Builder addHeaderSearchPaths(String root, Iterable<PathFragment> paths) { + this.headerSearchPaths.addAll(rootEach(root, paths)); + return this; + } + + /** + * Sets the Info.plist merging information. Used for applications. May be + * absent for other bundles. + */ + public Builder setInfoplistMerging(InfoplistMerging infoplistMerging) { + this.infoplistMerging = Optional.of(infoplistMerging); + return this; + } + + /** + * Adds items in the {@link NestedSet}s of the given target to the corresponding sets in this + * builder. This is useful if the given target is a dependency or like a dependency + * (e.g. a test host). The given provider is not registered as a dependency with this provider. + */ + private void addTransitiveSets(XcodeProvider dependencyish) { + inputsToXcodegen.addTransitive(dependencyish.inputsToXcodegen); + userHeaderSearchPaths.addTransitive(dependencyish.userHeaderSearchPaths); + headerSearchPaths.addTransitive(dependencyish.headerSearchPaths); + } + + /** + * Adds {@link XcodeProvider}s corresponding to direct dependencies of this target which should + * be added in the {@code .xcodeproj} file. + */ + public Builder addDependencies(Iterable<XcodeProvider> dependencies) { + for (XcodeProvider dependency : dependencies) { + this.dependencies.add(dependency); + this.dependencies.addTransitive(dependency.dependencies); + this.addTransitiveSets(dependency); + } + return this; + } + + /** + * Adds additional build settings of this target. + */ + public Builder addXcodeprojBuildSettings( + Iterable<XcodeprojBuildSetting> xcodeprojBuildSettings) { + this.xcodeprojBuildSettings.addAll(xcodeprojBuildSettings); + return this; + } + + /** + * Sets the copts to use when compiling the Xcode target. + */ + public Builder addCopts(Iterable<String> copts) { + this.copts.addAll(copts); + return this; + } + + /** + * Sets the copts derived from compilation mode to use when compiling the Xcode target. These + * will be included before the DEFINE options. + */ + public Builder addCompilationModeCopts(Iterable<String> copts) { + this.compilationModeCopts.addAll(copts); + return this; + } + + /** + * Sets the product type for the PBXTarget in the .xcodeproj file. + */ + public Builder setProductType(XcodeProductType productType) { + this.productType = productType; + return this; + } + + /** + * Adds to the header files of this target. It needs not to include the header files of + * dependencies. + */ + public Builder addHeaders(Iterable<Artifact> headers) { + this.headers.addAll(headers); + return this; + } + + /** + * The compilation artifacts for this target. + */ + public Builder setCompilationArtifacts(CompilationArtifacts compilationArtifacts) { + this.compilationArtifacts = Optional.of(compilationArtifacts); + return this; + } + + /** + * Sets the {@link ObjcProvider} corresponding to this target. + */ + public Builder setObjcProvider(ObjcProvider objcProvider) { + this.objcProvider = objcProvider; + return this; + } + + /** + * Sets the test host. This is used for xctest targets. + */ + public Builder setTestHost(XcodeProvider testHost) { + Preconditions.checkState(!this.testHost.isPresent()); + this.testHost = Optional.of(testHost); + this.addTransitiveSets(testHost); + return this; + } + + /** + * Adds inputs that are passed to Xcodegen when generating the project file. + */ + public Builder addInputsToXcodegen(Iterable<Artifact> inputsToXcodegen) { + this.inputsToXcodegen.addAll(inputsToXcodegen); + return this; + } + + public XcodeProvider build() { + Preconditions.checkArgument( + !testHost.isPresent() || (productType == XcodeProductType.UNIT_TEST), + "%s product types cannot have a test host (test host: %s).", productType, testHost); + return new XcodeProvider(this); + } + } + + /** + * A collection of top-level targets that can be used to create a complete project. + */ + public static final class Project { + private final NestedSet<Artifact> inputsToXcodegen; + private final ImmutableList<XcodeProvider> topLevelTargets; + + private Project( + NestedSet<Artifact> inputsToXcodegen, ImmutableList<XcodeProvider> topLevelTargets) { + this.inputsToXcodegen = inputsToXcodegen; + this.topLevelTargets = topLevelTargets; + } + + public static Project fromTopLevelTarget(XcodeProvider topLevelTarget) { + return fromTopLevelTargets(ImmutableList.of(topLevelTarget)); + } + + public static Project fromTopLevelTargets(Iterable<XcodeProvider> topLevelTargets) { + NestedSetBuilder<Artifact> inputsToXcodegen = NestedSetBuilder.stableOrder(); + for (XcodeProvider target : topLevelTargets) { + inputsToXcodegen.addTransitive(target.inputsToXcodegen); + } + return new Project(inputsToXcodegen.build(), ImmutableList.copyOf(topLevelTargets)); + } + + /** + * Returns artifacts that are passed to the Xcodegen action when generating a project file that + * contains all of the given targets. + */ + public NestedSet<Artifact> getInputsToXcodegen() { + return inputsToXcodegen; + } + + public ImmutableList<XcodeProvider> getTopLevelTargets() { + return topLevelTargets; + } + + /** + * Returns all the target controls that must be added to the xcodegen control. No other target + * controls are needed to generate a functional project file. This method creates a new list + * whenever it is called. + */ + public ImmutableList<TargetControl> targets() { + // Collect all the dependencies of all the providers, filtering out duplicates. + Set<XcodeProvider> providerSet = new LinkedHashSet<>(); + for (XcodeProvider target : topLevelTargets) { + Iterables.addAll(providerSet, target.providers()); + } + + ImmutableList.Builder<TargetControl> controls = new ImmutableList.Builder<>(); + for (XcodeProvider provider : providerSet) { + controls.add(provider.targetControl()); + } + return controls.build(); + } + } + + private final Label label; + private final NestedSet<String> userHeaderSearchPaths; + private final NestedSet<String> headerSearchPaths; + private final Optional<InfoplistMerging> infoplistMerging; + private final NestedSet<XcodeProvider> dependencies; + private final ImmutableList<XcodeprojBuildSetting> xcodeprojBuildSettings; + private final ImmutableList<String> copts; + private final ImmutableList<String> compilationModeCopts; + private final XcodeProductType productType; + private final ImmutableList<Artifact> headers; + private final Optional<CompilationArtifacts> compilationArtifacts; + private final ObjcProvider objcProvider; + private final Optional<XcodeProvider> testHost; + private final NestedSet<Artifact> inputsToXcodegen; + + private XcodeProvider(Builder builder) { + this.label = Preconditions.checkNotNull(builder.label); + this.userHeaderSearchPaths = builder.userHeaderSearchPaths.build(); + this.headerSearchPaths = builder.headerSearchPaths.build(); + this.infoplistMerging = builder.infoplistMerging; + this.dependencies = builder.dependencies.build(); + this.xcodeprojBuildSettings = builder.xcodeprojBuildSettings.build(); + this.copts = builder.copts.build(); + this.compilationModeCopts = builder.compilationModeCopts.build(); + this.productType = Preconditions.checkNotNull(builder.productType); + this.headers = builder.headers.build(); + this.compilationArtifacts = builder.compilationArtifacts; + this.objcProvider = Preconditions.checkNotNull(builder.objcProvider); + this.testHost = Preconditions.checkNotNull(builder.testHost); + this.inputsToXcodegen = builder.inputsToXcodegen.build(); + } + + /** + * Creates a builder whose values are all initialized to this provider. + */ + public Builder toBuilder() { + Builder builder = new Builder(); + builder.label = label; + builder.userHeaderSearchPaths.addAll(userHeaderSearchPaths); + builder.headerSearchPaths.addTransitive(headerSearchPaths); + builder.infoplistMerging = infoplistMerging; + builder.dependencies.addTransitive(dependencies); + builder.xcodeprojBuildSettings.addAll(xcodeprojBuildSettings); + builder.copts.addAll(copts); + builder.productType = productType; + builder.headers.addAll(headers); + builder.compilationArtifacts = compilationArtifacts; + builder.objcProvider = objcProvider; + builder.testHost = testHost; + builder.inputsToXcodegen.addTransitive(inputsToXcodegen); + return builder; + } + + /** + * Returns a list of this provider and all its transitive dependencies. + */ + private Iterable<XcodeProvider> providers() { + Set<XcodeProvider> providers = new LinkedHashSet<>(); + providers.add(this); + Iterables.addAll(providers, dependencies); + for (XcodeProvider justTestHost : testHost.asSet()) { + providers.add(justTestHost); + Iterables.addAll(providers, justTestHost.dependencies); + } + return ImmutableList.copyOf(providers); + } + + private static final EnumSet<XcodeProductType> CAN_LINK_PRODUCT_TYPES = EnumSet.of( + XcodeProductType.APPLICATION, XcodeProductType.BUNDLE, XcodeProductType.UNIT_TEST); + + private TargetControl targetControl() { + String buildFilePath = label.getPackageFragment().getSafePathString() + "/BUILD"; + // TODO(bazel-team): Add provisioning profile information when Xcodegen supports it. + TargetControl.Builder targetControl = TargetControl.newBuilder() + .setName(label.getName()) + .setLabel(label.toString()) + .setProductType(productType.getIdentifier()) + .addAllImportedLibrary(Artifact.toExecPaths(objcProvider.get(IMPORTED_LIBRARY))) + .addAllUserHeaderSearchPath(userHeaderSearchPaths) + .addAllHeaderSearchPath(headerSearchPaths) + .addAllSupportFile(Artifact.toExecPaths(headers)) + .addAllCopt(compilationModeCopts) + .addAllCopt(Interspersing.prependEach("-D", objcProvider.get(DEFINE))) + .addAllCopt(copts) + .addAllLinkopt( + Interspersing.beforeEach("-force_load", objcProvider.get(FORCE_LOAD_FOR_XCODEGEN))) + .addAllLinkopt(IosSdkCommands.DEFAULT_LINKER_FLAGS) + .addAllLinkopt(Interspersing.beforeEach( + "-weak_framework", SdkFramework.names(objcProvider.get(WEAK_SDK_FRAMEWORK)))) + .addAllBuildSetting(xcodeprojBuildSettings) + .addAllBuildSetting(IosSdkCommands.defaultWarningsForXcode()) + .addAllSdkFramework(SdkFramework.names(objcProvider.get(SDK_FRAMEWORK))) + .addAllFramework(PathFragment.safePathStrings(objcProvider.get(FRAMEWORK_DIR))) + .addAllXcassetsDir(PathFragment.safePathStrings(objcProvider.get(XCASSETS_DIR))) + .addAllXcdatamodel(PathFragment.safePathStrings( + Xcdatamodel.xcdatamodelDirs(objcProvider.get(XCDATAMODEL)))) + .addAllBundleImport(PathFragment.safePathStrings(objcProvider.get(BUNDLE_IMPORT_DIR))) + .addAllSdkDylib(objcProvider.get(SDK_DYLIB)) + .addAllGeneralResourceFile(Artifact.toExecPaths(objcProvider.get(GENERAL_RESOURCE_FILE))) + .addSupportFile(buildFilePath); + + if (CAN_LINK_PRODUCT_TYPES.contains(productType)) { + for (XcodeProvider dependency : dependencies) { + // Only add a library target to a binary's dependencies if it has source files to compile. + // Xcode cannot build targets without a source file in the PBXSourceFilesBuildPhase, so if + // such a target is present in the control file, it is only to get Xcodegen to put headers + // and resources not used by the final binary in the Project Navigator. + // + // The exception to this rule is the objc_bundle_library target. Bundles are generally used + // for resources and can lack a PBXSourceFilesBuildPhase in the project file and still be + // considered valid by Xcode. + boolean hasSources = dependency.compilationArtifacts.isPresent() + && dependency.compilationArtifacts.get().getArchive().isPresent(); + if (hasSources || (dependency.productType == XcodeProductType.BUNDLE)) { + targetControl.addDependency(DependencyControl.newBuilder() + .setTargetLabel(dependency.label.toString()) + .build()); + } + } + for (XcodeProvider justTestHost : testHost.asSet()) { + targetControl.addDependency(DependencyControl.newBuilder() + .setTargetLabel(justTestHost.label.toString()) + .setTestHost(true) + .build()); + } + } + + for (InfoplistMerging merging : infoplistMerging.asSet()) { + for (Artifact infoplist : merging.getPlistWithEverything().asSet()) { + targetControl.setInfoplist(infoplist.getExecPathString()); + } + } + for (CompilationArtifacts artifacts : compilationArtifacts.asSet()) { + targetControl + .addAllSourceFile(Artifact.toExecPaths(artifacts.getSrcs())) + .addAllNonArcSourceFile(Artifact.toExecPaths(artifacts.getNonArcSrcs())); + + for (Artifact pchFile : artifacts.getPchFile().asSet()) { + targetControl + .setPchPath(pchFile.getExecPathString()) + .addSupportFile(pchFile.getExecPathString()); + } + } + + if (objcProvider.is(Flag.USES_CPP)) { + targetControl.addSdkDylib("libc++"); + } + + return targetControl.build(); + } + + /** + * Prepends the given path to each path in {@code paths}. Empty paths are + * transformed to the value of {@code variable} rather than {@code variable + "/."} + */ + @VisibleForTesting + static Iterable<String> rootEach(final String prefix, Iterable<PathFragment> paths) { + Preconditions.checkArgument(prefix.startsWith("$"), + "prefix should start with a build setting variable like '$(NAME)': %s", prefix); + Preconditions.checkArgument(!prefix.endsWith("/"), + "prefix should not end with '/': %s", prefix); + return Iterables.transform(paths, new Function<PathFragment, String>() { + @Override + public String apply(PathFragment input) { + if (input.getSafePathString().equals(".")) { + return prefix; + } else { + return prefix + "/" + input.getSafePathString(); + } + } + }); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeSupport.java new file mode 100644 index 0000000..f64c6bd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XcodeSupport.java
@@ -0,0 +1,102 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + +import static com.google.devtools.build.lib.packages.ImplicitOutputsFunction.fromTemplates; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction; + +/** + * Support for Objc rule types that export an Xcode provider or generate xcode project files. + * + * <p>Methods on this class can be called in any order without impacting the result. + */ +public final class XcodeSupport { + + /** + * Template for a target's xcode project. + */ + public static final SafeImplicitOutputsFunction PBXPROJ = + fromTemplates("%{name}.xcodeproj/project.pbxproj"); + + private final RuleContext ruleContext; + + /** + * Creates a new xcode support for the given context. + */ + XcodeSupport(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + /** + * Adds xcode project files to the given builder. + * + * @return this xcode support + */ + XcodeSupport addFilesToBuild(NestedSetBuilder<Artifact> filesToBuild) { + filesToBuild.add(ruleContext.getImplicitOutputArtifact(PBXPROJ)); + return this; + } + + /** + * Registers actions that generate the rule's Xcode project. + * + * @param xcodeProvider information about this rule's xcode settings and that of its dependencies + * @return this xcode support + */ + XcodeSupport registerActions(XcodeProvider xcodeProvider) { + ObjcActionsBuilder actionsBuilder = ObjcRuleClasses.actionsBuilder(ruleContext); + actionsBuilder.registerXcodegenActions( + new ObjcRuleClasses.Tools(ruleContext), + ruleContext.getImplicitOutputArtifact(XcodeSupport.PBXPROJ), + XcodeProvider.Project.fromTopLevelTarget(xcodeProvider)); + return this; + } + + /** + * Adds common xcode settings to the given provider builder. + * + * @param objcProvider provider containing all dependencies' information as well as some of this + * rule's + * @param productType type of this rule's Xcode target + * + * @return this xcode support + */ + XcodeSupport addXcodeSettings(XcodeProvider.Builder xcodeProviderBuilder, + ObjcProvider objcProvider, XcodeProductType productType) { + xcodeProviderBuilder + .setLabel(ruleContext.getLabel()) + .setObjcProvider(objcProvider) + .setProductType(productType); + return this; + } + + /** + * Adds dependencies to the given provider builder from the {@code deps} and {@code bundles} + * attributes. + * + * @return this xcode support + */ + XcodeSupport addDependencies(XcodeProvider.Builder xcodeProviderBuilder) { + xcodeProviderBuilder + .addDependencies(ruleContext.getPrerequisites("deps", Mode.TARGET, XcodeProvider.class)) + .addDependencies(ruleContext.getPrerequisites("bundles", Mode.TARGET, XcodeProvider.class)); + return this; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/XibFiles.java b/src/main/java/com/google/devtools/build/lib/rules/objc/XibFiles.java new file mode 100644 index 0000000..9be1d06 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/XibFiles.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.objc; + + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; + +/** + * A sequence of xib source files. Each {@code .xib} file can be compiled to a {@code .nib} file or + * directory. Because it might be a directory, we always use zip files to store the output and use + * the {@code actooloribtoolzip} utility to run ibtool and zip the output. + */ +public final class XibFiles extends IterableWrapper<Artifact> { + public XibFiles(Iterable<Artifact> artifacts) { + super(artifacts); + } + + /** + * Returns a sequence where each element of this sequence is converted to the file which contains + * the compiled contents of the xib. + */ + public ImmutableList<Artifact> compiledZips(IntermediateArtifacts intermediateArtifacts) { + ImmutableList.Builder<Artifact> zips = new ImmutableList.Builder<>(); + for (Artifact xib : this) { + zips.add(intermediateArtifacts.compiledXibFileZip(xib)); + } + return zips.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoSourcesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoSourcesProvider.java new file mode 100644 index 0000000..0663f62 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoSourcesProvider.java
@@ -0,0 +1,66 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.proto; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Configured target classes that implement this class can contribute .proto files to the + * compilation of proto_library rules. + */ +@Immutable +public final class ProtoSourcesProvider implements TransitiveInfoProvider { + + private final NestedSet<Artifact> transitiveImports; + private final NestedSet<Artifact> transitiveProtoSources; + private final ImmutableList<Artifact> protoSources; + + public ProtoSourcesProvider(NestedSet<Artifact> transitiveImports, + NestedSet<Artifact> transitiveProtoSources, + ImmutableList<Artifact> protoSources) { + this.transitiveImports = transitiveImports; + this.transitiveProtoSources = transitiveProtoSources; + this.protoSources = protoSources; + } + + /** + * Transitive imports including weak dependencies + * This determines the order of "-I" arguments to the protocol compiler, and + * that is probably important + */ + public NestedSet<Artifact> getTransitiveImports() { + return transitiveImports; + } + + /** + * Returns the proto sources for this rule and all its dependent protocol + * buffer rules. + */ + public NestedSet<Artifact> getTransitiveProtoSources() { + return transitiveProtoSources; + } + + /** + * Returns the proto sources from the 'srcs' attribute. If the library is a proxy library + * that has no sources, return the sources from the direct deps. + */ + public ImmutableList<Artifact> getProtoSources() { + return protoSources; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageAction.java b/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageAction.java new file mode 100644 index 0000000..6a19f92 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageAction.java
@@ -0,0 +1,132 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Fingerprint; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +/** + * Generates baseline (empty) coverage for the given non-test target. + */ +public class BaselineCoverageAction extends AbstractFileWriteAction + implements NotifyOnActionCacheHit { + // TODO(bazel-team): Remove this list of languages by separately collecting offline and online + // instrumented files. + private static final List<String> OFFLINE_INSTRUMENTATION_SUFFIXES = ImmutableList.of( + ".c", ".cc", ".cpp", ".dart", ".go", ".h", ".java", ".py"); + private final Iterable<Artifact> instrumentedFiles; + + private BaselineCoverageAction( + ActionOwner owner, Iterable<Artifact> instrumentedFiles, Artifact output) { + super(owner, ImmutableList.<Artifact>of(), output, false); + this.instrumentedFiles = instrumentedFiles; + } + + @Override + public String getMnemonic() { + return "BaselineCoverage"; + } + + @Override + public String computeKey() { + return new Fingerprint() + .addStrings(getInstrumentedFilePathStrings()) + .hexDigestAndReset(); + } + + private Iterable<String> getInstrumentedFilePathStrings() { + List<String> result = new ArrayList<>(); + for (Artifact instrumentedFile : instrumentedFiles) { + String pathString = instrumentedFile.getExecPathString(); + for (String suffix : OFFLINE_INSTRUMENTATION_SUFFIXES) { + if (pathString.endsWith(suffix)) { + result.add(pathString); + break; + } + } + } + + return result; + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, + Executor executor) { + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + PrintWriter writer = new PrintWriter(out); + for (String execPath : getInstrumentedFilePathStrings()) { + writer.write("SF:" + execPath + "\n"); + writer.write("end_of_record\n"); + } + writer.flush(); + } + }; + } + + @Override + protected void afterWrite(Executor executor) { + notifyAboutBaselineCoverage(executor.getEventBus()); + } + + @Override + public void actionCacheHit(Executor executor) { + notifyAboutBaselineCoverage(executor.getEventBus()); + } + + /** + * Notify interested parties about new baseline coverage data. + */ + private void notifyAboutBaselineCoverage(EventBus eventBus) { + Artifact output = Iterables.getOnlyElement(getOutputs()); + String ownerString = Label.print(getOwner().getLabel()); + eventBus.post(new BaselineCoverageResult(output, ownerString)); + } + + /** + * Returns collection of baseline coverage artifacts associated with the given target. + * Will always return 0 or 1 elements. + */ + public static ImmutableList<Artifact> getBaselineCoverageArtifacts(RuleContext ruleContext, + Iterable<Artifact> instrumentedFiles) { + // Baseline coverage artifacts will still go into "testlogs" directory. + Artifact coverageData = ruleContext.getAnalysisEnvironment().getDerivedArtifact( + Util.getWorkspaceRelativePath(ruleContext.getTarget()).getChild("baseline_coverage.dat"), + ruleContext.getConfiguration().getTestLogsDirectory()); + ruleContext.registerAction(new BaselineCoverageAction( + ruleContext.getActionOwner(), instrumentedFiles, coverageData)); + + return ImmutableList.of(coverageData); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageResult.java b/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageResult.java new file mode 100644 index 0000000..4af2df0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/BaselineCoverageResult.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; + +/** + * This event is used to notify about a successfully built baseline coverage artifact. + */ +public class BaselineCoverageResult { + + private final Artifact baselineCoverageData; + private final String ownerString; + + public BaselineCoverageResult(Artifact baselineCoverageData, String ownerString) { + this.baselineCoverageData = Preconditions.checkNotNull(baselineCoverageData); + this.ownerString = Preconditions.checkNotNull(ownerString); + } + + public Artifact getArtifact() { + return baselineCoverageData; + } + + public String getOwnerString() { + return ownerString; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/CoverageReportActionFactory.java b/src/main/java/com/google/devtools/build/lib/rules/test/CoverageReportActionFactory.java new file mode 100644 index 0000000..5f7571a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/CoverageReportActionFactory.java
@@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; + +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A factory class to create coverage report actions. + */ +public interface CoverageReportActionFactory { + + /** + * Returns a coverage report Action. May return null if it's not necessary to create + * such an Action based on the input parameters and some other data available to + * the factory implementation, such as command line arguments. + */ + @Nullable + public Action createCoverageReportAction(Iterable<ConfiguredTarget> targetsToTest, + Set<Artifact> baselineCoverageArtifacts, + ArtifactFactory artifactFactory, ArtifactOwner artifactOwner); +} \ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/ExclusiveTestStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/test/ExclusiveTestStrategy.java new file mode 100644 index 0000000..3cb5750 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/ExclusiveTestStrategy.java
@@ -0,0 +1,55 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.view.test.TestStatus.TestResultData; + +import java.io.IOException; + +/** + * Test strategy wrapper called 'exclusive'. It should delegate to a test strategy for local + * execution. The name 'exclusive' triggers behavior it triggers behavior in + * SkyframeExecutor to schedule test execution sequentially after non-test actions. This + * ensures streamed test output is not polluted by other action output. + */ +@ExecutionStrategy(contextType = TestActionContext.class, + name = { "exclusive" }) +public class ExclusiveTestStrategy implements TestActionContext { + private TestActionContext parent; + + public ExclusiveTestStrategy(TestActionContext parent) { + this.parent = parent; + } + + @Override + public void exec(TestRunnerAction action, + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException { + parent.exec(action, actionExecutionContext); + } + + @Override + public TestResult newCachedTestResult( + Path execRoot, TestRunnerAction action, TestResultData cached) throws IOException { + return parent.newCachedTestResult(execRoot, action, cached); + } + + @Override + public String strategyLocality(TestRunnerAction testRunnerAction) { + return "exclusive"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/ExecutionInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/test/ExecutionInfoProvider.java new file mode 100644 index 0000000..6c0d73c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/ExecutionInfoProvider.java
@@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.Map; + +/** + * This provider can be implemented by rules which need special environments to run in (especially + * tests). + */ +@Immutable +public final class ExecutionInfoProvider implements TransitiveInfoProvider { + + private final ImmutableMap<String, String> executionInfo; + + public ExecutionInfoProvider(Map<String, String> requirements) { + this.executionInfo = ImmutableMap.copyOf(requirements); + } + + /** + * Returns a map to indicate special execution requirements, such as hardware + * platforms, web browsers, etc. Rule tags, such as "requires-XXX", may also be added + * as keys to the map. + */ + public ImmutableMap<String, String> getExecutionInfo() { + return executionInfo; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFileManifestAction.java b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFileManifestAction.java new file mode 100644 index 0000000..e5ab219 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFileManifestAction.java
@@ -0,0 +1,133 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.RegexFilter; +import com.google.devtools.build.lib.vfs.FileSystemUtils; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.Arrays; +import java.util.Collection; + +/** + * Creates instrumented file manifest to list instrumented source files. + */ +class InstrumentedFileManifestAction extends AbstractFileWriteAction { + + private static final String GUID = "d9ddb800-f9a1-01Da-238d-988311a8475b"; + + private final Collection<Artifact> collectedSourceFiles; + private final Collection<Artifact> metadataFiles; + private final RegexFilter instrumentationFilter; + + private InstrumentedFileManifestAction(ActionOwner owner, Collection<Artifact> inputs, + Collection<Artifact> additionalSourceFiles, Collection<Artifact> gcnoFiles, + Artifact output, RegexFilter instrumentationFilter) { + super(owner, inputs, output, false); + this.collectedSourceFiles = additionalSourceFiles; + this.metadataFiles = gcnoFiles; + this.instrumentationFilter = instrumentationFilter; + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) { + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + Writer writer = null; + try { + // Save exec paths for both instrumented source files and gcno files in the manifest + // in the naturally sorted order. + String[] fileNames = Iterables.toArray(Iterables.transform( + Iterables.concat(collectedSourceFiles, metadataFiles), + new Function<Artifact, String> () { + @Override + public String apply(Artifact artifact) { return artifact.getExecPathString(); } + }), String.class); + Arrays.sort(fileNames); + writer = new OutputStreamWriter(out, ISO_8859_1); + for (String name : fileNames) { + writer.write(name); + writer.write('\n'); + } + } finally { + if (writer != null) { + writer.close(); + } + } + } + }; + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addString(instrumentationFilter.toString()); + return f.hexDigestAndReset(); + } + + /** + * Instantiates instrumented file manifest for the given target. + * + * @param ruleContext context of the executable configured target + * @param additionalSourceFiles additional instrumented source files, as + * collected by the {@link InstrumentedFilesCollector} + * @param metadataFiles *.gcno/*.em files collected by the {@link InstrumentedFilesCollector} + * @return instrumented file manifest artifact + */ + public static Artifact getInstrumentedFileManifest(final RuleContext ruleContext, + final Collection<Artifact> additionalSourceFiles, final Collection<Artifact> metadataFiles) { + // Instrumented manifest makes sense only for rules with binary output. + Preconditions.checkState(ruleContext.getRule().hasBinaryOutput()); + final Artifact instrumentedFileManifest = + ruleContext.getAnalysisEnvironment().getDerivedArtifact( + // Do not use replaceExtension(), as we may get name conflicts (two target-names have the + // same base name and only differ by extension). + FileSystemUtils.appendExtension( + Util.getWorkspaceRelativePath(ruleContext.getTarget()), ".instrumented_files"), + ruleContext.getConfiguration().getBinDirectory()); + + // Instrumented manifest artifact might already exist in case when multiple test + // actions that use slightly different subsets of runfiles set are generated for the same rule. + // So check whether we need to create a new action instance. + ImmutableList<Artifact> inputs = ImmutableList.<Artifact>builder() + .addAll(additionalSourceFiles) + .addAll(metadataFiles) + .build(); + ruleContext.registerAction(new InstrumentedFileManifestAction( + ruleContext.getActionOwner(), inputs, additionalSourceFiles, metadataFiles, + instrumentedFileManifest, ruleContext.getConfiguration().getInstrumentationFilter())); + + return instrumentedFileManifest; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesCollector.java b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesCollector.java new file mode 100644 index 0000000..e62a3b8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesCollector.java
@@ -0,0 +1,211 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * A helper class for collecting instrumented files and metadata for a target. + */ +public final class InstrumentedFilesCollector { + + /** + * The set of file types and attributes to visit to collect instrumented files for a certain rule + * type. The class is intentionally immutable, so that a single instance is sufficient for all + * rules of the same type (and in some cases all rules of related types, such as all {@code foo_*} + * rules). + */ + @Immutable + public static final class InstrumentationSpec { + private final FileTypeSet instrumentedFileTypes; + private final Collection<String> instrumentedAttributes; + + public InstrumentationSpec(FileTypeSet instrumentedFileTypes, + Collection<String> instrumentedAttributes) { + this.instrumentedFileTypes = instrumentedFileTypes; + this.instrumentedAttributes = ImmutableList.copyOf(instrumentedAttributes); + } + + public InstrumentationSpec(FileTypeSet instrumentedFileTypes, + String... instrumentedAttributes) { + this(instrumentedFileTypes, ImmutableList.copyOf(instrumentedAttributes)); + } + + /** + * Returns a new instrumentation spec with the given attribute names replacing the ones + * stored in this object. + */ + public InstrumentationSpec withAttributes(String... instrumentedAttributes) { + return new InstrumentationSpec(instrumentedFileTypes, instrumentedAttributes); + } + } + + /** + * The implementation for the local metadata collection. The intention is that implementations + * recurse over the locally (i.e., for that configured target) created actions and collect + * metadata files. + */ + public abstract static class LocalMetadataCollector { + /** + * Recursively runs over the local actions and add metadata files to the metadataFilesBuilder. + */ + public abstract void collectMetadataArtifacts( + Iterable<Artifact> artifacts, AnalysisEnvironment analysisEnvironment, + NestedSetBuilder<Artifact> metadataFilesBuilder); + + /** + * Adds action output of a particular type to metadata files. + * + * <p>Only adds the first output that matches the given file type. + * + * @param metadataFilesBuilder builder to collect metadata files + * @param action the action whose outputs to scan + * @param fileType the filetype of outputs which should be collected + */ + protected void addOutputs(NestedSetBuilder<Artifact> metadataFilesBuilder, + Action action, FileType fileType) { + for (Artifact output : action.getOutputs()) { + if (fileType.matches(output.getFilename())) { + metadataFilesBuilder.add(output); + break; + } + } + } + } + + /** + * Only collects files transitively from srcs, deps, and data attributes. + */ + public static final InstrumentationSpec TRANSITIVE_COLLECTION_SPEC = new InstrumentationSpec( + FileTypeSet.NO_FILE, + "srcs", "deps", "data"); + + /** + * An explicit constant for a {@link LocalMetadataCollector} that doesn't collect anything. + */ + public static final LocalMetadataCollector NO_METADATA_COLLECTOR = null; + + private final RuleContext ruleContext; + private final InstrumentationSpec spec; + private final LocalMetadataCollector localMetadataCollector; + private final NestedSet<Artifact> instrumentationMetadataFiles; + private final NestedSet<Artifact> instrumentedFiles; + + public InstrumentedFilesCollector(RuleContext ruleContext, InstrumentationSpec spec, + LocalMetadataCollector localMetadataCollector, Iterable<Artifact> rootFiles) { + this.ruleContext = ruleContext; + this.spec = spec; + this.localMetadataCollector = localMetadataCollector; + Preconditions.checkNotNull(ruleContext, "RuleContext already cleared. That means that the" + + " collector data was already memoized. You do not have to call it again."); + if (!ruleContext.getConfiguration().isCodeCoverageEnabled()) { + instrumentedFiles = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + instrumentationMetadataFiles = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } else { + NestedSetBuilder<Artifact> instrumentedFilesBuilder = + NestedSetBuilder.stableOrder(); + NestedSetBuilder<Artifact> metadataFilesBuilder = NestedSetBuilder.stableOrder(); + collect(ruleContext.getAnalysisEnvironment(), instrumentedFilesBuilder, metadataFilesBuilder, + rootFiles); + instrumentedFiles = instrumentedFilesBuilder.build(); + instrumentationMetadataFiles = metadataFilesBuilder.build(); + } + } + + /** + * Returns instrumented source files for the target provided during construction. + */ + public final NestedSet<Artifact> getInstrumentedFiles() { + return instrumentedFiles; + } + + /** + * Returns instrumentation metadata files for the target provided during construction. + */ + public final NestedSet<Artifact> getInstrumentationMetadataFiles() { + return instrumentationMetadataFiles; + } + + /** + * Collects instrumented files and metadata files. + */ + private void collect(AnalysisEnvironment analysisEnvironment, + NestedSetBuilder<Artifact> instrumentedFilesBuilder, + NestedSetBuilder<Artifact> metadataFilesBuilder, + Iterable<Artifact> rootFiles) { + for (TransitiveInfoCollection dep : getAllPrerequisites()) { + InstrumentedFilesProvider provider = dep.getProvider(InstrumentedFilesProvider.class); + if (provider != null) { + instrumentedFilesBuilder.addTransitive(provider.getInstrumentedFiles()); + metadataFilesBuilder.addTransitive(provider.getInstrumentationMetadataFiles()); + } else if (shouldIncludeLocalSources()) { + for (Artifact artifact : dep.getProvider(FileProvider.class).getFilesToBuild()) { + if (artifact.isSourceArtifact() && + spec.instrumentedFileTypes.matches(artifact.getFilename())) { + instrumentedFilesBuilder.add(artifact); + } + } + } + } + + if (localMetadataCollector != null) { + localMetadataCollector.collectMetadataArtifacts(rootFiles, + analysisEnvironment, metadataFilesBuilder); + } + } + + /** + * Returns the list of attributes which should be (transitively) checked for sources and + * instrumentation metadata. + */ + private Collection<String> getSourceAttributes() { + return spec.instrumentedAttributes; + } + + private boolean shouldIncludeLocalSources() { + return ruleContext.getConfiguration().getInstrumentationFilter().isIncluded( + ruleContext.getLabel().toString()); + } + + private Iterable<TransitiveInfoCollection> getAllPrerequisites() { + List<TransitiveInfoCollection> prerequisites = new ArrayList<>(); + for (String attr : getSourceAttributes()) { + if (ruleContext.getRule().isAttrDefined(attr, Type.LABEL_LIST) || + ruleContext.getRule().isAttrDefined(attr, Type.LABEL)) { + Iterables.addAll(prerequisites, ruleContext.getPrerequisites(attr, Mode.DONT_CHECK)); + } + } + return prerequisites; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProvider.java new file mode 100644 index 0000000..b1f956c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProvider.java
@@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; + +/** + * A provider of instrumented file sources and instrumentation metadata. + */ +public interface InstrumentedFilesProvider extends TransitiveInfoProvider { + + /** + * Returns a collection of source files for instrumented binaries. + */ + NestedSet<Artifact> getInstrumentedFiles(); + + /** + * Returns a collection of instrumentation metadata files. + */ + NestedSet<Artifact> getInstrumentationMetadataFiles(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProviderImpl.java b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProviderImpl.java new file mode 100644 index 0000000..1452e2d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/InstrumentedFilesProviderImpl.java
@@ -0,0 +1,53 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; + +/** + * An implementation class for the InstrumentedFilesProvider interface. + */ +public final class InstrumentedFilesProviderImpl implements InstrumentedFilesProvider { + public static final InstrumentedFilesProvider EMPTY = new InstrumentedFilesProvider() { + @Override + public NestedSet<Artifact> getInstrumentedFiles() { + return NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER); + } + @Override + public NestedSet<Artifact> getInstrumentationMetadataFiles() { + return NestedSetBuilder.<Artifact>emptySet(Order.STABLE_ORDER); + } + }; + + private final NestedSet<Artifact> instrumentedFiles; + private final NestedSet<Artifact> instrumentationMetadataFiles; + + public InstrumentedFilesProviderImpl(InstrumentedFilesCollector collector) { + this.instrumentedFiles = collector.getInstrumentedFiles(); + this.instrumentationMetadataFiles = collector.getInstrumentationMetadataFiles(); + } + + @Override + public NestedSet<Artifact> getInstrumentedFiles() { + return instrumentedFiles; + } + + @Override + public NestedSet<Artifact> getInstrumentationMetadataFiles() { + return instrumentationMetadataFiles; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java new file mode 100644 index 0000000..006f789 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java
@@ -0,0 +1,224 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.TestExecException; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; +import com.google.devtools.build.lib.view.test.TestStatus.TestCase; +import com.google.devtools.build.lib.view.test.TestStatus.TestResultData; +import com.google.devtools.common.options.OptionsClassProvider; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Runs TestRunnerAction actions. + */ +@ExecutionStrategy(contextType = TestActionContext.class, + name = { "standalone" }) +public class StandaloneTestStrategy extends TestStrategy { + /* + TODO(bazel-team): + + * tests + * It would be nice to get rid of (cd $TEST_SRCDIR) in the test-setup script. + * test timeouts. + * parsing XML output. + + */ + protected final PathFragment runfilesPrefix; + + public StandaloneTestStrategy(OptionsClassProvider requestOptions, + OptionsClassProvider startupOptions, BinTools binTools, PathFragment runfilesPrefix) { + super(requestOptions, startupOptions, binTools); + + this.runfilesPrefix = runfilesPrefix; + } + + private static final String TEST_SETUP = "tools/test/test-setup.sh"; + + @Override + public void exec(TestRunnerAction action, ActionExecutionContext actionExecutionContext) + throws ExecException, InterruptedException { + Path runfilesDir = null; + try { + runfilesDir = TestStrategy.getLocalRunfilesDirectory( + action, actionExecutionContext, binTools); + } catch (ExecException e) { + throw new TestExecException(e.getMessage()); + } + + Path workingDirectory = runfilesDir.getRelative(runfilesPrefix); + Map<String, String> env = getEnv(action, runfilesDir); + Spawn spawn = new BaseSpawn(getArgs(action), env, + action.getTestProperties().getExecutionInfo(), + action, + action.getTestProperties().getLocalResourceUsage()); + + Executor executor = actionExecutionContext.getExecutor(); + try { + FileSystemUtils.createDirectoryAndParents(workingDirectory); + FileOutErr fileOutErr = new FileOutErr(action.getTestLog().getPath(), + action.resolve(actionExecutionContext.getExecutor().getExecRoot()).getTestStderr()); + TestResultData data = execute( + actionExecutionContext.withFileOutErr(fileOutErr), spawn, action); + appendStderr(fileOutErr.getOutputFile(), fileOutErr.getErrorFile()); + finalizeTest(actionExecutionContext, action, data); + } catch (IOException e) { + executor.getEventHandler().handle(Event.error("Caught I/O exception: " + e)); + throw new EnvironmentalExecException("unexpected I/O exception", e); + } + } + + private Map<String, String> getEnv(TestRunnerAction action, Path runfilesDir) { + Map<String, String> vars = getDefaultTestEnvironment(action); + BuildConfiguration config = action.getConfiguration(); + + vars.putAll(config.getDefaultShellEnvironment()); + vars.putAll(config.getTestEnv()); + vars.put("TEST_SRCDIR", runfilesDir.getRelative(runfilesPrefix).getPathString()); + + // TODO(bazel-team): set TEST_TMPDIR. + + return vars; + } + + private TestResultData execute( + ActionExecutionContext actionExecutionContext, Spawn spawn, TestRunnerAction action) + throws TestExecException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + Closeable streamed = null; + Path testLogPath = action.getTestLog().getPath(); + TestResultData.Builder builder = TestResultData.newBuilder(); + + try { + try { + if (executionOptions.testOutput.equals(TestOutputFormat.STREAMED)) { + streamed = new StreamedTestOutput( + Reporter.outErrForReporter( + actionExecutionContext.getExecutor().getEventHandler()), testLogPath); + } + executor.getSpawnActionContext(action.getMnemonic()).exec(spawn, actionExecutionContext); + + builder.setTestPassed(true) + .setStatus(BlazeTestStatus.PASSED) + .setCachable(true); + } catch (ExecException e) { + // Execution failed, which we consider a test failure. + + // TODO(bazel-team): set cachable==true for relevant statuses (failure, but not for + // timeout, etc.) + builder.setTestPassed(false) + .setStatus(BlazeTestStatus.FAILED); + } finally { + if (streamed != null) { + streamed.close(); + } + } + + TestCase details = parseTestResult( + action.resolve(actionExecutionContext.getExecutor().getExecRoot()).getXmlOutputPath()); + if (details != null) { + builder.setTestCase(details); + } + + return builder.build(); + } catch (IOException e) { + throw new TestExecException(e.getMessage()); + } + } + + /** + * Outputs test result to the stdout after test has finished (e.g. for --test_output=all or + * --test_output=errors). Will also try to group output lines together (up to 10000 lines) so + * parallel test outputs will not get interleaved. + */ + protected void processTestOutput(Executor executor, FileOutErr outErr, TestResult result) + throws IOException { + Path testOutput = executor.getExecRoot().getRelative(result.getTestLogPath().asFragment()); + boolean isPassed = result.getData().getTestPassed(); + try { + if (TestLogHelper.shouldOutputTestLog(executionOptions.testOutput, isPassed)) { + TestLogHelper.writeTestLog(testOutput, result.getTestName(), outErr.getOutputStream()); + } + } finally { + if (isPassed) { + executor.getEventHandler().handle(new Event(EventKind.PASS, null, result.getTestName())); + } else { + if (result.getData().getStatus() == BlazeTestStatus.TIMEOUT) { + executor.getEventHandler().handle( + new Event(EventKind.TIMEOUT, null, result.getTestName() + + " (see " + testOutput + ")")); + } else { + executor.getEventHandler().handle( + new Event(EventKind.FAIL, null, result.getTestName() + " (see " + testOutput + ")")); + } + } + } + } + + private final void finalizeTest(ActionExecutionContext actionExecutionContext, + TestRunnerAction action, TestResultData data) throws IOException, ExecException { + TestResult result = new TestResult(action, data, false); + postTestResult(actionExecutionContext.getExecutor(), result); + + processTestOutput(actionExecutionContext.getExecutor(), + actionExecutionContext.getFileOutErr(), result); + // TODO(bazel-team): handle --test_output=errors, --test_output=all. + + if (!executionOptions.testKeepGoing && data.getStatus() != BlazeTestStatus.PASSED) { + throw new TestExecException("Test failed: aborting"); + } + } + + private List<String> getArgs(TestRunnerAction action) { + List<String> args = Lists.newArrayList(TEST_SETUP); + TestTargetExecutionSettings execSettings = action.getExecutionSettings(); + + // Execute the test using the alias in the runfiles tree. + args.add(execSettings.getExecutable().getRootRelativePath().getPathString()); + args.addAll(execSettings.getArgs()); + + return args; + } + + @Override + public String strategyLocality(TestRunnerAction action) { return "standalone"; } + + @Override + public TestResult newCachedTestResult( + Path execRoot, TestRunnerAction action, TestResultData data) { + return new TestResult(action, data, /*cached*/ true); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestActionBuilder.java new file mode 100644 index 0000000..2ac9a0f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestActionBuilder.java
@@ -0,0 +1,270 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.TestSize; +import com.google.devtools.build.lib.packages.TestTimeout; +import com.google.devtools.build.lib.rules.test.TestProvider.TestParams; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.EnumConverter; + +import java.util.Collection; +import java.util.List; + +import javax.annotation.Nullable; + +/** + * Helper class to create test actions. + */ +public final class TestActionBuilder { + + private final RuleContext ruleContext; + private RunfilesSupport runfilesSupport; + private Artifact executable; + private ExecutionInfoProvider executionRequirements; + private InstrumentedFilesProvider instrumentedFiles; + private int explicitShardCount; + + public TestActionBuilder(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + /** + * Creates the test actions and artifacts using the previously set parameters. + * + * @return ordered list of test status artifacts + */ + public TestParams build() { + Preconditions.checkState(runfilesSupport != null); + boolean local = TargetUtils.isTestRuleAndRunsLocally(ruleContext.getRule()); + TestShardingStrategy strategy = ruleContext.getConfiguration().testShardingStrategy(); + int shards = strategy.getNumberOfShards( + local, explicitShardCount, isTestShardingCompliant(), + TestSize.getTestSize(ruleContext.getRule())); + Preconditions.checkState(shards >= 0); + return createTestAction(Util.getWorkspaceRelativePath(ruleContext.getLabel()), shards); + } + + private boolean isTestShardingCompliant() { + // See if it has a data dependency on the special target + // //tools:test_sharding_compliant. Test runners add this dependency + // to show they speak the sharding protocol. + // There are certain cases where this heuristic may fail, giving + // a "false positive" (where we shard the test even though the + // it isn't supported). We may want to refine this logic, but + // heuristically sharding is currently experimental. Also, we do detect + // false-positive cases and return an error. + return runfilesSupport.getRunfilesSymlinkNames().contains( + new PathFragment("tools/test_sharding_compliant")); + } + + /** + * Set the runfiles and executable to be run as a test. + */ + public TestActionBuilder setFilesToRunProvider(FilesToRunProvider provider) { + Preconditions.checkNotNull(provider.getRunfilesSupport()); + Preconditions.checkNotNull(provider.getExecutable()); + this.runfilesSupport = provider.getRunfilesSupport(); + this.executable = provider.getExecutable(); + return this; + } + + public TestActionBuilder setInstrumentedFiles( + @Nullable InstrumentedFilesProvider instrumentedFiles) { + this.instrumentedFiles = instrumentedFiles; + return this; + } + + public TestActionBuilder setExecutionRequirements( + @Nullable ExecutionInfoProvider executionRequirements) { + this.executionRequirements = executionRequirements; + return this; + } + + /** + * Set the explicit shard count. Note that this may be overridden by the sharding strategy. + */ + public TestActionBuilder setShardCount(int explicitShardCount) { + this.explicitShardCount = explicitShardCount; + return this; + } + + /** + * Converts to {@link TestActionBuilder.TestShardingStrategy}. + */ + public static class ShardingStrategyConverter extends EnumConverter<TestShardingStrategy> { + public ShardingStrategyConverter() { + super(TestShardingStrategy.class, "test sharding strategy"); + } + } + + /** + * A strategy for running the same tests in many processes. + */ + public static enum TestShardingStrategy { + EXPLICIT { + @Override public int getNumberOfShards(boolean isLocal, int shardCountFromAttr, + boolean testShardingCompliant, TestSize testSize) { + return Math.max(shardCountFromAttr, 0); + } + }, + + EXPERIMENTAL_HEURISTIC { + @Override public int getNumberOfShards(boolean isLocal, int shardCountFromAttr, + boolean testShardingCompliant, TestSize testSize) { + if (shardCountFromAttr >= 0) { + return shardCountFromAttr; + } + if (isLocal || !testShardingCompliant) { + return 0; + } + return testSize.getDefaultShards(); + } + }, + + DISABLED { + @Override public int getNumberOfShards(boolean isLocal, int shardCountFromAttr, + boolean testShardingCompliant, TestSize testSize) { + return 0; + } + }; + + public abstract int getNumberOfShards(boolean isLocal, int shardCountFromAttr, + boolean testShardingCompliant, TestSize testSize); + } + + /** + * Creates a test action and artifacts for the given rule. The test action will + * use the specified executable and runfiles. + * + * @param targetName the relative path of the target to run + * @return ordered list of test artifacts, one per action. These are used to drive + * execution in Skyframe, and by AggregatingTestListener and + * TestResultAnalyzer to keep track of completed and pending test runs. + */ + private TestParams createTestAction(PathFragment targetName, int shards) { + BuildConfiguration config = ruleContext.getConfiguration(); + AnalysisEnvironment env = ruleContext.getAnalysisEnvironment(); + Root root = config.getTestLogsDirectory(); + + NestedSetBuilder<Artifact> inputsBuilder = NestedSetBuilder.stableOrder(); + inputsBuilder.addTransitive( + NestedSetBuilder.create(Order.STABLE_ORDER, runfilesSupport.getRunfilesMiddleman())); + for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("$test_runtime", Mode.HOST)) { + inputsBuilder.addTransitive(dep.getProvider(FileProvider.class).getFilesToBuild()); + } + TestTargetProperties testProperties = new TestTargetProperties( + ruleContext, executionRequirements); + + // If the test rule does not provide InstrumentedFilesProvider, there's not much that we can do. + final boolean collectCodeCoverage = config.isCodeCoverageEnabled() + && instrumentedFiles != null; + + TestTargetExecutionSettings executionSettings; + if (collectCodeCoverage) { + // Add instrumented file manifest artifact to the list of inputs. This file will contain + // exec paths of all source files that should be included into the code coverage output. + Collection<Artifact> metadataFiles = + ImmutableList.copyOf(instrumentedFiles.getInstrumentationMetadataFiles()); + inputsBuilder.addTransitive(NestedSetBuilder.wrap(Order.STABLE_ORDER, metadataFiles)); + for (TransitiveInfoCollection dep : + ruleContext.getPrerequisites(":coverage_support", Mode.HOST)) { + inputsBuilder.addTransitive(dep.getProvider(FileProvider.class).getFilesToBuild()); + } + Artifact instrumentedFileManifest = + InstrumentedFileManifestAction.getInstrumentedFileManifest(ruleContext, + ImmutableList.copyOf(instrumentedFiles.getInstrumentedFiles()), + metadataFiles); + executionSettings = new TestTargetExecutionSettings(ruleContext, runfilesSupport, + executable, instrumentedFileManifest, shards); + inputsBuilder.add(instrumentedFileManifest); + } else { + executionSettings = new TestTargetExecutionSettings(ruleContext, runfilesSupport, + executable, null, shards); + } + + if (config.getRunUnder() != null) { + Artifact runUnderExecutable = executionSettings.getRunUnderExecutable(); + if (runUnderExecutable != null) { + inputsBuilder.add(runUnderExecutable); + } + } + + int runsPerTest = config.getRunsPerTestForLabel(ruleContext.getLabel()); + + Iterable<Artifact> inputs = inputsBuilder.build(); + int shardRuns = (shards > 0 ? shards : 1); + List<Artifact> results = Lists.newArrayListWithCapacity(runsPerTest * shardRuns); + ImmutableList.Builder<Artifact> coverageArtifacts = ImmutableList.builder(); + + for (int run = 0; run < runsPerTest; run++) { + // Use a 1-based index for user friendliness. + String runSuffix = + runsPerTest > 1 ? String.format("_run_%d_of_%d", run + 1, runsPerTest) : ""; + for (int shard = 0; shard < shardRuns; shard++) { + String suffix = (shardRuns > 1 ? String.format("_shard_%d_of_%d", shard + 1, shards) : "") + + runSuffix; + Artifact testLog = env.getDerivedArtifact( + targetName.getChild("test" + suffix + ".log"), root); + Artifact cacheStatus = env.getDerivedArtifact( + targetName.getChild("test" + suffix + ".cache_status"), root); + + Artifact coverageArtifact = null; + if (collectCodeCoverage) { + coverageArtifact = + env.getDerivedArtifact(targetName.getChild("coverage" + suffix + ".dat"), root); + coverageArtifacts.add(coverageArtifact); + } + + Artifact microCoverageArtifact = null; + if (collectCodeCoverage && config.isMicroCoverageEnabled()) { + microCoverageArtifact = + env.getDerivedArtifact(targetName.getChild("coverage" + suffix + ".micro.dat"), root); + } + + env.registerAction(new TestRunnerAction( + ruleContext.getActionOwner(), inputs, + testLog, cacheStatus, + coverageArtifact, microCoverageArtifact, + testProperties, executionSettings, + shard, run, config)); + results.add(cacheStatus); + } + } + // TODO(bazel-team): Passing the reportGenerator to every TestParams is a bit strange. + Artifact reportGenerator = collectCodeCoverage + ? ruleContext.getPrerequisiteArtifact(":coverage_report_generator", Mode.HOST) : null; + return new TestParams(runsPerTest, shards, TestTimeout.getTestTimeout(ruleContext.getRule()), + ruleContext.getRule().getRuleClass(), ImmutableList.copyOf(results), + coverageArtifacts.build(), reportGenerator); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestActionContext.java new file mode 100644 index 0000000..7f7a916 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestActionContext.java
@@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.view.test.TestStatus.TestResultData; + +import java.io.IOException; + +/** + * A context for the execution of test actions ({@link TestRunnerAction}). + */ +public interface TestActionContext extends ActionContext { + + /** + * Executes the test command, directing standard out / err to {@code outErr}. The status of + * the test should be communicated by posting a {@link TestResult} object to the eventbus. + */ + void exec(TestRunnerAction action, + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException; + + /** + * String describing where the action will run. + */ + String strategyLocality(TestRunnerAction action); + + /** + * Creates a cached test result. + */ + TestResult newCachedTestResult(Path execRoot, TestRunnerAction action, TestResultData cached) + throws IOException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestLogHelper.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestLogHelper.java new file mode 100644 index 0000000..462c24c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestLogHelper.java
@@ -0,0 +1,141 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.BufferedOutputStream; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; + +/** + * A helper class for test log handling. It determines whether the test log should + * be output and formats the test log for console display. + */ +public class TestLogHelper { + + public static final String HEADER_DELIMITER = + "-----------------------------------------------------------------------------"; + + /** + * Determines whether the test log should be output from the current outputMode + * and whether the test has passed or not. + */ + public static boolean shouldOutputTestLog(TestOutputFormat outputMode, boolean hasPassed) { + return (outputMode == TestOutputFormat.ALL) || + (!hasPassed && (outputMode == TestOutputFormat.ERRORS)); + } + + /** + * Reads the contents of the test log from the provided testOutput file, adds + * header and footer and returns the result. + * This method also looks for a header delimiter and cuts off the text before it, + * except if the header is 50 lines or longer. + */ + public static void writeTestLog(Path testOutput, String testName, OutputStream out) + throws IOException { + InputStream input = null; + PrintStream printOut = new PrintStream(new BufferedOutputStream(out)); + try { + final String outputHeader = + "==================== Test output for " + testName + ":"; + final String outputFooter = + "================================================================================"; + + printOut.println(outputHeader); + printOut.flush(); + + input = testOutput.getInputStream(); + FilterTestHeaderOutputStream filteringOutputStream = getHeaderFilteringOutputStream(printOut); + ByteStreams.copy(input, filteringOutputStream); + + if (!filteringOutputStream.foundHeader()) { + InputStream inputAgain = testOutput.getInputStream(); + try { + ByteStreams.copy(inputAgain, out); + } finally { + inputAgain.close(); + } + } + + printOut.println(outputFooter); + } finally { + printOut.flush(); + if (input != null) { + input.close(); + } + } + } + + /** + * Returns an output stream that doesn't write to original until it + * sees HEADER_DELIMITER by itself on a line. + */ + public static FilterTestHeaderOutputStream getHeaderFilteringOutputStream(OutputStream original) { + return new FilterTestHeaderOutputStream(original); + } + + private TestLogHelper() { + // Prevent Java from creating a public constructor. + } + + /** + * Use this class to filter the streaming output of a test until we see the + * header delimiter. + */ + public static class FilterTestHeaderOutputStream extends FilterOutputStream { + + private boolean seenDelimiter = false; + private StringBuilder lineBuilder = new StringBuilder(); + + private static final int NEWLINE = '\n'; + + public FilterTestHeaderOutputStream(OutputStream out) { + super(out); + } + + @Override + public void write(int b) throws IOException { + if (seenDelimiter) { + out.write(b); + } else if (b == NEWLINE) { + String line = lineBuilder.toString(); + lineBuilder = new StringBuilder(); + if (line.equals(TestLogHelper.HEADER_DELIMITER)) { + seenDelimiter = true; + } + } else if (lineBuilder.length() <= TestLogHelper.HEADER_DELIMITER.length()) { + lineBuilder.append((char) b); + } + } + + @Override + public void write(byte b[], int off, int len) throws IOException { + if (seenDelimiter) { + out.write(b, off, len); + } else { + super.write(b, off, len); + } + } + + public boolean foundHeader() { + return seenDelimiter; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestProvider.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestProvider.java new file mode 100644 index 0000000..d24fe6b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestProvider.java
@@ -0,0 +1,143 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.packages.TestTimeout; + +import java.util.List; + +/** + * A {@link TransitiveInfoProvider} for configured targets that implement test rules. + */ +@Immutable +public final class TestProvider implements TransitiveInfoProvider { + private final TestParams testParams; + private final ImmutableList<String> testTags; + + public TestProvider(TestParams testParams, ImmutableList<String> testTags) { + this.testParams = testParams; + this.testTags = testTags; + } + + /** + * Returns the {@link TestParams} object for the test represented by the corresponding configured + * target. + */ + public TestParams getTestParams() { + return testParams; + } + + /** + * Temporary hack to allow dependencies on test_suite targets to continue to work for the time + * being. + */ + public List<String> getTestTags() { + return testTags; + } + + /** + * Returns the test status artifacts for a specified configured target + * + * @param target the configured target. Should belong to a test rule. + * @return the test status artifacts + */ + public static ImmutableList<Artifact> getTestStatusArtifacts(TransitiveInfoCollection target) { + return target.getProvider(TestProvider.class).getTestParams().getTestStatusArtifacts(); + } + + /** + * A value class describing the properties of a test. + */ + public static class TestParams { + private final int runs; + private final int shards; + private final TestTimeout timeout; + private final String testRuleClass; + private final ImmutableList<Artifact> testStatusArtifacts; + private final ImmutableList<Artifact> coverageArtifacts; + private final Artifact coverageReportGenerator; + + /** + * Don't call this directly. Instead use {@link TestActionBuilder}. + */ + TestParams(int runs, int shards, TestTimeout timeout, String testRuleClass, + ImmutableList<Artifact> testStatusArtifacts, + ImmutableList<Artifact> coverageArtifacts, + Artifact coverageReportGenerator) { + this.runs = runs; + this.shards = shards; + this.timeout = timeout; + this.testRuleClass = testRuleClass; + this.testStatusArtifacts = testStatusArtifacts; + this.coverageArtifacts = coverageArtifacts; + this.coverageReportGenerator = coverageReportGenerator; + } + + /** + * Returns the number of times this test should be run. + */ + public int getRuns() { + return runs; + } + + /** + * Returns the number of shards for this test. + */ + public int getShards() { + return shards; + } + + /** + * Returns the timeout of this test. + */ + public TestTimeout getTimeout() { + return timeout; + } + + /** + * Returns the test rule class. + */ + public String getTestRuleClass() { + return testRuleClass; + } + + /** + * Returns a list of test status artifacts that represent serialized test status protobuffers + * produced by testing this target. + */ + public ImmutableList<Artifact> getTestStatusArtifacts() { + return testStatusArtifacts; + } + + /** + * Returns the coverageArtifacts + */ + public ImmutableList<Artifact> getCoverageArtifacts() { + return coverageArtifacts; + } + + /** + * Returns the coverage report generator tool. + */ + public Artifact getCoverageReportGenerator() { + return coverageReportGenerator; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestResult.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestResult.java new file mode 100644 index 0000000..b0de6cd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestResult.java
@@ -0,0 +1,133 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; +import com.google.devtools.build.lib.view.test.TestStatus.TestResultData; + +/** + * This is the event passed from the various test strategies to the {@code RecordingTestListener} + * upon test completion. + */ +@ThreadSafe +@Immutable +public class TestResult { + + private final TestRunnerAction testAction; + private final TestResultData data; + private final boolean cached; + + /** + * Construct the TestResult for the given test / status. + * + * @param testAction The test that was run. + * @param data test result protobuffer. + * @param cached true if this is a cached test result. + */ + public TestResult(TestRunnerAction testAction, TestResultData data, boolean cached) { + this.testAction = Preconditions.checkNotNull(testAction); + this.data = data; + this.cached = cached; + } + + public static boolean isBlazeTestStatusPassed(BlazeTestStatus status) { + return status == BlazeTestStatus.PASSED || status == BlazeTestStatus.FLAKY; + } + + /** + * @return The test action. + */ + public TestRunnerAction getTestAction() { + return testAction; + } + + /** + * @return The test log path. Note, that actual log file may no longer + * correspond to this artifact - use getActualLogPath() method if + * you need log location. + */ + public Path getTestLogPath() { + return testAction.getTestLog().getPath(); + } + + /** + * Return if result was loaded from local action cache. + */ + public final boolean isCached() { + return cached; + } + + /** + * @return Coverage data artifact, if available and null otherwise. + */ + public PathFragment getCoverageData() { + if (data.getHasCoverage()) { + return testAction.getCoverageData().getExecPath(); + } + return null; + } + + /** + * @return The test status artifact. + */ + public Artifact getTestStatusArtifact() { + // these artifacts are used to keep track of the number of pending and completed tests. + return testAction.getCacheStatusArtifact(); + } + + + /** + * Gets the test name in a user-friendly format. + * Will generally include the target name and shard number, if applicable. + * + * @return The test name. + */ + public String getTestName() { + return testAction.getTestName(); + } + + /** + * @return The test label. + */ + public String getLabel() { + return Label.print(testAction.getOwner().getLabel()); + } + + /** + * @return The test shard number. + */ + public int getShardNum() { + return testAction.getShardNum(); + } + + /** + * @return Total number of test shards. 0 means + * no sharding, whereas 1 means degenerate sharding. + */ + public int getTotalShards() { + return testAction.getExecutionSettings().getTotalShards(); + } + + public TestResultData getData() { + return data; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestRunnerAction.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestRunnerAction.java new file mode 100644 index 0000000..28500a7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestRunnerAction.java
@@ -0,0 +1,607 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.RunUnder; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.test.TestStatus.TestResultData; +import com.google.devtools.common.options.TriState; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.logging.Level; + +import javax.annotation.Nullable; + +/** + * An Action representing a test with the associated environment (runfiles, + * environment variables, test result, etc). It consumes test executable and + * runfiles artifacts and produces test result and test status artifacts. + */ +// Not final so that we can mock it in tests. +public class TestRunnerAction extends AbstractAction implements NotifyOnActionCacheHit { + + private static final String GUID = "94857c93-f11c-4cbc-8c1b-e0a281633f9e"; + + private final BuildConfiguration configuration; + private final Artifact testLog; + private final Artifact cacheStatus; + private final PathFragment testWarningsPath; + private final PathFragment splitLogsPath; + private final PathFragment splitLogsDir; + private final PathFragment undeclaredOutputsDir; + private final PathFragment undeclaredOutputsZipPath; + private final PathFragment undeclaredOutputsAnnotationsDir; + private final PathFragment undeclaredOutputsManifestPath; + private final PathFragment undeclaredOutputsAnnotationsPath; + private final PathFragment xmlOutputPath; + @Nullable + private final PathFragment testShard; + private final PathFragment testExitSafe; + private final PathFragment testStderr; + private final PathFragment testInfrastructureFailure; + private final PathFragment baseDir; + private final String namePrefix; + private final Artifact coverageData; + private final Artifact microCoverageData; + private final TestTargetProperties testProperties; + private final TestTargetExecutionSettings executionSettings; + private final int shardNum; + private final int runNumber; + + // Mutable state related to test caching. + private boolean checkedCaching = false; + private boolean unconditionalExecution = false; + + private static ImmutableList<Artifact> list(Artifact... artifacts) { + ImmutableList.Builder<Artifact> builder = ImmutableList.builder(); + for (Artifact artifact : artifacts) { + if (artifact != null) { + builder.add(artifact); + } + } + return builder.build(); + } + + /** + * Create new TestRunnerAction instance. Should not be called directly. + * Use {@link TestActionBuilder} instead. + * + * @param shardNum The shard number. Must be 0 if totalShards == 0 + * (no sharding). Otherwise, must be >= 0 and < totalShards. + * @param runNumber test run number + */ + TestRunnerAction(ActionOwner owner, + Iterable<Artifact> inputs, + Artifact testLog, + Artifact cacheStatus, + Artifact coverageArtifact, + Artifact microCoverageArtifact, + TestTargetProperties testProperties, + TestTargetExecutionSettings executionSettings, + int shardNum, + int runNumber, + BuildConfiguration configuration) { + super(owner, inputs, list(testLog, cacheStatus, coverageArtifact, microCoverageArtifact)); + this.configuration = Preconditions.checkNotNull(configuration); + this.testLog = testLog; + this.cacheStatus = cacheStatus; + this.coverageData = coverageArtifact; + this.microCoverageData = microCoverageArtifact; + this.shardNum = shardNum; + this.runNumber = runNumber; + this.testProperties = Preconditions.checkNotNull(testProperties); + this.executionSettings = Preconditions.checkNotNull(executionSettings); + + this.baseDir = cacheStatus.getExecPath().getParentDirectory(); + this.namePrefix = FileSystemUtils.removeExtension(cacheStatus.getExecPath().getBaseName()); + + int totalShards = executionSettings.getTotalShards(); + Preconditions.checkState((totalShards == 0 && shardNum == 0) || + (totalShards > 0 && 0 <= shardNum && shardNum < totalShards)); + this.testExitSafe = baseDir.getChild(namePrefix + ".exited_prematurely"); + // testShard Path should be set only if sharding is enabled. + this.testShard = totalShards > 1 + ? baseDir.getChild(namePrefix + ".shard") + : null; + this.xmlOutputPath = baseDir.getChild(namePrefix + ".xml"); + this.testWarningsPath = baseDir.getChild(namePrefix + ".warnings"); + this.testStderr = baseDir.getChild(namePrefix + ".err"); + this.splitLogsDir = baseDir.getChild(namePrefix + ".raw_splitlogs"); + // See note in {@link #getSplitLogsPath} on the choice of file name. + this.splitLogsPath = splitLogsDir.getChild("test.splitlogs"); + this.undeclaredOutputsDir = baseDir.getChild(namePrefix + ".outputs"); + this.undeclaredOutputsZipPath = undeclaredOutputsDir.getChild("outputs.zip"); + this.undeclaredOutputsAnnotationsDir = baseDir.getChild(namePrefix + ".outputs_manifest"); + this.undeclaredOutputsManifestPath = undeclaredOutputsAnnotationsDir.getChild("MANIFEST"); + this.undeclaredOutputsAnnotationsPath = undeclaredOutputsAnnotationsDir.getChild("ANNOTATIONS"); + this.testInfrastructureFailure = baseDir.getChild(namePrefix + ".infrastructure_failure"); + } + + public BuildConfiguration getConfiguration() { + return configuration; + } + + public final PathFragment getBaseDir() { + return baseDir; + } + + public final String getNamePrefix() { + return namePrefix; + } + + @Override + public boolean showsOutputUnconditionally() { + return true; + } + + @Override + public int getInputCount() { + return Iterables.size(getInputs()); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addStrings(executionSettings.getArgs()); + f.addString(executionSettings.getTestFilter() == null ? "" : executionSettings.getTestFilter()); + RunUnder runUnder = executionSettings.getRunUnder(); + f.addString(runUnder == null ? "" : runUnder.getValue()); + f.addStringMap(configuration.getTestEnv()); + f.addString(testProperties.getSize().toString()); + f.addString(testProperties.getTimeout().toString()); + f.addStrings(testProperties.getTags()); + f.addInt(testProperties.isLocal() ? 1 : 0); + f.addInt(shardNum); + f.addInt(executionSettings.getTotalShards()); + f.addInt(runNumber); + f.addInt(configuration.getRunsPerTestForLabel(getOwner().getLabel())); + f.addInt(configuration.isCodeCoverageEnabled() ? 1 : 0); + return f.hexDigestAndReset(); + } + + @Override + public boolean executeUnconditionally() { + // Note: isVolatile must return true if executeUnconditionally can ever return true + // for this instance. + unconditionalExecution = updateExecuteUnconditionallyFromTestStatus(); + checkedCaching = true; + return unconditionalExecution; + } + + @Override + public boolean isVolatile() { + return true; + } + + /** + * Saves cache status to disk. + */ + public void saveCacheStatus(TestResultData data) throws IOException { + try (OutputStream out = cacheStatus.getPath().getOutputStream()) { + data.writeTo(out); + } + } + + /** + * Returns the cache from disk, or null if there is an error. + */ + @Nullable + private TestResultData readCacheStatus() { + try (InputStream in = cacheStatus.getPath().getInputStream()) { + return TestResultData.parseFrom(in); + } catch (IOException expected) { + + } + return null; + } + + private boolean updateExecuteUnconditionallyFromTestStatus() { + if (configuration.cacheTestResults() == TriState.NO || testProperties.isExternal() + || (configuration.cacheTestResults() == TriState.AUTO + && configuration.getRunsPerTestForLabel(getOwner().getLabel()) > 1)) { + return true; + } + + // Test will not be executed unconditionally - check whether test result exists and is + // valid. If it is, method will return false and we will rely on the dependency checker + // to make a decision about test execution. + TestResultData status = readCacheStatus(); + if (status != null) { + if (!status.getCachable()) { + return true; + } + + return (configuration.cacheTestResults() == TriState.AUTO + && !status.getTestPassed()); + } + + // CacheStatus is an artifact, so if it does not exist, the dependency checker will rebuild + // it. We can't return "true" here, as it also signals to not accept cached remote results. + return false; + } + + /** + * May only be called after the dependency checked called executeUnconditionally(). + * Returns whether caching has been deemed safe by looking at the previous test run + * (for local caching). If the previous run is not present, return "true" here, as + * remote execution caching should be safe. + */ + public boolean shouldCacheResult() { + Preconditions.checkState(checkedCaching); + return !unconditionalExecution; + } + + @Override + public void actionCacheHit(Executor executor) { + checkedCaching = false; + try { + executor.getEventBus().post( + executor.getContext(TestActionContext.class).newCachedTestResult( + executor.getExecRoot(), this, readCacheStatus())); + } catch (IOException e) { + LoggingUtil.logToRemote(Level.WARNING, "Failed creating cached protocol buffer", e); + } + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + // return null here to indicate that resources would be managed manually + // during action execution. + return null; + } + + @Override + protected String getRawProgressMessage() { + return "Testing " + getTestName(); + } + + @Override + public String describeStrategy(Executor executor) { + return executor.getContext(TestActionContext.class).strategyLocality(this); + } + + /** + * Deletes <b>all</b> possible test outputs. + * + * TestRunnerAction potentially can create many more non-declared outputs - xml output, + * coverage data file and logs for failed attempts. All those outputs are uniquely + * identified by the test log base name with arbitrary prefix and extension. + */ + @Override + protected void deleteOutputs(Path execRoot) throws IOException { + super.deleteOutputs(execRoot); + + // We do not rely on globs, as it causes quadratic behavior in --runs_per_test and test + // shard count. + + // We also need to remove *.(xml|data|shard|warnings|zip) files if they are present. + execRoot.getRelative(xmlOutputPath).delete(); + execRoot.getRelative(testWarningsPath).delete(); + // Note that splitLogsPath points to a file inside the splitLogsDir so + // it's not necessary to delete it explicitly. + FileSystemUtils.deleteTree(execRoot.getRelative(splitLogsDir)); + FileSystemUtils.deleteTree(execRoot.getRelative(undeclaredOutputsDir)); + FileSystemUtils.deleteTree(execRoot.getRelative(undeclaredOutputsAnnotationsDir)); + execRoot.getRelative(testStderr).delete(); + execRoot.getRelative(testExitSafe).delete(); + if (testShard != null) { + execRoot.getRelative(testShard).delete(); + } + execRoot.getRelative(testInfrastructureFailure).delete(); + + // Coverage files use "coverage" instead of "test". + String coveragePrefix = "coverage" + namePrefix.substring(4); + + // We cannot use coverageData artifact since it may be null. Generate coverage name instead. + execRoot.getRelative(baseDir.getChild(coveragePrefix + ".dat")).delete(); + // We cannot use microcoverageData artifact since it may be null. Generate filename instead. + execRoot.getRelative(baseDir.getChild(coveragePrefix + ".micro.dat")).delete(); + + // Delete files fetched from remote execution. + execRoot.getRelative(baseDir.getChild(namePrefix + ".zip")).delete(); + deleteTestAttemptsDirMaybe(execRoot.getRelative(baseDir), namePrefix); + } + + private void deleteTestAttemptsDirMaybe(Path outputDir, String namePrefix) throws IOException { + Path testAttemptsDir = outputDir.getChild(namePrefix + "_attempts"); + if (testAttemptsDir.exists()) { + // Normally we should have used deleteTree(testAttemptsDir). However, if test output is + // in a FUSE filesystem implemented with the high-level API, there may be .fuse??????? + // entries, which prevent removing the directory. As a workaround, code below will throw + // IOException if it will fail to remove something inside testAttemptsDir, but will + // silently suppress any exceptions when deleting testAttemptsDir itself. + FileSystemUtils.deleteTreesBelow(testAttemptsDir); + try { + testAttemptsDir.delete(); + } catch (IOException e) { + // Do nothing. + } + } + } + + /** + * Gets the test name in a user-friendly format. + * Will generally include the target name and run/shard numbers, if applicable. + */ + public String getTestName() { + String suffix = getTestSuffix(); + String label = Label.print(getOwner().getLabel()); + return suffix.isEmpty() ? label : label + " " + suffix; + } + + /** + * Gets the test suffix in a user-friendly format, eg "(shard 1 of 7)". + * Will include the target name and run/shard numbers, if applicable. + */ + public String getTestSuffix() { + int totalShards = executionSettings.getTotalShards(); + // Use a 1-based index for user friendliness. + int runsPerTest = configuration.getRunsPerTestForLabel(getOwner().getLabel()); + if (totalShards > 1 && runsPerTest > 1) { + return String.format("(shard %d of %d, run %d of %d)", shardNum + 1, totalShards, + runNumber + 1, runsPerTest); + } else if (totalShards > 1) { + return String.format("(shard %d of %d)", shardNum + 1, totalShards); + } else if (runsPerTest > 1) { + return String.format("(run %d of %d)", runNumber + 1, runsPerTest); + } else { + return ""; + } + } + + public Artifact getTestLog() { + return testLog; + } + + public ResolvedPaths resolve(Path execRoot) { + return new ResolvedPaths(execRoot); + } + + public Artifact getCacheStatusArtifact() { + return cacheStatus; + } + + public PathFragment getTestWarningsPath() { + return testWarningsPath; + } + + public PathFragment getSplitLogsPath() { + return splitLogsPath; + } + + /** + * @return path to the optional zip file of undeclared test outputs. + */ + public PathFragment getUndeclaredOutputsZipPath() { + return undeclaredOutputsZipPath; + } + + /** + * @return path to the undeclared output manifest file. + */ + public PathFragment getUndeclaredOutputsManifestPath() { + return undeclaredOutputsManifestPath; + } + + /** + * @return path to the undeclared output annotations file. + */ + public PathFragment getUndeclaredOutputsAnnotationsPath() { + return undeclaredOutputsAnnotationsPath; + } + + public PathFragment getTestShard() { + return testShard; + } + + public PathFragment getExitSafeFile() { + return testExitSafe; + } + + public PathFragment getInfrastructureFailureFile() { + return testInfrastructureFailure; + } + + /** + * @return path to the optionally created XML output file created by the test. + */ + public PathFragment getXmlOutputPath() { + return xmlOutputPath; + } + + /** + * @return coverage data artifact or null if code coverage was not requested. + */ + @Nullable public Artifact getCoverageData() { + return coverageData; + } + + /** + * @return microcoverage data artifact or null if code coverage was not requested. + */ + @Nullable public Artifact getMicroCoverageData() { + return microCoverageData; + } + + public TestTargetProperties getTestProperties() { + return testProperties; + } + + public TestTargetExecutionSettings getExecutionSettings() { + return executionSettings; + } + + public boolean isSharded() { + return testShard != null; + } + + /** + * @return the shard number for this action. + * If getTotalShards() > 0, must be >= 0 and < getTotalShards(). + * Otherwise, must be 0. + */ + public int getShardNum() { + return shardNum; + } + + /** + * @return run number. + */ + public int getRunNumber() { + return runNumber; + } + + @Override + public Artifact getPrimaryOutput() { + return testLog; + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + TestActionContext context = + actionExecutionContext.getExecutor().getContext(TestActionContext.class); + try { + context.exec(this, actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException(this); + } finally { + checkedCaching = false; + } + } + + @Override + public String getMnemonic() { + return "TestRunner"; + } + + /** + * The same set of paths as the parent test action, resolved against a given exec root. + */ + public final class ResolvedPaths { + private final Path execRoot; + + ResolvedPaths(Path execRoot) { + this.execRoot = Preconditions.checkNotNull(execRoot); + } + + private Path getPath(PathFragment relativePath) { + return execRoot.getRelative(relativePath); + } + + public final Path getBaseDir() { + return getPath(baseDir); + } + + /** + * In rare cases, error messages will be printed to stderr instead of stdout. The test action is + * responsible for appending anything in the stderr file to the real test.log. + */ + public Path getTestStderr() { + return getPath(testStderr); + } + + public Path getTestWarningsPath() { + return getPath(testWarningsPath); + } + + public Path getSplitLogsPath() { + return getPath(splitLogsPath); + } + + /** + * @return path to the directory containing the split logs (raw and proto file). + */ + public Path getSplitLogsDir() { + return getPath(splitLogsDir); + } + + /** + * @return path to the optional zip file of undeclared test outputs. + */ + public Path getUndeclaredOutputsZipPath() { + return getPath(undeclaredOutputsZipPath); + } + + /** + * @return path to the directory to hold undeclared test outputs. + */ + public Path getUndeclaredOutputsDir() { + return getPath(undeclaredOutputsDir); + } + + /** + * @return path to the directory to hold undeclared output annotations parts. + */ + public Path getUndeclaredOutputsAnnotationsDir() { + return getPath(undeclaredOutputsAnnotationsDir); + } + + /** + * @return path to the undeclared output manifest file. + */ + public Path getUndeclaredOutputsManifestPath() { + return getPath(undeclaredOutputsManifestPath); + } + + /** + * @return path to the undeclared output annotations file. + */ + public Path getUndeclaredOutputsAnnotationsPath() { + return getPath(undeclaredOutputsAnnotationsPath); + } + + @Nullable + public Path getTestShard() { + return testShard == null ? null : getPath(testShard); + } + + public Path getExitSafeFile() { + return getPath(testExitSafe); + } + + public Path getInfrastructureFailureFile() { + return getPath(testInfrastructureFailure); + } + + /** + * @return path to the optionally created XML output file created by the test. + */ + public Path getXmlOutputPath() { + return getPath(xmlOutputPath); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestStrategy.java new file mode 100644 index 0000000..4905e15 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestStrategy.java
@@ -0,0 +1,388 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.io.ByteStreams; +import com.google.common.io.Closeables; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.exec.SymlinkTreeHelper; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.runtime.BlazeServerStartupOptions; +import com.google.devtools.build.lib.util.io.FileWatcher; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.FileStatus; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.test.TestStatus.TestCase; +import com.google.devtools.common.options.Converters.RangeConverter; +import com.google.devtools.common.options.EnumConverter; +import com.google.devtools.common.options.OptionsClassProvider; +import com.google.devtools.common.options.OptionsParsingException; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * A strategy for executing a {@link TestRunnerAction}. + */ +public abstract class TestStrategy implements TestActionContext { + /** + * Converter for the --flaky_test_attempts option. + */ + public static class TestAttemptsConverter extends RangeConverter { + public TestAttemptsConverter() { + super(1, 10); + } + + @Override + public Integer convert(String input) throws OptionsParsingException { + if ("default".equals(input)) { + return -1; + } else { + return super.convert(input); + } + } + + @Override + public String getTypeDescription() { + return super.getTypeDescription() + " or the string \"default\""; + } + } + + public enum TestOutputFormat { + SUMMARY, // Provide summary output only. + ERRORS, // Print output from failed tests to the stderr after the test failure. + ALL, // Print output from all tests to the stderr after the test completion. + STREAMED; // Stream output for each test. + + /** + * Converts to {@link TestOutputFormat}. + */ + public static class Converter extends EnumConverter<TestOutputFormat> { + public Converter() { + super(TestOutputFormat.class, "test output"); + } + } + } + + public enum TestSummaryFormat { + SHORT, // Print information only about tests. + TERSE, // Like "SHORT", but even shorter: Do not print PASSED tests. + DETAILED, // Print information only about failed test cases. + NONE; // Do not print summary. + + /** + * Converts to {@link TestSummaryFormat}. + */ + public static class Converter extends EnumConverter<TestSummaryFormat> { + public Converter() { + super(TestSummaryFormat.class, "test summary"); + } + } + } + + public static final PathFragment TEST_TMP_ROOT = new PathFragment("_tmp"); + + // Used for selecting subset of testcase / testmethods. + private static final String TEST_BRIDGE_TEST_FILTER_ENV = "TESTBRIDGE_TEST_ONLY"; + + private final boolean statusServerRunning; + protected final ExecutionOptions executionOptions; + protected final BinTools binTools; + + public TestStrategy(OptionsClassProvider requestOptionsProvider, + OptionsClassProvider startupOptionsProvider, BinTools binTools) { + this.executionOptions = requestOptionsProvider.getOptions(ExecutionOptions.class); + this.binTools = binTools; + BlazeServerStartupOptions startupOptions = + startupOptionsProvider.getOptions(BlazeServerStartupOptions.class); + statusServerRunning = startupOptions != null && startupOptions.useWebStatusServer > 0; + } + + @Override + public abstract void exec(TestRunnerAction action, ActionExecutionContext actionExecutionContext) + throws ExecException, InterruptedException; + + @Override + public abstract String strategyLocality(TestRunnerAction action); + + /** + * Callback for determining the strategy locality. + * + * @param action the test action + * @param localRun whether to run it locally + */ + protected String strategyLocality(TestRunnerAction action, boolean localRun) { + return strategyLocality(action); + } + + /** + * Returns mutable map of default testing shell environment. By itself it is incomplete and is + * modified further by the specific test strategy implementations (mostly due to the fact that + * environments used locally and remotely are different). + */ + protected Map<String, String> getDefaultTestEnvironment(TestRunnerAction action) { + Map<String, String> env = new HashMap<>(); + + env.putAll(action.getConfiguration().getDefaultShellEnvironment()); + env.remove("LANG"); + env.put("TZ", "UTC"); + env.put("TEST_SIZE", action.getTestProperties().getSize().toString()); + env.put("TEST_TIMEOUT", Integer.toString(getTimeout(action))); + + if (action.isSharded()) { + env.put("TEST_SHARD_INDEX", Integer.toString(action.getShardNum())); + env.put("TEST_TOTAL_SHARDS", + Integer.toString(action.getExecutionSettings().getTotalShards())); + } + + // When we run test multiple times, set different TEST_RANDOM_SEED values for each run. + if (action.getConfiguration().getRunsPerTestForLabel(action.getOwner().getLabel()) > 1) { + env.put("TEST_RANDOM_SEED", Integer.toString(action.getRunNumber() + 1)); + } + + String testFilter = action.getExecutionSettings().getTestFilter(); + if (testFilter != null) { + env.put(TEST_BRIDGE_TEST_FILTER_ENV, testFilter); + } + + return env; + } + + /** + * Returns the number of attempts specific test action can be retried. + * + * <p>For rules with "flaky = 1" attribute, this method will return 3 unless --flaky_test_attempts + * option is given and specifies another value. + */ + @VisibleForTesting /* protected */ + public int getTestAttempts(TestRunnerAction action) { + if (executionOptions.testAttempts == -1) { + return action.getTestProperties().isFlaky() ? 3 : 1; + } else { + return executionOptions.testAttempts; + } + } + + /** + * Returns timeout value in seconds that should be used for the given test action. We always use + * the "categorical timeouts" which are based on the --test_timeout flag. A rule picks its timeout + * but ends up with the same effective value as all other rules in that bucket. + */ + protected final int getTimeout(TestRunnerAction testAction) { + return executionOptions.testTimeout.get(testAction.getTestProperties().getTimeout()); + } + + /** + * Returns a subset of the environment from the current shell. + * + * <p>Warning: Since these variables are not part of the configuration's fingerprint, they + * MUST NOT be used by any rule or action in such a way as to affect the semantics of that + * build step. + */ + public Map<String, String> getAdmissibleShellEnvironment(BuildConfiguration config, + Iterable<String> variables) { + return getMapping(variables, config.getClientEnv()); + } + + /* + * Finalize test run: persist the result, and post on the event bus. + */ + protected void postTestResult(Executor executor, TestResult result) throws IOException { + result.getTestAction().saveCacheStatus(result.getData()); + executor.getEventBus().post(result); + } + + /** + * Parse a test result XML file into a {@link TestCase}. + */ + @Nullable + protected TestCase parseTestResult(Path resultFile) { + /* xml files. We avoid parsing it unnecessarily, since test results can potentially consume + a large amount of memory. */ + if (executionOptions.testSummary != TestSummaryFormat.DETAILED && !statusServerRunning) { + return null; + } + + try (InputStream fileStream = resultFile.getInputStream()) { + return new TestXmlOutputParser().parseXmlIntoTestResult(fileStream); + } catch (IOException | TestXmlOutputParserException e) { + return null; + } + } + + /** + * For an given environment, returns a subset containing all variables in the given list if they + * are defined in the given environment. + */ + @VisibleForTesting + public static Map<String, String> getMapping(Iterable<String> variables, + Map<String, String> environment) { + Map<String, String> result = new HashMap<>(); + for (String var : variables) { + if (environment.containsKey(var)) { + result.put(var, environment.get(var)); + } + } + return result; + } + + /** + * Returns the runfiles directory associated with the test executable, + * creating/updating it if necessary and --build_runfile_links is specified. + */ + protected static Path getLocalRunfilesDirectory(TestRunnerAction testAction, + ActionExecutionContext actionExecutionContext, BinTools binTools) throws ExecException, + InterruptedException { + TestTargetExecutionSettings execSettings = testAction.getExecutionSettings(); + + // --nobuild_runfile_links disables runfiles generation only for C++ rules. + // In that case, getManifest returns the .runfiles_manifest (input) file, + // not the MANIFEST output file of the build-runfiles action. So the + // extension ".runfiles_manifest" indicates no runfiles tree. + if (!execSettings.getManifest().equals(execSettings.getInputManifest())) { + return execSettings.getManifest().getPath().getParentDirectory(); + } + + // We might need to build runfiles tree now, since it was not created yet + // local testing is needed. + Path program = execSettings.getExecutable().getPath(); + Path runfilesDir = program.getParentDirectory().getChild(program.getBaseName() + ".runfiles"); + + // Synchronize runfiles tree generation on the runfiles manifest artifact. + // This is necessary, because we might end up with multiple test runner actions + // trying to generate same runfiles tree in case of --runs_per_test > 1 or + // local test sharding. + long startTime = Profiler.nanoTimeMaybe(); + synchronized (execSettings.getManifest()) { + Profiler.instance().logSimpleTask(startTime, ProfilerTask.WAIT, testAction); + updateLocalRunfilesDirectory(testAction, runfilesDir, actionExecutionContext, binTools); + } + + return runfilesDir; + } + + /** + * Ensure the runfiles tree exists and is consistent with the TestAction's manifest + * ($0.runfiles_manifest), bringing it into consistency if not. The contents of the output file + * $0.runfiles/MANIFEST, if it exists, are used a proxy for the set of existing symlinks, to avoid + * the need for recursion. + */ + private static void updateLocalRunfilesDirectory(TestRunnerAction testAction, Path runfilesDir, + ActionExecutionContext actionExecutionContext, BinTools binTools) throws ExecException, + InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + + TestTargetExecutionSettings execSettings = testAction.getExecutionSettings(); + try { + if (Arrays.equals(runfilesDir.getRelative("MANIFEST").getMD5Digest(), + execSettings.getManifest().getPath().getMD5Digest())) { + return; + } + } catch (IOException e1) { + // Ignore it - we will just try to create runfiles directory. + } + + executor.getEventHandler().handle(Event.progress( + "Building runfiles directory for '" + execSettings.getExecutable().prettyPrint() + "'.")); + + new SymlinkTreeHelper(execSettings.getManifest().getExecPath(), + runfilesDir.relativeTo(executor.getExecRoot()), /* filesetTree= */ false) + .createSymlinks(testAction, actionExecutionContext, binTools); + + executor.getEventHandler().handle(Event.progress(testAction.getProgressMessage())); + } + + /** + * In rare cases, we might write something to stderr. Append it to the real test.log. + */ + protected static void appendStderr(Path stdOut, Path stdErr) throws IOException { + FileStatus stat = stdErr.statNullable(); + OutputStream out = null; + InputStream in = null; + if (stat != null) { + try { + if (stat.getSize() > 0) { + if (stdOut.exists()) { + stdOut.setWritable(true); + } + out = stdOut.getOutputStream(true); + in = stdErr.getInputStream(); + ByteStreams.copy(in, out); + } + } finally { + Closeables.close(out, true); + Closeables.close(in, true); + stdErr.delete(); + } + } + } + + /** + * Implements the --test_output=streamed option. + */ + protected static class StreamedTestOutput implements Closeable { + private final TestLogHelper.FilterTestHeaderOutputStream headerFilter; + private final FileWatcher watcher; + private final Path testLogPath; + private final OutErr outErr; + + public StreamedTestOutput(OutErr outErr, Path testLogPath) throws IOException { + this.testLogPath = testLogPath; + this.outErr = outErr; + this.headerFilter = TestLogHelper.getHeaderFilteringOutputStream(outErr.getOutputStream()); + this.watcher = new FileWatcher(testLogPath, OutErr.create(headerFilter, headerFilter), false); + watcher.start(); + } + + @Override + public void close() throws IOException { + watcher.stopPumping(); + try { + // The watcher thread might leak if the following call is interrupted. + // This is a relatively minor issue since the worst it could do is + // write one additional line from the test.log to the console later on + // in the build. + watcher.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (!headerFilter.foundHeader()) { + InputStream input = testLogPath.getInputStream(); + try { + ByteStreams.copy(input, outErr.getOutputStream()); + } finally { + input.close(); + } + } + } + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestSuite.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestSuite.java new file mode 100644 index 0000000..ef795aa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestSuite.java
@@ -0,0 +1,99 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.test; + +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.packages.TestTargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.util.Pair; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Implementation for the "test_suite" rule. + */ +public class TestSuite implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + checkTestsAndSuites(ruleContext, "tests"); + checkTestsAndSuites(ruleContext, "suites"); + if (ruleContext.hasErrors()) { + return null; + } + + // + // CAUTION! Keep this logic consistent with lib.query2.TestsExpression! + // + + List<String> tagsAttribute = new ArrayList<>( + ruleContext.attributes().get("tags", Type.STRING_LIST)); + tagsAttribute.remove("manual"); + Pair<Collection<String>, Collection<String>> requiredExcluded = + TestTargetUtils.sortTagsBySense(tagsAttribute); + + List<TransitiveInfoCollection> directTestsAndSuitesBuilder = new ArrayList<>(); + + // The set of implicit tests is determined in + // {@link com.google.devtools.build.lib.packages.Package}. + // Manual tests are already filtered out there. That is what $implicit_tests is about. + for (TransitiveInfoCollection dep : + Iterables.concat( + ruleContext.getPrerequisites("tests", Mode.TARGET), + ruleContext.getPrerequisites("suites", Mode.TARGET), + ruleContext.getPrerequisites("$implicit_tests", Mode.TARGET))) { + if (dep.getProvider(TestProvider.class) != null) { + List<String> tags = dep.getProvider(TestProvider.class).getTestTags(); + if (!TestTargetUtils.testMatchesFilters( + tags, requiredExcluded.first, requiredExcluded.second, true)) { + // This test does not match our filter. Ignore it. + continue; + } + } + directTestsAndSuitesBuilder.add(dep); + } + + Runfiles runfiles = new Runfiles.Builder() + .addTargets(directTestsAndSuitesBuilder, RunfilesProvider.DATA_RUNFILES) + .build(); + + return new RuleConfiguredTargetBuilder(ruleContext) + .add(RunfilesProvider.class, + RunfilesProvider.withData(Runfiles.EMPTY, runfiles)) + .add(TransitiveTestsProvider.class, new TransitiveTestsProvider()) + .build(); + } + + private void checkTestsAndSuites(RuleContext ruleContext, String attributeName) { + for (TransitiveInfoCollection dep : ruleContext.getPrerequisites(attributeName, Mode.TARGET)) { + // TODO(bazel-team): Maybe convert the TransitiveTestsProvider into an inner interface. + TransitiveTestsProvider provider = dep.getProvider(TransitiveTestsProvider.class); + TestProvider testProvider = dep.getProvider(TestProvider.class); + if (provider == null && testProvider == null) { + ruleContext.attributeError(attributeName, + "expecting a test or a test_suite rule but '" + dep.getLabel() + "' is not one"); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetExecutionSettings.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetExecutionSettings.java new file mode 100644 index 0000000..20ad8af --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetExecutionSettings.java
@@ -0,0 +1,133 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.RunUnder; +import com.google.devtools.build.lib.packages.TargetUtils; + +import java.util.List; + +/** + * Container for common test execution settings shared by all + * all TestRunnerAction instances for the given test target. + */ +public final class TestTargetExecutionSettings { + + private final List<String> testArguments; + private final String testFilter; + private final int totalShards; + private final RunUnder runUnder; + private final Artifact runUnderExecutable; + private final Artifact executable; + private final Artifact runfilesManifest; + private final Artifact runfilesInputManifest; + private final Artifact instrumentedFileManifest; + + TestTargetExecutionSettings(RuleContext ruleContext, RunfilesSupport runfiles, + Artifact executable, Artifact instrumentedFileManifest, int shards) { + Preconditions.checkArgument(TargetUtils.isTestRule(ruleContext.getRule())); + Preconditions.checkArgument(shards >= 0); + BuildConfiguration config = ruleContext.getConfiguration(); + + List<String> targetArgs = runfiles.getArgs(); + testArguments = targetArgs.isEmpty() + ? config.getTestArguments() + : ImmutableList.copyOf(Iterables.concat(targetArgs, config.getTestArguments())); + + totalShards = shards; + runUnder = config.getRunUnder(); + runUnderExecutable = getRunUnderExecutable(ruleContext); + + this.testFilter = config.getTestFilter(); + this.executable = executable; + this.runfilesManifest = runfiles.getRunfilesManifest(); + this.runfilesInputManifest = runfiles.getRunfilesInputManifest(); + this.instrumentedFileManifest = instrumentedFileManifest; + } + + private static Artifact getRunUnderExecutable(RuleContext ruleContext) { + TransitiveInfoCollection runUnderTarget = ruleContext + .getPrerequisite(":run_under", Mode.DATA); + return runUnderTarget == null + ? null + : runUnderTarget.getProvider(FilesToRunProvider.class).getExecutable(); + } + + public List<String> getArgs() { + return testArguments; + } + + public String getTestFilter() { + return testFilter; + } + + public int getTotalShards() { + return totalShards; + } + + public RunUnder getRunUnder() { + return runUnder; + } + + public Artifact getRunUnderExecutable() { + return runUnderExecutable; + } + + public Artifact getExecutable() { + return executable; + } + + /** + * Returns the runfiles manifest for this test. + * + * <p>This returns either the input manifest outside of the runfiles tree, + * if blaze is run with --nobuild_runfile_links or the manifest inside the + * runfiles tree, if blaze is run with --build_runfile_links. + * + * @see com.google.devtools.build.lib.analysis.RunfilesSupport#getRunfilesManifest() + */ + public Artifact getManifest() { + return runfilesManifest; + } + + /** + * Returns the input runfiles manifest for this test. + * + * <p>This always returns the input manifest outside of the runfiles tree. + * + * @see com.google.devtools.build.lib.analysis.RunfilesSupport#getRunfilesInputManifest() + */ + public Artifact getInputManifest() { + return runfilesInputManifest; + } + + /** + * Returns instrumented file manifest or null if code coverage is not + * collected. + */ + public Artifact getInstrumentedFileManifest() { + return instrumentedFileManifest; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java new file mode 100644 index 0000000..8cf26b8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java
@@ -0,0 +1,131 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.TestSize; +import com.google.devtools.build.lib.packages.TestTimeout; +import com.google.devtools.build.lib.packages.Type; + +import java.util.List; +import java.util.Map; + +/** + * Container for test target properties available to the + * TestRunnerAction instance. + */ +public class TestTargetProperties { + + /** + * Resources used by local tests of various sizes. + */ + private static final ResourceSet SMALL_RESOURCES = new ResourceSet(20, 0.9, 0.00); + private static final ResourceSet MEDIUM_RESOURCES = new ResourceSet(100, 0.9, 0.1); + private static final ResourceSet LARGE_RESOURCES = new ResourceSet(300, 0.8, 0.1); + private static final ResourceSet ENORMOUS_RESOURCES = new ResourceSet(800, 0.7, 0.4); + + private static ResourceSet getResourceSetFromSize(TestSize size) { + switch (size) { + case SMALL: return SMALL_RESOURCES; + case MEDIUM: return MEDIUM_RESOURCES; + case LARGE: return LARGE_RESOURCES; + default: return ENORMOUS_RESOURCES; + } + } + + private final TestSize size; + private final TestTimeout timeout; + private final List<String> tags; + private final boolean isLocal; + private final boolean isFlaky; + private final boolean isExternal; + private final String language; + private final ImmutableMap<String, String> executionInfo; + + /** + * Creates test target properties instance. Constructor expects that it + * will be called only for test configured targets. + */ + TestTargetProperties(RuleContext ruleContext, + ExecutionInfoProvider executionRequirements) { + Rule rule = ruleContext.getRule(); + + Preconditions.checkState(TargetUtils.isTestRule(rule)); + size = TestSize.getTestSize(rule); + timeout = TestTimeout.getTestTimeout(rule); + tags = ruleContext.attributes().get("tags", Type.STRING_LIST); + isLocal = TargetUtils.isLocalTestRule(rule) || TargetUtils.isExclusiveTestRule(rule); + + // We need to use method on ruleConfiguredTarget to perform validation. + isFlaky = ruleContext.attributes().get("flaky", Type.BOOLEAN); + isExternal = TargetUtils.isExternalTestRule(rule); + + Map<String, String> executionInfo = Maps.newLinkedHashMap(); + executionInfo.putAll(TargetUtils.getExecutionInfo(rule)); + if (executionRequirements != null) { + // This will overwrite whatever TargetUtils put there, which might be confusing. + executionInfo.putAll(executionRequirements.getExecutionInfo()); + } + this.executionInfo = ImmutableMap.copyOf(executionInfo); + + language = TargetUtils.getRuleLanguage(rule); + } + + public TestSize getSize() { + return size; + } + + public TestTimeout getTimeout() { + return timeout; + } + + public List<String> getTags() { + return tags; + } + + public boolean isLocal() { + return isLocal; + } + + public boolean isFlaky() { + return isFlaky; + } + + public boolean isExternal() { + return isExternal; + } + + public ResourceSet getLocalResourceUsage() { + return TestTargetProperties.getResourceSetFromSize(size); + } + + /** + * Returns a map of execution info. See {@link Spawn#getExecutionInfo}. + */ + public ImmutableMap<String, String> getExecutionInfo() { + return executionInfo; + } + + public String getLanguage() { + return language; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParser.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParser.java new file mode 100644 index 0000000..8d660ec --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParser.java
@@ -0,0 +1,345 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.view.test.TestStatus.TestCase; +import com.google.devtools.build.lib.view.test.TestStatus.TestCase.Type; +import com.google.protobuf.UninitializedMessageException; + +import java.io.InputStream; +import java.util.Collection; + +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +/** + * Parses a test.xml generated by jUnit or any testing framework + * into a protocol buffer. The schema of the test.xml is a bit hazy, so there is + * some guesswork involved. + */ +class TestXmlOutputParser { + // jUnit can use either "testsuites" or "testsuite". + private static final Collection<String> TOPLEVEL_ELEMENT_NAMES = + ImmutableSet.of("testsuites", "testsuite"); + + public TestCase parseXmlIntoTestResult(InputStream xmlStream) + throws TestXmlOutputParserException { + return parseXmlToTree(xmlStream); + } + + /** + * Parses the a test result XML file into the corresponding protocol buffer. + * @param xmlStream the XML data stream + * @return the protocol buffer with the parsed data, or null if there was + * an error while parsing the file. + * + * @throws TestXmlOutputParserException when the XML file cannot be parsed + */ + private TestCase parseXmlToTree(InputStream xmlStream) + throws TestXmlOutputParserException { + XMLStreamReader parser = null; + + try { + parser = XMLInputFactory.newInstance().createXMLStreamReader(xmlStream); + + while (true) { + int event = parser.next(); + if (event == XMLStreamConstants.END_DOCUMENT) { + return null; + } + + // First find the topmost node. + if (event == XMLStreamConstants.START_ELEMENT) { + String elementName = parser.getLocalName(); + if (TOPLEVEL_ELEMENT_NAMES.contains(elementName)) { + TestCase result = parseTestSuite(parser, elementName); + return result; + } + } + } + } catch (XMLStreamException e) { + throw new TestXmlOutputParserException(e); + } catch (NumberFormatException e) { + // The parser is definitely != null here. + throw new TestXmlOutputParserException( + "Number could not be parsed at " + + parser.getLocation().getLineNumber() + ":" + + parser.getLocation().getColumnNumber(), + e); + } catch (UninitializedMessageException e) { + // This happens when the XML does not contain a field that is required + // in the protocol buffer + throw new TestXmlOutputParserException(e); + } catch (RuntimeException e) { + + // Seems like that an XNIException can leak through, even though it is not + // specified anywhere. + // + // It's a bad idea to refer to XNIException directly because the Xerces + // documentation says that it may not be available here soon (and it + // results in a compile-time warning anyway), so we do it the roundabout + // way: check if the class name has something to do with Xerces, and if + // so, wrap it in our own exception type, otherwise, let the stack + // unwinding continue. + String name = e.getClass().getCanonicalName(); + if (name != null && name.contains("org.apache.xerces")) { + throw new TestXmlOutputParserException(e); + } else { + throw e; + } + } finally { + if (parser != null) { + try { + parser.close(); + } catch (XMLStreamException e) { + + // Ignore errors during closure so that we do not interfere with an + // already propagating exception. + } + } + } + } + + /** + * Creates an exception suitable to be thrown when and a bad end tag appears. + * The exception could also be thrown from here but that would result in an + * extra stack frame, whereas this way, the topmost frame shows the location + * where the error occurred. + */ + private TestXmlOutputParserException createBadElementException( + String expected, XMLStreamReader parser) { + return new TestXmlOutputParserException("Expected end of XML element '" + + expected + "' , but got '" + parser.getLocalName() + "' at " + + parser.getLocation().getLineNumber() + ":" + + parser.getLocation().getColumnNumber()); + } + + /** + * Parses a 'testsuite' element. + * + * @throws TestXmlOutputParserException if the XML document is malformed + * @throws XMLStreamException if there was an error processing the XML + * @throws NumberFormatException if one of the numeric fields does not contain + * a valid number + */ + private TestCase parseTestSuite(XMLStreamReader parser, String elementName) + throws XMLStreamException, TestXmlOutputParserException { + TestCase.Builder builder = TestCase.newBuilder(); + builder.setType(Type.TEST_SUITE); + for (int i = 0; i < parser.getAttributeCount(); i++) { + String name = parser.getAttributeLocalName(i).intern(); + String value = parser.getAttributeValue(i); + + if (name.equals("name")) { + builder.setName(value); + } else if (name.equals("time")) { + builder.setRunDurationMillis(parseTime(value)); + } + } + + parseContainedElements(parser, elementName, builder); + return builder.build(); + } + + /** + * Parses a time in test.xml format. + * + * @throws NumberFormatException if the time is malformed (i.e. is neither an + * integer nor a decimal fraction with '.' as the fraction separator) + */ + private long parseTime(String string) { + + // This is ugly. For Historical Reasons, we have to check whether the number + // contains a decimal point or not. If it does, the number is expressed in + // milliseconds, otherwise, in seconds. + if (string.contains(".")) { + return Math.round(Float.parseFloat(string) * 1000); + } else { + return Long.parseLong(string); + } + } + + /** + * Parses a 'decorator' element. + * + * @throws TestXmlOutputParserException if the XML document is malformed + * @throws XMLStreamException if there was an error processing the XML + * @throws NumberFormatException if one of the numeric fields does not contain + * a valid number + */ + private TestCase parseTestDecorator(XMLStreamReader parser) + throws XMLStreamException, TestXmlOutputParserException { + TestCase.Builder builder = TestCase.newBuilder(); + builder.setType(Type.TEST_DECORATOR); + for (int i = 0; i < parser.getAttributeCount(); i++) { + String name = parser.getAttributeLocalName(i); + String value = parser.getAttributeValue(i); + + builder.setName(name); + if (name.equals("classname")) { + builder.setClassName(value); + } else if (name.equals("time")) { + builder.setRunDurationMillis(parseTime(value)); + } + } + + parseContainedElements(parser, "testdecorator", builder); + return builder.build(); + } + + /** + * Parses child elements of the specified tag. Strictly speaking, not every + * element can be a child of every other, but the HierarchicalTestResult can + * handle that, and (in this case) it does not hurt to be a bit more flexible + * than necessary. + * + * @throws TestXmlOutputParserException if the XML document is malformed + * @throws XMLStreamException if there was an error processing the XML + * @throws NumberFormatException if one of the numeric fields does not contain + * a valid number + */ + private void parseContainedElements( + XMLStreamReader parser, String elementName, TestCase.Builder builder) + throws XMLStreamException, TestXmlOutputParserException { + int failures = 0; + int errors = 0; + + while (true) { + int event = parser.next(); + switch (event) { + case XMLStreamConstants.START_ELEMENT: + String childElementName = parser.getLocalName().intern(); + + // We are not parsing four elements here: system-out, system-err, + // failure and error. They potentially contain useful information, but + // they can be too big to fit in the memory. We add failure and error + // elements to the output without a message, so that there is a + // difference between passed and failed test cases. + if (childElementName.equals("testsuite")) { + builder.addChild(parseTestSuite(parser, childElementName)); + } else if (childElementName.equals("testcase")) { + builder.addChild(parseTestCase(parser)); + } else if (childElementName.equals("failure")) { + failures += 1; + skipCompleteElement(parser); + } else if (childElementName.equals("error")) { + errors += 1; + skipCompleteElement(parser); + } else if (childElementName.equals("testdecorator")) { + builder.addChild(parseTestDecorator(parser)); + } else { + + // Unknown element encountered. Since the schema of the input file + // is a bit hazy, just skip it and go merrily on our way. Ignorance + // is bliss. + skipCompleteElement(parser); + } + break; + + case XMLStreamConstants.END_ELEMENT: + // Propagate errors/failures from children up to the current case + for (int i = 0; i < builder.getChildCount(); i += 1) { + if (builder.getChild(i).getStatus() == TestCase.Status.ERROR) { + errors += 1; + } + if (builder.getChild(i).getStatus() == TestCase.Status.FAILED) { + failures += 1; + } + } + + if (errors > 0) { + builder.setStatus(TestCase.Status.ERROR); + } else if (failures > 0) { + builder.setStatus(TestCase.Status.FAILED); + } else { + builder.setStatus(TestCase.Status.PASSED); + } + // This is the end tag of the element we are supposed to parse. + // Hooray, tell our superiors that our mission is complete. + if (!parser.getLocalName().equals(elementName)) { + throw createBadElementException(elementName, parser); + } + return; + } + } + } + + + /** + * Parses a 'testcase' element. + * + * @throws TestXmlOutputParserException if the XML document is malformed + * @throws XMLStreamException if there was an error processing the XML + * @throws NumberFormatException if the time field does not contain a valid + * number + */ + private TestCase parseTestCase(XMLStreamReader parser) + throws XMLStreamException, TestXmlOutputParserException { + TestCase.Builder builder = TestCase.newBuilder(); + builder.setType(Type.TEST_CASE); + for (int i = 0; i < parser.getAttributeCount(); i++) { + String name = parser.getAttributeLocalName(i).intern(); + String value = parser.getAttributeValue(i); + + if (name.equals("name")) { + builder.setName(value); + } else if (name.equals("classname")) { + builder.setClassName(value); + } else if (name.equals("time")) { + builder.setRunDurationMillis(parseTime(value)); + } else if (name.equals("result")) { + builder.setResult(value); + } else if (name.equals("status")) { + if (value.equals("notrun")) { + builder.setRun(false); + } else if (value.equals("run")) { + builder.setRun(true); + } + } + } + + parseContainedElements(parser, "testcase", builder); + return builder.build(); + } + + /** + * Skips over a complete XML element on the input. + * Precondition: the cursor is at a START_ELEMENT. + * Postcondition: the cursor is at an END_ELEMENT. + * + * @throws XMLStreamException if the XML is malformed + */ + private void skipCompleteElement(XMLStreamReader parser) throws XMLStreamException { + int depth = 1; + while (true) { + int event = parser.next(); + + switch (event) { + case XMLStreamConstants.START_ELEMENT: + depth++; + break; + + case XMLStreamConstants.END_ELEMENT: + if (--depth == 0) { + return; + } + break; + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParserException.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParserException.java new file mode 100644 index 0000000..c27ca9d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestXmlOutputParserException.java
@@ -0,0 +1,33 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +/** + * This exception gets thrown if there was a problem with parsing a test.xml + * file. + */ +class TestXmlOutputParserException extends Exception { + public TestXmlOutputParserException(String message, Throwable cause) { + super(message, cause); + } + + public TestXmlOutputParserException(Throwable cause) { + super(cause); + } + + public TestXmlOutputParserException(String message) { + super(message); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TransitiveTestsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/test/TransitiveTestsProvider.java new file mode 100644 index 0000000..c46b2a7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TransitiveTestsProvider.java
@@ -0,0 +1,25 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.test; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Marker transitive info provider for test_suite rules to recognize one another. + */ +@Immutable +public final class TransitiveTestsProvider implements TransitiveInfoProvider { +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/workspace/Bind.java b/src/main/java/com/google/devtools/build/lib/rules/workspace/Bind.java new file mode 100644 index 0000000..49f829a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/workspace/Bind.java
@@ -0,0 +1,125 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.workspace; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.UnmodifiableIterator; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.syntax.ClassObject; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkNestedSet; + +/** + * Implementation for the bind rule. + */ +public class Bind implements RuleConfiguredTargetFactory { + + /** + * This configured target pretends to be whatever type of target "actual" is, returning its + * transitive info providers and target, but returning the label for the //external target. + */ + private static class BindConfiguredTarget implements ConfiguredTarget, ClassObject { + + private Label label; + private ConfiguredTarget configuredTarget; + private BuildConfiguration config; + + BindConfiguredTarget(RuleContext ruleContext) { + label = ruleContext.getRule().getLabel(); + config = ruleContext.getConfiguration(); + // TODO(bazel-team): we should special case ConfiguredTargetFactory.createConfiguredTarget, + // not cast down here. + configuredTarget = (ConfiguredTarget) ruleContext.getPrerequisite("actual", Mode.TARGET); + } + + @Override + public <P extends TransitiveInfoProvider> P getProvider(Class<P> provider) { + return configuredTarget.getProvider(provider); + } + + @Override + public Label getLabel() { + return label; + } + + @Override + public Object get(String providerKey) { + return configuredTarget.get(providerKey); + } + + @Override + public UnmodifiableIterator<TransitiveInfoProvider> iterator() { + return configuredTarget.iterator(); + } + + @Override + public Target getTarget() { + return configuredTarget.getTarget(); + } + + @Override + public BuildConfiguration getConfiguration() { + return config; + } + + /* ClassObject methods */ + + @Override + public Object getValue(String name) { + if (name.equals("label")) { + return getLabel(); + } else if (name.equals("files")) { + // A shortcut for files to build in Skylark. FileConfiguredTarget and RunleConfiguredTarget + // always has FileProvider and Error- and PackageGroupConfiguredTarget-s shouldn't be + // accessible in Skylark. + return SkylarkNestedSet.of( + Artifact.class, getProvider(FileProvider.class).getFilesToBuild()); + } + return configuredTarget.get(name); + } + + @SuppressWarnings("cast") + @Override + public ImmutableCollection<String> getKeys() { + return new ImmutableList.Builder<String>() + .add("label", "files") + .addAll(configuredTarget.getProvider(RuleConfiguredTarget.SkylarkProviders.class) + .getKeys()) + .build(); + } + + @Override + public String errorMessage(String name) { + // Use the default error message. + return null; + } + } + + @Override + public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException { + return new BindConfiguredTarget(ruleContext); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/workspace/BindRule.java b/src/main/java/com/google/devtools/build/lib/rules/workspace/BindRule.java new file mode 100644 index 0000000..c3f2dd2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/workspace/BindRule.java
@@ -0,0 +1,126 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.workspace; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.LABEL; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses.BaseRule; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType; + +/** + * Binds an existing target to a target in the virtual //external package. + */ +@BlazeRule(name = "bind", + type = RuleClassType.WORKSPACE, + ancestors = {BaseRule.class}, + factoryClass = Bind.class) +public final class BindRule implements RuleDefinition { + + @Override + public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment environment) { + return builder + /* <!-- #BLAZE_RULE(bind).ATTRIBUTE(actual) --> + The target to be aliased. + + <p>This target must exist, but can be any type of rule (including bind).</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("actual", LABEL).allowedFileTypes()) + .setWorkspaceOnly() + .build(); + } +} +/*<!-- #BLAZE_RULE (NAME = bind, TYPE = OTHER, FAMILY = General)[GENERIC_RULE] --> + +${ATTRIBUTE_SIGNATURE} + +<p>Gives a target an alias in the <code>//external</code> package.</p> + +${ATTRIBUTE_DEFINITION} + +<p>The <code>//external</code> package is not a "normal" package: there is no external/ directory, + so it can be thought of as a "virtual package" that contains all bound targets.</p> + +<h4 id="bind_examples">Examples</h4> + +<p>To give a target an alias, bind it in the <i>WORKSPACE</i> file. For example, suppose there is + a <code>java_library</code> target called <code>//third_party/javacc-v2</code>. This could be + aliased by adding the following to the <i>WORKSPACE</i> file:</p> + +<pre class="code"> +bind( + name = "javacc-latest", + actual = "//third_party/javacc-v2", +) +</pre> + +<p>Now targets can depend on <code>//external:javacc-latest</code> instead of + <code>//third_party/javacc-v2</code>. If javacc-v3 is released, the binding can be updated and + all of the BUILD files depending on <code>//external:javacc-latest</code> will now depend on + javacc-v3 without needing to be edited.</p> + +<p>Bind can also be used to refer to external repositories' targets. For example, if there is a + remote repository named <code>@my-ssl</code> imported in the WORKSPACE file. If the + <code>@my-ssl</code> repository has a cc_library target <code>//src:openssl-lib</code>, you + could make this target accessible for your program to depend on by using <code>bind</code>:</p> + +<pre class="code"> +bind( + name = "openssl", + actual = "@my-ssl//src:openssl-lib", +) +</pre> + +<p>BUILD files cannot use labels that include a repository name + ("@repository-name//package-name:target-name"), so the only way to depend on a target from + another repository is to <code>bind</code> it in the WORKSPACE file and then refer to it by its + aliased name in <code>//external</code> from a BUILD file.</p> + +<p>For example, in a BUILD file, the bound target could be used as follows:</p> + +<pre class="code"> +cc_library( + name = "sign-in", + srcs = ["sign_in.cc"], + hdrs = ["sign_in.h"], + deps = ["//external:openssl"], +) +</pre> + +<p>Within <code>sign_in.cc</code> and <code>sign_in.h</code>, the header files exposed by + <code>//external:openssl</code> can be referred to by their path relative to their repository + root. For example, if the rule definition for <code>@my-ssl//src:openssl-lib</code> looks like + this:</p> + +<pre class="code"> +cc_library( + name = "openssl-lib", + srcs = ["openssl.cc"], + hdrs = ["openssl.h"], +) +</pre> + +<p>Then <code>sign_in.cc</code>'s first lines might look like this:</p> + +<pre class="code"> +#include "sign_in.h" +#include "src/openssl.h" +</pre> + +<!-- #END_BLAZE_RULE -->*/
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/AbstractCriticalPathComponent.java b/src/main/java/com/google/devtools/build/lib/runtime/AbstractCriticalPathComponent.java new file mode 100644 index 0000000..9bf7a3f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/AbstractCriticalPathComponent.java
@@ -0,0 +1,120 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; + +import javax.annotation.Nullable; + +/** + * This class records the critical path for the graph of actions executed. + */ +@ThreadCompatible +public class AbstractCriticalPathComponent<C extends AbstractCriticalPathComponent<C>> { + + /** Wall time start time for the action. In milliseconds. */ + private final long startTime; + /** Wall time finish time for the action. In milliseconds. */ + private long finishTime = 0; + protected volatile boolean isRunning = true; + + /** We keep here the critical path time for the most expensive child. */ + private long childAggregatedWallTime = 0; + + /** The action for which we are storing the stat. */ + private final Action action; + + /** + * Child with the maximum critical path. + */ + @Nullable + private C child; + + public AbstractCriticalPathComponent(Action action, long startTime) { + this.action = action; + this.startTime = startTime; + } + + /** Sets the finish time for the action in milliseconds. */ + public void setFinishTimeMillis(long finishTime) { + Preconditions.checkState(isRunning, "Already stopped! %s.", action); + this.finishTime = finishTime; + isRunning = false; + } + + /** The action for which we are storing the stat. */ + public Action getAction() { + return action; + } + + /** + * Add statistics for one dependency of this action. + */ + public void addDepInfo(C dep) { + Preconditions.checkState(!dep.isRunning, + "Cannot add critical path stats when the action is not finished. %s. %s", action, + dep.getAction()); + long childAggregatedWallTime = dep.getAggregatedWallTime(); + // Replace the child if its critical path had the maximum wall time. + if (child == null || childAggregatedWallTime > this.childAggregatedWallTime) { + this.childAggregatedWallTime = childAggregatedWallTime; + child = dep; + } + } + + public long getActionWallTime() { + Preconditions.checkState(!isRunning, "Still running %s", action); + return finishTime - startTime; + } + + /** + * Returns the current critical path for the action in milliseconds. + * + * <p>Critical path is defined as : action_execution_time + max(child_critical_path). + */ + public long getAggregatedWallTime() { + Preconditions.checkState(!isRunning, "Still running %s", action); + return getActionWallTime() + childAggregatedWallTime; + } + + /** Time when the action started to execute. Milliseconds since epoch time. */ + public long getStartTime() { + return startTime; + } + + /** + * Get the child critical path component. + * + * <p>The component dependency with the maximum total critical path time. + */ + @Nullable + public C getChild() { + return child; + } + + /** + * Returns a human readable representation of the critical path stats with all the details. + */ + @Override + public String toString() { + String currentTime = "still running "; + if (!isRunning) { + currentTime = String.format("%.2f", getActionWallTime() / 1000.0) + "s "; + } + return currentTime + action.describe(); + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/AggregatedCriticalPath.java b/src/main/java/com/google/devtools/build/lib/runtime/AggregatedCriticalPath.java new file mode 100644 index 0000000..dd70c35 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/AggregatedCriticalPath.java
@@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; + +/** + * Aggregates all the critical path components in one object. This allows us to easily access the + * components data and have a proper toString(). + */ +public class AggregatedCriticalPath<T extends AbstractCriticalPathComponent> { + + private final long totalTime; + private final ImmutableList<T> criticalPathComponents; + + protected AggregatedCriticalPath(long totalTime, ImmutableList<T> criticalPathComponents) { + this.totalTime = totalTime; + this.criticalPathComponents = criticalPathComponents; + } + + /** Total wall time in ms spent running the critical path actions. */ + public long totalTime() { + return totalTime; + } + + /** Returns a list of all the component stats for the critical path. */ + public ImmutableList<T> components() { + return criticalPathComponents; + } + + @Override + public String toString() { + return toString(false); + } + + /** + * Returns a summary version of the critical path stats that omits stats that are not useful + * to the user. + */ + public String toStringSummary() { + return toString(true); + } + + private String toString(boolean summary) { + StringBuilder sb = new StringBuilder("Critical Path: "); + double totalMillis = totalTime; + sb.append(String.format("%.2f", totalMillis / 1000.0)); + sb.append("s"); + if (summary || criticalPathComponents.isEmpty()) { + return sb.toString(); + } + sb.append("\n "); + Joiner.on("\n ").appendTo(sb, criticalPathComponents); + return sb.toString(); + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/AggregatingTestListener.java b/src/main/java/com/google/devtools/build/lib/runtime/AggregatingTestListener.java new file mode 100644 index 0000000..cc240c4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/AggregatingTestListener.java
@@ -0,0 +1,255 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.MapMaker; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import com.google.common.eventbus.AllowConcurrentEvents; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisFailureEvent; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.LabelAndConfiguration; +import com.google.devtools.build.lib.analysis.TargetCompleteEvent; +import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent; +import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent; +import com.google.devtools.build.lib.buildtool.buildevent.TestFilteringCompleteEvent; +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.events.ExceptionListener; +import com.google.devtools.build.lib.rules.test.TestProvider; +import com.google.devtools.build.lib.rules.test.TestResult; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +/** + * This class aggregates and reports target-wide test statuses in real-time. + * It must be public for EventBus invocation. + */ +@ThreadSafety.ThreadSafe +public class AggregatingTestListener { + private final ConcurrentMap<Artifact, TestResult> statusMap = new MapMaker().makeMap(); + + private final TestResultAnalyzer analyzer; + private final EventBus eventBus; + private final EventHandlerPreconditions preconditionHelper; + private volatile boolean blazeHalted = false; + + + // summaryLock guards concurrent access to these two collections, which should be kept + // synchronized with each other. + private final Map<LabelAndConfiguration, TestSummary.Builder> summaries; + private final Multimap<LabelAndConfiguration, Artifact> remainingRuns; + private final Object summaryLock = new Object(); + + public AggregatingTestListener(TestResultAnalyzer analyzer, + EventBus eventBus, + ExceptionListener listener) { + this.analyzer = analyzer; + this.eventBus = eventBus; + this.preconditionHelper = new EventHandlerPreconditions(listener); + + this.summaries = Maps.newHashMap(); + this.remainingRuns = HashMultimap.create(); + } + + /** + * @return An unmodifiable copy of the map of test results. + */ + public Map<Artifact, TestResult> getStatusMap() { + return ImmutableMap.copyOf(statusMap); + } + + /** + * Populates the test summary map as soon as test filtering is complete. + * This is the earliest at which the final set of targets to test is known. + */ + @Subscribe + @AllowConcurrentEvents + public void populateTests(TestFilteringCompleteEvent event) { + // Add all target runs to the map, assuming 1:1 status artifact <-> result. + synchronized (summaryLock) { + for (ConfiguredTarget target : event.getTestTargets()) { + Iterable<Artifact> statusArtifacts = + target.getProvider(TestProvider.class).getTestParams().getTestStatusArtifacts(); + preconditionHelper.checkState(remainingRuns.putAll(asKey(target), statusArtifacts)); + + // And create an empty summary suitable for incremental analysis. + // Also has the nice side effect of mapping labels to RuleConfiguredTargets. + TestSummary.Builder summary = TestSummary.newBuilder() + .setTarget(target) + .setStatus(BlazeTestStatus.NO_STATUS); + preconditionHelper.checkState(summaries.put(asKey(target), summary) == null); + } + } + } + + /** + * Records a new test run result and incrementally updates the target status. + * This event is sent upon completion of executed test runs. + */ + @Subscribe + @AllowConcurrentEvents + public void testEvent(TestResult result) { + Preconditions.checkState( + statusMap.put(result.getTestStatusArtifact(), result) == null, + "Duplicate result reported for an individual test shard"); + + ActionOwner testOwner = result.getTestAction().getOwner(); + LabelAndConfiguration targetLabel = LabelAndConfiguration.of( + testOwner.getLabel(), result.getTestAction().getConfiguration()); + + TestSummary finalTestSummary = null; + synchronized (summaryLock) { + TestSummary.Builder summary = summaries.get(targetLabel); + preconditionHelper.checkNotNull(summary); + if (!remainingRuns.remove(targetLabel, result.getTestStatusArtifact())) { + // This can happen if a buildCompleteEvent() was processed before this event reached us. + // This situation is likely to happen if --notest_keep_going is set with multiple targets. + return; + } + + summary = analyzer.incrementalAnalyze(summary, result); + + // If all runs are processed, the target is finished and ready to report. + if (!remainingRuns.containsKey(targetLabel)) { + finalTestSummary = summary.build(); + } + } + + // Report finished targets. + if (finalTestSummary != null) { + eventBus.post(finalTestSummary); + } + } + + private void targetFailure(LabelAndConfiguration label) { + TestSummary finalSummary; + synchronized (summaryLock) { + if (!remainingRuns.containsKey(label)) { + // Blaze does not guarantee that BuildResult.getSuccessfulTargets() and posted TestResult + // events are in sync. Thus, it is possible that a test event was posted, but the target is + // not present in the set of successful targets. + return; + } + + TestSummary.Builder summary = summaries.get(label); + if (summary == null) { + // Not a test target; nothing to do. + return; + } + finalSummary = analyzer.markUnbuilt(summary, blazeHalted).build(); + + // These are never going to run; removing them marks the target complete. + remainingRuns.removeAll(label); + } + eventBus.post(finalSummary); + } + + @VisibleForTesting + void buildComplete( + Collection<ConfiguredTarget> actualTargets, Collection<ConfiguredTarget> successfulTargets) { + if (actualTargets == null || successfulTargets == null) { + return; + } + + for (ConfiguredTarget target: Sets.difference( + ImmutableSet.copyOf(actualTargets), ImmutableSet.copyOf(successfulTargets))) { + targetFailure(asKey(target)); + } + } + + @Subscribe + public void buildCompleteEvent(BuildCompleteEvent event) { + if (event.getResult().wasCatastrophe()) { + blazeHalted = true; + } + buildComplete(event.getResult().getActualTargets(), event.getResult().getSuccessfulTargets()); + } + + @Subscribe + public void analysisFailure(AnalysisFailureEvent event) { + targetFailure(event.getFailedTarget()); + } + + @Subscribe + @AllowConcurrentEvents + public void buildInterrupted(BuildInterruptedEvent event) { + blazeHalted = true; + } + + /** + * Called when a build action is not executed (e.g. because a dependency failed to build). We want + * to catch such events in order to determine when a test target has failed to build. + */ + @Subscribe + @AllowConcurrentEvents + public void targetComplete(TargetCompleteEvent event) { + if (event.failed()) { + targetFailure(new LabelAndConfiguration(event.getTarget())); + } + } + + /** + * Returns the known aggregate results for the given target at the current moment. + */ + public TestSummary.Builder getCurrentSummary(ConfiguredTarget target) { + synchronized (summaryLock) { + return summaries.get(asKey(target)); + } + } + + /** + * Returns all test status artifacts associated with a given target + * whose runs have yet to finish. + */ + public Collection<Artifact> getIncompleteRuns(ConfiguredTarget target) { + synchronized (summaryLock) { + return Collections.unmodifiableCollection(remainingRuns.get(asKey(target))); + } + } + + /** + * Returns true iff all runs of the target are accounted for. + */ + public boolean targetReported(ConfiguredTarget target) { + synchronized (summaryLock) { + return summaries.containsKey(asKey(target)) && !remainingRuns.containsKey(asKey(target)); + } + } + + /** + * Returns the {@link TestResultAnalyzer} associated with this listener. + */ + public TestResultAnalyzer getAnalyzer() { + return analyzer; + } + + private LabelAndConfiguration asKey(ConfiguredTarget target) { + return new LabelAndConfiguration(target); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommand.java new file mode 100644 index 0000000..61f46a8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommand.java
@@ -0,0 +1,63 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +/** + * Interface implemented by Blaze commands. In addition to implementing this interface, each + * command must be annotated with a {@link Command} annotation. + */ +public interface BlazeCommand { + /** + * This method provides the imperative portion of the command. It takes + * a {@link OptionsProvider} instance {@code options}, which provides access + * to the options instances via {@link OptionsProvider#getOptions(Class)}, + * and access to the residue (the remainder of the command line) via + * {@link OptionsProvider#getResidue()}. The framework parses and makes + * available exactly the options that the command class specifies via the + * annotation {@link Command#options()}. The command may write to standard + * out and standard error via {@code outErr}. It indicates success / failure + * via its return value, which becomes the Unix exit status of the Blaze + * client process. It may indicate a shutdown request by throwing + * {@link BlazeCommandDispatcher.ShutdownBlazeServerException}. In that case, + * the Blaze server process (the memory resident portion of Blaze) will + * shut down and the exit status will be 0 (in case the shutdown succeeds + * without error). + * + * @param runtime The Blaze runtime requesting the execution of the command + * @param options A parsed options instance initialized with the values for + * the options specified in {@link Command#options()}. + * + * @return The Unix exit status for the Blaze client. + * @throws BlazeCommandDispatcher.ShutdownBlazeServerException Indicates + * that the command wants to shutdown the Blaze server. + */ + ExitCode exec(BlazeRuntime runtime, OptionsProvider options) + throws BlazeCommandDispatcher.ShutdownBlazeServerException; + + /** + * Allows the command to provide command-specific option defaults and/or + * requirements. This method is called after all command-line and rc file options have been + * parsed. + * + * @param runtime The Blaze runtime requesting the execution of the command + * + * @throws AbruptExitException if something went wrong + */ + void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) throws AbruptExitException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java new file mode 100644 index 0000000..cee47ee --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java
@@ -0,0 +1,692 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.common.io.Flushables; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.AnsiStrippingOutputStream; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.io.DelegatingOutErr; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.OptionPriority; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.logging.Level; + +/** + * Dispatches to the Blaze commands; that is, given a command line, this + * abstraction looks up the appropriate command object, parses the options + * required by the object, and calls its exec method. Also, this object provides + * the runtime state (BlazeRuntime) to the commands. + */ +public class BlazeCommandDispatcher { + + // Keep in sync with options added in OptionProcessor::AddRcfileArgsAndOptions() + private static final Set<String> INTERNAL_COMMAND_OPTIONS = ImmutableSet.of( + "rc_source", "default_override", "isatty", "terminal_columns", "ignore_client_env", + "client_env", "client_cwd"); + + private static final ImmutableList<String> HELP_COMMAND = ImmutableList.of("help"); + + private static final Set<String> ALL_HELP_OPTIONS = ImmutableSet.of("--help", "-help", "-h"); + + /** + * By throwing this exception, a command indicates that it wants to shutdown + * the Blaze server process. + * See {@link BlazeCommandDispatcher#exec(List, OutErr, long)}. + */ + public static class ShutdownBlazeServerException extends Exception { + private final int exitStatus; + + public ShutdownBlazeServerException(int exitStatus, Throwable cause) { + super(cause); + this.exitStatus = exitStatus; + } + + public ShutdownBlazeServerException(int exitStatus) { + this.exitStatus = exitStatus; + } + + public int getExitStatus() { + return exitStatus; + } + } + + private final BlazeRuntime runtime; + private final Map<String, BlazeCommand> commandsByName = new LinkedHashMap<>(); + + private OutputStream logOutputStream = null; + + /** + * Create a Blaze dispatcher that uses the specified {@code BlazeRuntime} + * instance, and no default options, and delegates to {@code commands} as + * appropriate. + */ + @VisibleForTesting + public BlazeCommandDispatcher(BlazeRuntime runtime, BlazeCommand... commands) { + this(runtime, ImmutableList.copyOf(commands)); + } + + /** + * Create a Blaze dispatcher that uses the specified {@code BlazeRuntime} + * instance, and delegates to {@code commands} as appropriate. + */ + public BlazeCommandDispatcher(BlazeRuntime runtime, Iterable<BlazeCommand> commands) { + this.runtime = runtime; + for (BlazeCommand command : commands) { + addCommandByName(command); + } + + for (BlazeModule module : runtime.getBlazeModules()) { + for (BlazeCommand command : module.getCommands()) { + addCommandByName(command); + } + } + + runtime.setCommandMap(commandsByName); + } + + /** + * Adds the given command under the given name to the map of commands. + * + * @throws AssertionError if the name is already used by another command. + */ + private void addCommandByName(BlazeCommand command) { + String name = command.getClass().getAnnotation(Command.class).name(); + if (commandsByName.containsKey(name)) { + throw new IllegalStateException("Command name or alias " + name + " is already used."); + } + commandsByName.put(name, command); + } + + /** + * Only some commands work if cwd != workspaceSuffix in Blaze. In that case, also check if Blaze + * was called from the output directory and fail if it was. + */ + private ExitCode checkCwdInWorkspace(Command commandAnnotation, String commandName, + OutErr outErr) { + if (!commandAnnotation.mustRunInWorkspace()) { + return ExitCode.SUCCESS; + } + + if (!runtime.inWorkspace()) { + outErr.printErrLn("The '" + commandName + "' command is only supported from within a " + + "workspace."); + return ExitCode.COMMAND_LINE_ERROR; + } + + Path workspace = runtime.getWorkspace(); + Path doNotBuild = workspace.getParentDirectory().getRelative( + BlazeRuntime.DO_NOT_BUILD_FILE_NAME); + if (doNotBuild.exists()) { + if (!commandAnnotation.canRunInOutputDirectory()) { + outErr.printErrLn(getNotInRealWorkspaceError(doNotBuild)); + return ExitCode.COMMAND_LINE_ERROR; + } else { + outErr.printErrLn("WARNING: Blaze is run from output directory. This is unsound."); + } + } + return ExitCode.SUCCESS; + } + + private CommonCommandOptions checkOptions(OptionsParser optionsParser, + Command commandAnnotation, List<String> args, List<String> rcfileNotes, OutErr outErr) + throws OptionsParsingException { + Function<String, String> commandOptionSourceFunction = new Function<String, String>() { + @Override + public String apply(String input) { + if (INTERNAL_COMMAND_OPTIONS.contains(input)) { + return "options generated by Blaze launcher"; + } else { + return "command line options"; + } + } + }; + + // Explicit command-line options: + List<String> cmdLineAfterCommand = args.subList(1, args.size()); + optionsParser.parseWithSourceFunction(OptionPriority.COMMAND_LINE, + commandOptionSourceFunction, cmdLineAfterCommand); + + // Command-specific options from .blazerc passed in via --default_override + // and --rc_source. A no-op if none are provided. + CommonCommandOptions rcFileOptions = optionsParser.getOptions(CommonCommandOptions.class); + List<Pair<String, ListMultimap<String, String>>> optionsMap = + getOptionsMap(outErr, rcFileOptions.rcSource, rcFileOptions.optionsOverrides, + commandsByName.keySet()); + + parseOptionsForCommand(rcfileNotes, commandAnnotation, optionsParser, optionsMap, null); + + // Fix-point iteration until all configs are loaded. + List<String> configsLoaded = ImmutableList.of(); + CommonCommandOptions commonOptions = optionsParser.getOptions(CommonCommandOptions.class); + while (!commonOptions.configs.equals(configsLoaded)) { + Set<String> missingConfigs = new LinkedHashSet<>(commonOptions.configs); + missingConfigs.removeAll(configsLoaded); + parseOptionsForCommand(rcfileNotes, commandAnnotation, optionsParser, optionsMap, + missingConfigs); + configsLoaded = commonOptions.configs; + commonOptions = optionsParser.getOptions(CommonCommandOptions.class); + } + + return commonOptions; + } + + /** + * Sends {@code EventKind.{STDOUT|STDERR}} messages to the given {@link OutErr}. + * + * <p>This is necessary because we cannot delete the output files from the previous Blaze run + * because there can be processes spawned by the previous invocation that are still processing + * them, in which case we need to print a warning message about that. + * + * <p>Thus, messages sent to {@link Reporter#getOutErr} get sent to this event handler, then + * to its {@link OutErr}. We need to go deeper! + */ + private static class OutErrEventHandler implements EventHandler { + private final OutErr outErr; + + private OutErrEventHandler(OutErr outErr) { + this.outErr = outErr; + } + + @Override + public void handle(Event event) { + try { + switch (event.getKind()) { + case STDOUT: + outErr.getOutputStream().write(event.getMessageBytes()); + break; + case STDERR: + outErr.getErrorStream().write(event.getMessageBytes()); + break; + } + } catch (IOException e) { + // We cannot do too much here -- ErrorEventListener#handle does not provide us with ways to + // report an error. + } + } + } + + /** + * Executes a single command. Returns the Unix exit status for the Blaze + * client process, or throws {@link ShutdownBlazeServerException} to + * indicate that a command wants to shutdown the Blaze server. + */ + public int exec(List<String> args, OutErr originalOutErr, long firstContactTime) + throws ShutdownBlazeServerException { + // Record the start time for the profiler and the timestamp granularity monitor. Do not put + // anything before this! + long execStartTimeNanos = runtime.getClock().nanoTime(); + + // Record the command's starting time for use by the commands themselves. + runtime.recordCommandStartTime(firstContactTime); + + // Record the command's starting time again, for use by + // TimestampGranularityMonitor.waitForTimestampGranularity(). + // This should be done as close as possible to the start of + // the command's execution - that's why we do this separately, + // rather than in runtime.beforeCommand(). + runtime.getTimestampGranularityMonitor().setCommandStartTime(); + runtime.initEventBus(); + + // Give a chance for module.beforeCommand() to report an errors to stdout and stderr. + // Once we can close the old streams, this event handler is removed. + OutErrEventHandler originalOutErrEventHandler = + new OutErrEventHandler(originalOutErr); + runtime.getReporter().addHandler(originalOutErrEventHandler); + OutErr outErr = originalOutErr; + runtime.getReporter().removeHandler(originalOutErrEventHandler); + + if (args.isEmpty()) { // Default to help command if no arguments specified. + args = HELP_COMMAND; + } + String commandName = args.get(0); + + // Be gentle to users who want to find out about Blaze invocation. + if (ALL_HELP_OPTIONS.contains(commandName)) { + commandName = "help"; + } + + BlazeCommand command = commandsByName.get(commandName); + if (command == null) { + outErr.printErrLn("Command '" + commandName + "' not found. " + "Try 'blaze help'."); + return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode(); + } + Command commandAnnotation = command.getClass().getAnnotation(Command.class); + + AbruptExitException exitCausingException = null; + for (BlazeModule module : runtime.getBlazeModules()) { + try { + module.beforeCommand(runtime, commandAnnotation); + } catch (AbruptExitException e) { + // Don't let one module's complaints prevent the other modules from doing necessary + // setup. We promised to call beforeCommand exactly once per-module before each command + // and will be calling afterCommand soon in the future - a module's afterCommand might + // rightfully assume its beforeCommand has already been called. + outErr.printErrLn(e.getMessage()); + // It's not ideal but we can only return one exit code, so we just pick the code of the + // last exception. + exitCausingException = e; + } + } + if (exitCausingException != null) { + return exitCausingException.getExitCode().getNumericExitCode(); + } + + try { + Path commandLog = getCommandLogPath(runtime.getOutputBase()); + + // Unlink old command log from previous build, if present, so scripts + // reading it don't conflate it with the command log we're about to write. + commandLog.delete(); + + logOutputStream = commandLog.getOutputStream(); + outErr = tee(originalOutErr, OutErr.create(logOutputStream, logOutputStream)); + } catch (IOException ioException) { + LoggingUtil.logToRemote( + Level.WARNING, "Unable to delete or open command.log", ioException); + } + + // Create the UUID for this command. + runtime.setCommandId(UUID.randomUUID()); + + ExitCode result = checkCwdInWorkspace(commandAnnotation, commandName, outErr); + if (result != ExitCode.SUCCESS) { + return result.getNumericExitCode(); + } + + OptionsParser optionsParser; + CommonCommandOptions commonOptions; + // Delay output of notes regarding the parsed rc file, so it's possible to disable this in the + // rc file. + List<String> rcfileNotes = new ArrayList<>(); + try { + optionsParser = createOptionsParser(command); + commonOptions = checkOptions(optionsParser, commandAnnotation, args, rcfileNotes, outErr); + } catch (OptionsParsingException e) { + for (String note : rcfileNotes) { + outErr.printErrLn("INFO: " + note); + } + outErr.printErrLn(e.getMessage()); + return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode(); + } + + // Setup log filtering + BlazeCommandEventHandler.Options eventHandlerOptions = + optionsParser.getOptions(BlazeCommandEventHandler.Options.class); + if (!eventHandlerOptions.useColor()) { + if (!commandAnnotation.binaryStdOut()) { + outErr = ansiStripOut(outErr); + } + + if (!commandAnnotation.binaryStdErr()) { + outErr = ansiStripErr(outErr); + } + } + + BlazeRuntime.setupLogging(commonOptions.verbosity); + + // Do this before an actual crash so we don't have to worry about + // allocating memory post-crash. + String[] crashData = runtime.getCrashData(); + int numericExitCode = ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode(); + PrintStream savedOut = System.out; + PrintStream savedErr = System.err; + + EventHandler handler = createEventHandler(outErr, eventHandlerOptions); + Reporter reporter = runtime.getReporter(); + reporter.addHandler(handler); + try { + // While a Blaze command is active, direct all errors to the client's + // event handler (and out/err streams). + OutErr reporterOutErr = reporter.getOutErr(); + System.setOut(new PrintStream(reporterOutErr.getOutputStream(), /*autoflush=*/true)); + System.setErr(new PrintStream(reporterOutErr.getErrorStream(), /*autoflush=*/true)); + + if (commonOptions.announceRcOptions) { + for (String note : rcfileNotes) { + reporter.handle(Event.info(note)); + } + } + + try { + // Notify the BlazeRuntime, so it can do some initial setup. + runtime.beforeCommand(commandName, optionsParser, commonOptions, execStartTimeNanos); + // Allow the command to edit options after parsing: + command.editOptions(runtime, optionsParser); + } catch (AbruptExitException e) { + reporter.handle(Event.error(e.getMessage())); + return e.getExitCode().getNumericExitCode(); + } + + // Print warnings for odd options usage + for (String warning : optionsParser.getWarnings()) { + reporter.handle(Event.warn(warning)); + } + + ExitCode outcome = command.exec(runtime, optionsParser); + outcome = runtime.precompleteCommand(outcome); + numericExitCode = outcome.getNumericExitCode(); + return numericExitCode; + } catch (ShutdownBlazeServerException e) { + numericExitCode = e.getExitStatus(); + throw e; + } catch (Throwable e) { + BugReport.printBug(outErr, e); + BugReport.sendBugReport(e, args, crashData); + numericExitCode = e instanceof OutOfMemoryError + ? ExitCode.OOM_ERROR.getNumericExitCode() + : ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode(); + throw new ShutdownBlazeServerException(numericExitCode, e); + } finally { + runtime.afterCommand(numericExitCode); + // Swallow IOException, as we are already in a finally clause + Flushables.flushQuietly(outErr.getOutputStream()); + Flushables.flushQuietly(outErr.getErrorStream()); + + System.setOut(savedOut); + System.setErr(savedErr); + reporter.removeHandler(handler); + releaseHandler(handler); + runtime.getTimestampGranularityMonitor().waitForTimestampGranularity(outErr); + } + } + + /** + * For testing ONLY. Same as {@link #exec(List, OutErr, long)}, but automatically uses the current + * time. + */ + @VisibleForTesting + public int exec(List<String> args, OutErr originalOutErr) throws ShutdownBlazeServerException { + return exec(args, originalOutErr, runtime.getClock().currentTimeMillis()); + } + + /** + * Parses the options from .rc files for a command invocation. It works in one of two modes; + * either it loads the non-config options, or the config options that are specified in the {@code + * configs} parameter. + * + * <p>This method adds every option pertaining to the specified command to the options parser. To + * do that, it needs the command -> option mapping that is generated from the .rc files. + * + * <p>It is not as trivial as simply taking the list of options for the specified command because + * commands can inherit arguments from each other, and we have to respect that (e.g. if an option + * is specified for 'build', it needs to take effect for the 'test' command, too). + * + * <p>Note that the order in which the options are parsed is well-defined: all options from the + * same rc file are parsed at the same time, and the rc files are handled in the order in which + * they were passed in from the client. + * + * @param rcfileNotes note message that would be printed during parsing + * @param commandAnnotation the command for which options should be parsed. + * @param optionsParser parser to receive parsed options. + * @param optionsMap .rc files in structured format: a list of pairs, where the first part is the + * name of the rc file, and the second part is a multimap of command name (plus config, if + * present) to the list of options for that command + * @param configs the configs for which to parse options; if {@code null}, non-config options are + * parsed + * @throws OptionsParsingException + */ + protected static void parseOptionsForCommand(List<String> rcfileNotes, Command commandAnnotation, + OptionsParser optionsParser, List<Pair<String, ListMultimap<String, String>>> optionsMap, + Iterable<String> configs) throws OptionsParsingException { + for (String commandToParse : getCommandNamesToParse(commandAnnotation)) { + for (Pair<String, ListMultimap<String, String>> entry : optionsMap) { + List<String> allOptions = new ArrayList<>(); + if (configs == null) { + allOptions.addAll(entry.second.get(commandToParse)); + } else { + for (String config : configs) { + allOptions.addAll(entry.second.get(commandToParse + ":" + config)); + } + } + processOptionList(optionsParser, commandToParse, + commandAnnotation.name(), rcfileNotes, entry.first, allOptions); + if (allOptions.isEmpty()) { + continue; + } + } + } + } + + // Processes the option list for an .rc file - command pair. + private static void processOptionList(OptionsParser optionsParser, String commandToParse, + String originalCommand, List<String> rcfileNotes, String rcfile, List<String> rcfileOptions) + throws OptionsParsingException { + if (!rcfileOptions.isEmpty()) { + String inherited = commandToParse.equals(originalCommand) ? "" : "Inherited "; + rcfileNotes.add("Reading options for '" + originalCommand + + "' from " + rcfile + ":\n" + + " " + inherited + "'" + commandToParse + "' options: " + + Joiner.on(' ').join(rcfileOptions)); + optionsParser.parse(OptionPriority.RC_FILE, rcfile, rcfileOptions); + } + } + + private static List<String> getCommandNamesToParse(Command commandAnnotation) { + List<String> result = new ArrayList<>(); + getCommandNamesToParseHelper(commandAnnotation, result); + result.add("common"); + // TODO(bazel-team): This statement is a NO-OP: Lists.reverse(result); + return result; + } + + private static void getCommandNamesToParseHelper(Command commandAnnotation, + List<String> accumulator) { + for (Class<? extends BlazeCommand> base : commandAnnotation.inherits()) { + getCommandNamesToParseHelper(base.getAnnotation(Command.class), accumulator); + } + accumulator.add(commandAnnotation.name()); + } + + private OutErr ansiStripOut(OutErr outErr) { + OutputStream wrappedOut = new AnsiStrippingOutputStream(outErr.getOutputStream()); + return OutErr.create(wrappedOut, outErr.getErrorStream()); + } + + private OutErr ansiStripErr(OutErr outErr) { + OutputStream wrappedErr = new AnsiStrippingOutputStream(outErr.getErrorStream()); + return OutErr.create(outErr.getOutputStream(), wrappedErr); + } + + private String getNotInRealWorkspaceError(Path doNotBuildFile) { + String message = "Blaze should not be called from a Blaze output directory. "; + try { + String realWorkspace = + new String(FileSystemUtils.readContentAsLatin1(doNotBuildFile)); + message += String.format("The pertinent workspace directory is: '%s'", + realWorkspace); + } catch (IOException e) { + // We are exiting anyway. + } + + return message; + } + + /** + * For a given output_base directory, returns the command log file path. + */ + public static Path getCommandLogPath(Path outputBase) { + return outputBase.getRelative("command.log"); + } + + private OutErr tee(OutErr outErr1, OutErr outErr2) { + DelegatingOutErr outErr = new DelegatingOutErr(); + outErr.addSink(outErr1); + outErr.addSink(outErr2); + return outErr; + } + + private void closeSilently(OutputStream logOutputStream) { + if (logOutputStream != null) { + try { + logOutputStream.close(); + } catch (IOException e) { + LoggingUtil.logToRemote(Level.WARNING, "Unable to close command.log", e); + } + } + } + + /** + * Creates an option parser using the common options classes and the + * command-specific options classes. + * + * <p>An overriding method should first call this method and can then + * override default values directly or by calling {@link + * #parseOptionsForCommand} for command-specific options. + * + * @throws OptionsParsingException + */ + protected OptionsParser createOptionsParser(BlazeCommand command) + throws OptionsParsingException { + Command annotation = command.getClass().getAnnotation(Command.class); + List<Class<? extends OptionsBase>> allOptions = Lists.newArrayList(); + allOptions.addAll(BlazeCommandUtils.getOptions( + command.getClass(), getRuntime().getBlazeModules(), getRuntime().getRuleClassProvider())); + OptionsParser parser = OptionsParser.newOptionsParser(allOptions); + parser.setAllowResidue(annotation.allowResidue()); + return parser; + } + + /** + * Convert a list of option override specifications to a more easily digestible + * form. + * + * @param overrides list of option override specifications + */ + @VisibleForTesting + static List<Pair<String, ListMultimap<String, String>>> getOptionsMap( + OutErr outErr, + List<String> rcFiles, + List<CommonCommandOptions.OptionOverride> overrides, + Set<String> validCommands) { + List<Pair<String, ListMultimap<String, String>>> result = new ArrayList<>(); + + String lastRcFile = null; + ListMultimap<String, String> lastMap = null; + for (CommonCommandOptions.OptionOverride override : overrides) { + if (override.blazeRc < 0 || override.blazeRc >= rcFiles.size()) { + outErr.printErrLn("WARNING: inconsistency in generated command line " + + "args. Ignoring bogus argument\n"); + continue; + } + String rcFile = rcFiles.get(override.blazeRc); + + String command = override.command; + int index = command.indexOf(':'); + if (index > 0) { + command = command.substring(0, index); + } + if (!validCommands.contains(command) && !command.equals("common")) { + outErr.printErrLn("WARNING: while reading option defaults file '" + + rcFile + "':\n" + + " invalid command name '" + override.command + "'."); + continue; + } + + if (!rcFile.equals(lastRcFile)) { + if (lastRcFile != null) { + result.add(Pair.of(lastRcFile, lastMap)); + } + lastRcFile = rcFile; + lastMap = ArrayListMultimap.create(); + } + lastMap.put(override.command, override.option); + } + if (lastRcFile != null) { + result.add(Pair.of(lastRcFile, lastMap)); + } + + return result; + } + + /** + * Returns the event handler to use for this Blaze command. + */ + private EventHandler createEventHandler(OutErr outErr, + BlazeCommandEventHandler.Options eventOptions) { + EventHandler eventHandler; + if ((eventOptions.useColor() || eventOptions.useCursorControl())) { + eventHandler = new FancyTerminalEventHandler(outErr, eventOptions); + } else { + eventHandler = new BlazeCommandEventHandler(outErr, eventOptions); + } + + return RateLimitingEventHandler.create(eventHandler, eventOptions.showProgressRateLimit); + } + + /** + * Unsets the event handler. + */ + private void releaseHandler(EventHandler eventHandler) { + if (eventHandler instanceof FancyTerminalEventHandler) { + // Make sure that the terminal state of the old event handler is clear + // before creating a new one. + ((FancyTerminalEventHandler)eventHandler).resetTerminal(); + } + } + + /** + * Returns the runtime instance shared by the commands that this dispatcher + * dispatches to. + */ + public BlazeRuntime getRuntime() { + return runtime; + } + + /** + * The map from command names to commands that this dispatcher dispatches to. + */ + Map<String, BlazeCommand> getCommandsByName() { + return Collections.unmodifiableMap(commandsByName); + } + + /** + * Shuts down all the registered commands to give them a chance to cleanup or + * close resources. Should be called by the owner of this command dispatcher + * in all termination cases. + */ + public void shutdown() { + closeSilently(logOutputStream); + logOutputStream = null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java new file mode 100644 index 0000000..603b0be --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java
@@ -0,0 +1,246 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.common.options.EnumConverter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.EnumSet; +import java.util.Set; + +/** + * BlazeCommandEventHandler: an event handler established for the duration of a + * single Blaze command. + */ +public class BlazeCommandEventHandler implements EventHandler { + + public enum UseColor { YES, NO, AUTO } + public enum UseCurses { YES, NO, AUTO } + + public static class UseColorConverter extends EnumConverter<UseColor> { + public UseColorConverter() { + super(UseColor.class, "--color setting"); + } + } + + public static class UseCursesConverter extends EnumConverter<UseCurses> { + public UseCursesConverter() { + super(UseCurses.class, "--curses setting"); + } + } + + public static class Options extends OptionsBase { + + @Option(name = "show_progress", + defaultValue = "true", + category = "verbosity", + help = "Display progress messages during a build.") + public boolean showProgress; + + @Option(name = "show_task_finish", + defaultValue = "false", + category = "verbosity", + help = "Display progress messages when tasks complete, not just when they start.") + public boolean showTaskFinish; + + @Option(name = "show_progress_rate_limit", + defaultValue = "0.03", // A nice middle ground; snappy but not too spammy in logs. + category = "verbosity", + help = "Minimum number of seconds between progress messages in the output.") + public double showProgressRateLimit; + + @Option(name = "color", + defaultValue = "auto", + converter = UseColorConverter.class, + category = "verbosity", + help = "Use terminal controls to colorize output.") + public UseColor useColorEnum; + + @Option(name = "curses", + defaultValue = "auto", + converter = UseCursesConverter.class, + category = "verbosity", + help = "Use terminal cursor controls to minimize scrolling output") + public UseCurses useCursesEnum; + + @Option(name = "terminal_columns", + defaultValue = "80", + category = "hidden", + help = "A system-generated parameter which specifies the terminal " + + " width in columns.") + public int terminalColumns; + + @Option(name = "isatty", + defaultValue = "false", + category = "hidden", + help = "A system-generated parameter which is used to notify the " + + "server whether this client is running in a terminal. " + + "If this is set to false, then '--color=auto' will be treated as '--color=no'. " + + "If this is set to true, then '--color=auto' will be treated as '--color=yes'.") + public boolean isATty; + + // This lives here (as opposed to the more logical BuildRequest.Options) + // because the client passes it to the server *always*. We don't want the + // client to have to figure out when it should or shouldn't to send it. + @Option(name = "emacs", + defaultValue = "false", + category = "undocumented", + help = "A system-generated parameter which is true iff EMACS=t in the environment of " + + "the client. This option controls certain display features.") + public boolean runningInEmacs; + + @Option(name = "show_timestamps", + defaultValue = "false", + category = "verbosity", + help = "Include timestamps in messages") + public boolean showTimestamp; + + @Option(name = "progress_in_terminal_title", + defaultValue = "false", + category = "verbosity", + help = "Show the command progress in the terminal title. " + + "Useful to see what blaze is doing when having multiple terminal tabs.") + public boolean progressInTermTitle; + + + public boolean useColor() { + return useColorEnum == UseColor.YES || (useColorEnum == UseColor.AUTO && isATty); + } + + public boolean useCursorControl() { + return useCursesEnum == UseCurses.YES || (useCursesEnum == UseCurses.AUTO && isATty); + } + } + + private static final DateTimeFormatter TIMESTAMP_FORMAT = + DateTimeFormat.forPattern("(MM-dd HH:mm:ss.SSS) "); + + protected final OutErr outErr; + + private final PrintStream errPrintStream; + + protected final Set<EventKind> eventMask = + EnumSet.copyOf(EventKind.ERRORS_WARNINGS_AND_INFO_AND_OUTPUT); + + protected final boolean showTimestamp; + + public BlazeCommandEventHandler(OutErr outErr, Options eventOptions) { + this.outErr = outErr; + this.errPrintStream = new PrintStream(outErr.getErrorStream(), true); + if (eventOptions.showProgress) { + eventMask.add(EventKind.PROGRESS); + eventMask.add(EventKind.START); + } else { + // Skip PASS events if --noshow_progress is requested. + eventMask.remove(EventKind.PASS); + } + if (eventOptions.showTaskFinish) { + eventMask.add(EventKind.FINISH); + } + eventMask.add(EventKind.SUBCOMMAND); + this.showTimestamp = eventOptions.showTimestamp; + } + + /** See EventHandler.handle. */ + @Override + public void handle(Event event) { + if (!eventMask.contains(event.getKind())) { + return; + } + String prefix; + switch (event.getKind()) { + case STDOUT: + putOutput(outErr.getOutputStream(), event); + return; + case STDERR: + putOutput(outErr.getErrorStream(), event); + return; + case PASS: + case FAIL: + case TIMEOUT: + case ERROR: + case WARNING: + case DEPCHECKER: + prefix = event.getKind() + ": "; + break; + case SUBCOMMAND: + prefix = ">>>>>>>>> "; + break; + case INFO: + case PROGRESS: + case START: + case FINISH: + prefix = "____"; + break; + default: + throw new IllegalStateException("" + event.getKind()); + } + StringBuilder buf = new StringBuilder(); + buf.append(prefix); + + if (showTimestamp) { + buf.append(timestamp()); + } + + Location location = event.getLocation(); + if (location != null) { + buf.append(location.print()).append(": "); + } + + buf.append(event.getMessage()); + if (event.getKind() == EventKind.FINISH) { + buf.append(" DONE"); + } + + // Add a trailing period for ERROR and WARNING messages, which are + // typically English sentences composed from exception messages. + if (event.getKind() == EventKind.WARNING || + event.getKind() == EventKind.ERROR) { + buf.append('.'); + } + + // Event messages go to stderr; results (e.g. 'blaze query') go to stdout. + errPrintStream.println(buf); + } + + private void putOutput(OutputStream out, Event event) { + try { + out.write(event.getMessageBytes()); + out.flush(); + } catch (IOException e) { + // This can happen in server mode if the blaze client has exited, + // or if output is redirected to a file and the disk is full, etc. + // Ignore. + } + } + + /** + * @return a string representing the current time, eg "04-26 13:47:32.124". + */ + protected String timestamp() { + return TIMESTAMP_FORMAT.print(System.currentTimeMillis()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandUtils.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandUtils.java new file mode 100644 index 0000000..ff738db --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandUtils.java
@@ -0,0 +1,166 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.util.ResourceFileLoader; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Utility class for functionality related to Blaze commands. + */ +public class BlazeCommandUtils { + /** + * Options classes used as startup options in Blaze core. + */ + private static final List<Class<? extends OptionsBase>> DEFAULT_STARTUP_OPTIONS = + ImmutableList.<Class<? extends OptionsBase>>of( + BlazeServerStartupOptions.class, + HostJvmStartupOptions.class); + + /** + * The set of option-classes that are common to all Blaze commands. + */ + private static final Collection<Class<? extends OptionsBase>> COMMON_COMMAND_OPTIONS = + ImmutableList.of(CommonCommandOptions.class, BlazeCommandEventHandler.Options.class); + + + private BlazeCommandUtils() {} + + public static ImmutableList<Class<? extends OptionsBase>> getStartupOptions( + Iterable<BlazeModule> modules) { + Set<Class<? extends OptionsBase>> options = new HashSet<>(); + options.addAll(DEFAULT_STARTUP_OPTIONS); + for (BlazeModule blazeModule : modules) { + Iterables.addAll(options, blazeModule.getStartupOptions()); + } + + return ImmutableList.copyOf(options); + } + + /** + * Returns the set of all options (including those inherited directly and + * transitively) for this AbstractCommand's @Command annotation. + * + * <p>Why does metaprogramming always seem like such a bright idea in the + * beginning? + */ + public static ImmutableList<Class<? extends OptionsBase>> getOptions( + Class<? extends BlazeCommand> clazz, + Iterable<BlazeModule> modules, + ConfiguredRuleClassProvider ruleClassProvider) { + Command commandAnnotation = clazz.getAnnotation(Command.class); + if (commandAnnotation == null) { + throw new IllegalStateException("@Command missing for " + clazz.getName()); + } + + Set<Class<? extends OptionsBase>> options = new HashSet<>(); + options.addAll(COMMON_COMMAND_OPTIONS); + Collections.addAll(options, commandAnnotation.options()); + + if (commandAnnotation.usesConfigurationOptions()) { + options.addAll(ruleClassProvider.getConfigurationOptions()); + } + + for (BlazeModule blazeModule : modules) { + Iterables.addAll(options, blazeModule.getCommandOptions(commandAnnotation)); + } + + for (Class<? extends BlazeCommand> base : commandAnnotation.inherits()) { + options.addAll(getOptions(base, modules, ruleClassProvider)); + } + return ImmutableList.copyOf(options); + } + + /** + * Returns the expansion of the specified help topic. + * + * @param topic the name of the help topic; used in %{command} expansion. + * @param help the text template of the help message. Certain %{x} variables + * will be expanded. A prefix of "resource:" means use the .jar + * resource of that name. + * @param categoryDescriptions a mapping from option category names to + * descriptions, passed to {@link OptionsParser#describeOptions}. + * @param helpVerbosity a tri-state verbosity option selecting between just + * names, names and syntax, and full description. + */ + public static final String expandHelpTopic(String topic, String help, + Class<? extends BlazeCommand> commandClass, + Collection<Class<? extends OptionsBase>> options, + Map<String, String> categoryDescriptions, + OptionsParser.HelpVerbosity helpVerbosity) { + OptionsParser parser = OptionsParser.newOptionsParser(options); + + String template; + if (help.startsWith("resource:")) { + String resourceName = help.substring("resource:".length()); + try { + template = ResourceFileLoader.loadResource(commandClass, resourceName); + } catch (IOException e) { + throw new IllegalStateException("failed to load help resource '" + resourceName + + "' due to I/O error: " + e.getMessage(), e); + } + } else { + template = help; + } + + if (!template.contains("%{options}")) { + throw new IllegalStateException("Help template for '" + topic + "' omits %{options}!"); + } + + return template. + replace("%{command}", topic). + replace("%{options}", parser.describeOptions(categoryDescriptions, helpVerbosity)). + trim() + + "\n\n" + + (helpVerbosity == OptionsParser.HelpVerbosity.MEDIUM + ? "(Use 'help --long' for full details or --short to just enumerate options.)\n" + : ""); + } + + /** + * The help page for this command. + * + * @param categoryDescriptions a mapping from option category names to + * descriptions, passed to {@link OptionsParser#describeOptions}. + * @param verbosity a tri-state verbosity option selecting between just names, + * names and syntax, and full description. + */ + public static String getUsage( + Class<? extends BlazeCommand> commandClass, + Map<String, String> categoryDescriptions, + OptionsParser.HelpVerbosity verbosity, + Iterable<BlazeModule> blazeModules, + ConfiguredRuleClassProvider ruleClassProvider) { + Command commandAnnotation = commandClass.getAnnotation(Command.class); + return BlazeCommandUtils.expandHelpTopic( + commandAnnotation.name(), + commandAnnotation.help(), + commandClass, + BlazeCommandUtils.getOptions(commandClass, blazeModules, ruleClassProvider), + categoryDescriptions, + verbosity); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java new file mode 100644 index 0000000..6855cbd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java
@@ -0,0 +1,420 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Predicate; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.ActionContextConsumer; +import com.google.devtools.build.lib.actions.ActionContextProvider; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.exec.OutputService; +import com.google.devtools.build.lib.packages.MakeEnvironment; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.PackageFactory.PackageArgument; +import com.google.devtools.build.lib.packages.Preprocessor; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; +import com.google.devtools.build.lib.query2.output.OutputFormatter; +import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory; +import com.google.devtools.build.lib.skyframe.DiffAwareness; +import com.google.devtools.build.lib.skyframe.PrecomputedValue.Injected; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.skyframe.SkyframeExecutorFactory; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import javax.annotation.Nullable; + +/** + * A module Blaze can load at the beginning of its execution. Modules are supplied with extension + * points to augment the functionality at specific, well-defined places. + * + * <p>The constructors of individual Blaze modules should be empty. All work should be done in the + * methods (e.g. {@link #blazeStartup}). + */ +public abstract class BlazeModule { + + /** + * Returns the extra startup options this module contributes. + * + * <p>This method will be called at the beginning of Blaze startup (before #blazeStartup). + */ + public Iterable<Class<? extends OptionsBase>> getStartupOptions() { + return ImmutableList.of(); + } + + /** + * Called before {@link #getFileSystem} and {@link #blazeStartup}. + * + * <p>This method will be called at the beginning of Blaze startup. + */ + @SuppressWarnings("unused") + public void globalInit(OptionsProvider startupOptions) throws AbruptExitException { + } + + /** + * Returns the file system implementation used by Blaze. It is an error if more than one module + * returns a file system. If all return null, the default unix file system is used. + * + * <p>This method will be called at the beginning of Blaze startup (in-between #globalInit and + * #blazeStartup). + */ + @SuppressWarnings("unused") + public FileSystem getFileSystem(OptionsProvider startupOptions, PathFragment outputPath) { + return null; + } + + /** + * Called when Blaze starts up. + */ + @SuppressWarnings("unused") + public void blazeStartup(OptionsProvider startupOptions, + BlazeVersionInfo versionInfo, UUID instanceId, BlazeDirectories directories, + Clock clock) throws AbruptExitException { + } + + /** + * Returns the set of directories under which blaze may assume all files are immutable. + */ + public Set<Path> getImmutableDirectories() { + return ImmutableSet.<Path>of(); + } + + /** + * May yield a supplier that provides factories for the Preprocessor to apply. Only one of the + * configured modules may return non-null. + * + * The factory yielded by the supplier will be checked with + * {@link Preprocessor.Factory#isStillValid} at the beginning of each incremental build. This + * allows modules to have preprocessors customizable by flags. + * + * <p>This method will be called during Blaze startup (after #blazeStartup). + */ + public Preprocessor.Factory.Supplier getPreprocessorFactorySupplier() { + return null; + } + + /** + * Adds the rule classes supported by this module. + * + * <p>This method will be called during Blaze startup (after #blazeStartup). + */ + @SuppressWarnings("unused") + public void initializeRuleClasses(ConfiguredRuleClassProvider.Builder builder) { + } + + /** + * Returns the list of commands this module contributes to Blaze. + * + * <p>This method will be called during Blaze startup (after #blazeStartup). + */ + public Iterable<? extends BlazeCommand> getCommands() { + return ImmutableList.of(); + } + + /** + * Returns the list of query output formatters this module provides. + * + * <p>This method will be called during Blaze startup (after #blazeStartup). + */ + public Iterable<OutputFormatter> getQueryOutputFormatters() { + return ImmutableList.of(); + } + + /** + * Returns the {@link DiffAwareness} strategies this module contributes. These will be used to + * determine which files, if any, changed between Blaze commands. + * + * <p>This method will be called during Blaze startup (after #blazeStartup). + */ + @SuppressWarnings("unused") + public Iterable<? extends DiffAwareness.Factory> getDiffAwarenessFactories(boolean watchFS) { + return ImmutableList.of(); + } + + /** + * Returns the workspace status action factory contributed by this module. + * + * <p>There should always be exactly one of these in a Blaze instance. + */ + public WorkspaceStatusAction.Factory getWorkspaceStatusActionFactory() { + return null; + } + + /** + * PlatformSet is a group of platforms characterized by a regular expression. For example, the + * entry "oldlinux": "i[34]86-libc[345]-linux" might define a set of platforms representing + * certain older linux releases. + * + * <p>Platform-set names are used in BUILD files in the third argument to <tt>vardef</tt>, to + * define per-platform tweaks to variables such as CFLAGS. + * + * <p>vardef is a legacy mechanism: it needs explicit support in the rule implementations, + * and cannot express conditional dependencies, only conditional attribute values. This + * mechanism will be supplanted by configuration dependent attributes, and its effect can + * usually also be achieved with abi_deps. + * + * <p>This method will be called during Blaze startup (after #blazeStartup). + */ + public Map<String, String> getPlatformSetRegexps() { + return ImmutableMap.<String, String>of(); + } + + /** + * Services provided for Blaze modules via BlazeRuntime. + */ + public interface ModuleEnvironment { + /** + * Gets a file from the depot based on its label and returns the {@link Path} where it can + * be found. + */ + Path getFileFromDepot(Label label) + throws NoSuchThingException, InterruptedException, IOException; + + /** + * Exits Blaze as early as possible. This is currently a hack and should only be called in + * event handlers for {@code BuildStartingEvent}, {@code GotOptionsEvent} and + * {@code LoadingPhaseCompleteEvent}. + */ + void exit(AbruptExitException exception); + } + + /** + * Called before each command. + */ + @SuppressWarnings("unused") + public void beforeCommand(BlazeRuntime blazeRuntime, Command command) + throws AbruptExitException { + } + + /** + * Returns the output service to be used. It is an error if more than one module returns an + * output service. + * + * <p>This method will be called at the beginning of each command (after #beforeCommand). + */ + @SuppressWarnings("unused") + public OutputService getOutputService() throws AbruptExitException { + return null; + } + + /** + * Returns the extra options this module contributes to a specific command. + * + * <p>This method will be called at the beginning of each command (after #beforeCommand). + */ + @SuppressWarnings("unused") + public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) { + return ImmutableList.of(); + } + + /** + * Returns a map of option categories to descriptive strings. This is used by {@code HelpCommand} + * to show a more readable list of flags. + */ + public Map<String, String> getOptionCategories() { + return ImmutableMap.of(); + } + + /** + * A item that is returned by "blaze info". + */ + public interface InfoItem { + /** + * The name of the info key. + */ + String getName(); + + /** + * The help description of the info key. + */ + String getDescription(); + + /** + * Whether the key is printed when "blaze info" is invoked without arguments. + * + * <p>This is usually true for info keys that take multiple lines, thus, cannot really be + * included in the output of argumentless "blaze info". + */ + boolean isHidden(); + + /** + * Returns the value of the info key. The return value is directly printed to stdout. + */ + byte[] get(Supplier<BuildConfiguration> configurationSupplier) throws AbruptExitException; + } + + /** + * Returns the additional information this module provides to "blaze info". + * + * <p>This method will be called at the beginning of each "blaze info" command (after + * #beforeCommand). + */ + public Iterable<InfoItem> getInfoItems() { + return ImmutableList.of(); + } + + /** + * Returns the list of query functions this module provides to "blaze query". + * + * <p>This method will be called at the beginning of each "blaze query" command (after + * #beforeCommand). + */ + public Iterable<QueryFunction> getQueryFunctions() { + return ImmutableList.of(); + } + + /** + * Returns the action context provider the module contributes to Blaze, if any. + * + * <p>This method will be called at the beginning of the execution phase, e.g. of the + * "blaze build" command. + */ + public ActionContextProvider getActionContextProvider() { + return null; + } + + /** + * Returns the action context consumer that pulls in action contexts required by this module, + * if any. + * + * <p>This method will be called at the beginning of the execution phase, e.g. of the + * "blaze build" command. + */ + public ActionContextConsumer getActionContextConsumer() { + return null; + } + + /** + * Called after each command. + */ + public void afterCommand() { + } + + /** + * Called when Blaze shuts down. + */ + public void blazeShutdown() { + } + + /** + * Action inputs are allowed to be missing for all inputs where this predicate returns true. + */ + public Predicate<PathFragment> getAllowedMissingInputs() { + return null; + } + + /** + * Optionally specializes the cache that ensures source files are looked at just once during + * a build. Only one module may do so. + */ + public ActionInputFileCache createActionInputCache(String cwd, FileSystem fs) { + return null; + } + + /** + * Returns the extensions this module contributes to the global namespace of the BUILD language. + */ + public PackageFactory.EnvironmentExtension getPackageEnvironmentExtension() { + return new PackageFactory.EnvironmentExtension() { + @Override + public void update( + Environment environment, MakeEnvironment.Builder pkgMakeEnv, Label buildFileLabel) { + } + + @Override + public Iterable<PackageArgument<?>> getPackageArguments() { + return ImmutableList.of(); + } + }; + } + + /** + * Returns a factory for creating {@link SkyframeExecutor} objects. If the module does not + * provide any SkyframeExecutorFactory, it returns null. Note that only one factory per + * Bazel/Blaze runtime is allowed. + */ + public SkyframeExecutorFactory getSkyframeExecutorFactory() { + return null; + } + + /** Returns a map of "extra" SkyFunctions for SkyValues that this module may want to build. */ + public ImmutableMap<SkyFunctionName, SkyFunction> getSkyFunctions(BlazeDirectories directories) { + return ImmutableMap.of(); + } + + /** + * Returns the extra precomputed values that the module makes available in Skyframe. + * + * <p>This method is called once per Blaze instance at the very beginning of its life. + * If it creates the injected values by using a {@code com.google.common.base.Supplier}, + * that supplier is asked for the value it contains just before the loading phase begins. This + * functionality can be used to implement precomputed values that are not constant during the + * lifetime of a Blaze instance (naturally, they must be constant over the course of a build) + * + * <p>The following things must be done in order to define a new precomputed values: + * <ul> + * <li> Create a public static final variable of type + * {@link com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed} + * <li> Set its value by adding an {@link Injected} in this method (it can be created using the + * aforementioned variable and the value or a supplier of the value) + * <li> Reference the value in Skyframe functions by calling get {@code get} method on the + * {@link com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed} variable. This + * will never return null, because its value will have been injected before most of the Skyframe + * values are computed. + * </ul> + */ + public Iterable<Injected> getPrecomputedSkyframeValues() { + return ImmutableList.of(); + } + + /** + * Optionally returns a provider for project files that can be used to bundle targets and + * command-line options. + */ + @Nullable + public ProjectFile.Provider createProjectFileProvider() { + return null; + } + + /** + * Optionally returns a factory to create coverage report actions. + */ + @Nullable + public CoverageReportActionFactory getCoverageReportFactory() { + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java new file mode 100644 index 0000000..0251e83 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
@@ -0,0 +1,1795 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.SubscriberExceptionContext; +import com.google.common.eventbus.SubscriberExceptionHandler; +import com.google.common.util.concurrent.Uninterruptibles; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.actions.cache.ActionCache; +import com.google.devtools.build.lib.actions.cache.CompactPersistentActionCache; +import com.google.devtools.build.lib.actions.cache.NullActionCache; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.analysis.BuildView; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationFactory; +import com.google.devtools.build.lib.analysis.config.DefaultsPackage; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.buildtool.BuildTool; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.OutputFilter; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.exec.OutputService; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.Preprocessor; +import com.google.devtools.build.lib.packages.RuleClassProvider; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.pkgcache.PackageManager; +import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator; +import com.google.devtools.build.lib.profiler.MemoryProfiler; +import com.google.devtools.build.lib.profiler.ProfilePhase; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.Profiler.ProfiledTaskKinds; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.query2.output.OutputFormatter; +import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory; +import com.google.devtools.build.lib.runtime.commands.BuildCommand; +import com.google.devtools.build.lib.runtime.commands.CanonicalizeCommand; +import com.google.devtools.build.lib.runtime.commands.CleanCommand; +import com.google.devtools.build.lib.runtime.commands.HelpCommand; +import com.google.devtools.build.lib.runtime.commands.InfoCommand; +import com.google.devtools.build.lib.runtime.commands.ProfileCommand; +import com.google.devtools.build.lib.runtime.commands.QueryCommand; +import com.google.devtools.build.lib.runtime.commands.RunCommand; +import com.google.devtools.build.lib.runtime.commands.ShutdownCommand; +import com.google.devtools.build.lib.runtime.commands.SkylarkCommand; +import com.google.devtools.build.lib.runtime.commands.TestCommand; +import com.google.devtools.build.lib.runtime.commands.VersionCommand; +import com.google.devtools.build.lib.server.RPCServer; +import com.google.devtools.build.lib.server.ServerCommand; +import com.google.devtools.build.lib.server.signal.InterruptSignalHandler; +import com.google.devtools.build.lib.skyframe.DiffAwareness; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutorFactory; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.skyframe.SkyframeExecutorFactory; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.OsUtils; +import com.google.devtools.build.lib.util.ThreadUtils; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.JavaIoFileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.UnixFileSystem; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionPriority; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsClassProvider; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; +import com.google.devtools.common.options.TriState; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +/** + * The BlazeRuntime class encapsulates the runtime settings and services that + * are available to most parts of any Blaze application for the duration of the + * batch run or server lifetime. A single instance of this runtime will exist + * and will be passed around as needed. + */ +public final class BlazeRuntime { + /** + * The threshold for memory reserved by a 32-bit JVM before trouble may be expected. + * + * <p>After the JVM starts, it reserves memory for heap (controlled by -Xmx) and non-heap + * (code, PermGen, etc.). Furthermore, as Blaze spawns threads, each thread reserves memory + * for the stack (controlled by -Xss). Thus even if Blaze starts fine, with high memory settings + * it will die from a stack allocation failure in the middle of a build. We prefer failing + * upfront by setting a safe threshold. + * + * <p>This does not apply to 64-bit VMs. + */ + private static final long MAX_BLAZE32_RESERVED_MEMORY = 3400 * 1048576L; + + // Less than this indicates tampering with -Xmx settings. + private static final long MIN_BLAZE32_HEAP_SIZE = 3000 * 1000000L; + + public static final String DO_NOT_BUILD_FILE_NAME = "DO_NOT_BUILD_HERE"; + + private static final Pattern suppressFromLog = Pattern.compile(".*(auth|pass|cookie).*", + Pattern.CASE_INSENSITIVE); + + private static final Logger LOG = Logger.getLogger(BlazeRuntime.class.getName()); + + private final BlazeDirectories directories; + private Path workingDirectory; + private long commandStartTime; + + // Application-specified constants + private final PathFragment runfilesPrefix; + + private final SkyframeExecutor skyframeExecutor; + + private final Reporter reporter; + private EventBus eventBus; + private final LoadingPhaseRunner loadingPhaseRunner; + private final PackageFactory packageFactory; + private final ConfigurationFactory configurationFactory; + private final ConfiguredRuleClassProvider ruleClassProvider; + private final BuildView view; + private ActionCache actionCache; + private final TimestampGranularityMonitor timestampGranularityMonitor; + private final Clock clock; + private final BuildTool buildTool; + + private OutputService outputService; + + private final Iterable<BlazeModule> blazeModules; + private final BlazeModule.ModuleEnvironment blazeModuleEnvironment; + + private UUID commandId; // Unique identifier for the command being run + + private final AtomicInteger storedExitCode = new AtomicInteger(); + + private final Map<String, String> clientEnv; + + // We pass this through here to make it available to the MasterLogWriter. + private final OptionsProvider startupOptionsProvider; + + private String outputFileSystem; + private Map<String, BlazeCommand> commandMap; + + private AbruptExitException pendingException; + + private final SubscriberExceptionHandler eventBusExceptionHandler; + + private final BinTools binTools; + + private final WorkspaceStatusAction.Factory workspaceStatusActionFactory; + + private final ProjectFile.Provider projectFileProvider; + + private class BlazeModuleEnvironment implements BlazeModule.ModuleEnvironment { + @Override + public Path getFileFromDepot(Label label) + throws NoSuchThingException, InterruptedException, IOException { + Target target = getPackageManager().getTarget(reporter, label); + return (outputService != null) + ? outputService.stageTool(target) + : target.getPackage().getPackageDirectory().getRelative(target.getName()); + } + + @Override + public void exit(AbruptExitException exception) { + Preconditions.checkState(pendingException == null); + pendingException = exception; + } + } + + private BlazeRuntime(BlazeDirectories directories, Reporter reporter, + WorkspaceStatusAction.Factory workspaceStatusActionFactory, + final SkyframeExecutor skyframeExecutor, + PackageFactory pkgFactory, ConfiguredRuleClassProvider ruleClassProvider, + ConfigurationFactory configurationFactory, PathFragment runfilesPrefix, Clock clock, + OptionsProvider startupOptionsProvider, Iterable<BlazeModule> blazeModules, + Map<String, String> clientEnv, + TimestampGranularityMonitor timestampGranularityMonitor, + SubscriberExceptionHandler eventBusExceptionHandler, + BinTools binTools, ProjectFile.Provider projectFileProvider) { + this.workspaceStatusActionFactory = workspaceStatusActionFactory; + this.directories = directories; + this.workingDirectory = directories.getWorkspace(); + this.reporter = reporter; + this.runfilesPrefix = runfilesPrefix; + this.packageFactory = pkgFactory; + this.binTools = binTools; + this.projectFileProvider = projectFileProvider; + + this.skyframeExecutor = skyframeExecutor; + this.loadingPhaseRunner = new LoadingPhaseRunner( + skyframeExecutor.getPackageManager(), + pkgFactory.getRuleClassNames()); + + this.clientEnv = clientEnv; + + this.blazeModules = blazeModules; + this.ruleClassProvider = ruleClassProvider; + this.configurationFactory = configurationFactory; + this.view = new BuildView(directories, getPackageManager(), ruleClassProvider, + skyframeExecutor, binTools, getCoverageReportActionFactory(blazeModules)); + this.clock = clock; + this.timestampGranularityMonitor = Preconditions.checkNotNull(timestampGranularityMonitor); + this.startupOptionsProvider = startupOptionsProvider; + + this.eventBusExceptionHandler = eventBusExceptionHandler; + this.blazeModuleEnvironment = new BlazeModuleEnvironment(); + this.buildTool = new BuildTool(this); + initEventBus(); + + if (inWorkspace()) { + writeOutputBaseReadmeFile(); + writeOutputBaseDoNotBuildHereFile(); + } + setupExecRoot(); + } + + @Nullable private CoverageReportActionFactory getCoverageReportActionFactory( + Iterable<BlazeModule> blazeModules) { + CoverageReportActionFactory firstFactory = null; + for (BlazeModule module : blazeModules) { + CoverageReportActionFactory factory = module.getCoverageReportFactory(); + if (factory != null) { + Preconditions.checkState(firstFactory == null, + "only one Blaze Module can have a Coverage Report Factory"); + firstFactory = factory; + } + } + return firstFactory; + } + + /** + * Figures out what file system we are writing output to. Here we use + * outputBase instead of outputPath because we need a file system to create the latter. + */ + private String determineOutputFileSystem() { + if (getOutputService() != null) { + return getOutputService().getFilesSystemName(); + } + long startTime = Profiler.nanoTimeMaybe(); + String fileSystem = FileSystemUtils.getFileSystem(getOutputBase()); + Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO, "Finding output file system"); + return fileSystem; + } + + public String getOutputFileSystem() { + return outputFileSystem; + } + + @VisibleForTesting + public void initEventBus() { + setEventBus(new EventBus(eventBusExceptionHandler)); + } + + private void clearEventBus() { + // EventBus does not have an unregister() method, so this is how we release memory associated + // with handlers. + setEventBus(null); + } + + private void setEventBus(EventBus eventBus) { + this.eventBus = eventBus; + skyframeExecutor.setEventBus(eventBus); + } + + /** + * Conditionally enable profiling. + */ + private final boolean initProfiler(CommonCommandOptions options, + UUID buildID, long execStartTimeNanos) { + OutputStream out = null; + boolean recordFullProfilerData = false; + ProfiledTaskKinds profiledTasks = ProfiledTaskKinds.NONE; + + try { + if (options.profilePath != null) { + Path profilePath = getWorkspace().getRelative(options.profilePath); + + recordFullProfilerData = options.recordFullProfilerData; + out = new BufferedOutputStream(profilePath.getOutputStream(), 1024 * 1024); + getReporter().handle(Event.info("Writing profile data to '" + profilePath + "'")); + profiledTasks = ProfiledTaskKinds.ALL; + } else if (options.alwaysProfileSlowOperations) { + recordFullProfilerData = false; + out = null; + profiledTasks = ProfiledTaskKinds.SLOWEST; + } + if (profiledTasks != ProfiledTaskKinds.NONE) { + Profiler.instance().start(profiledTasks, out, + "Blaze profile for " + getOutputBase() + " at " + new Date() + + ", build ID: " + buildID, + recordFullProfilerData, clock, execStartTimeNanos); + return true; + } + } catch (IOException e) { + getReporter().handle(Event.error("Error while creating profile file: " + e.getMessage())); + } + return false; + } + + /** + * Generates a README file in the output base directory. This README file + * contains the name of the workspace directory, so that users can figure out + * which output base directory corresponds to which workspace. + */ + private void writeOutputBaseReadmeFile() { + Preconditions.checkNotNull(getWorkspace()); + Path outputBaseReadmeFile = getOutputBase().getRelative("README"); + try { + FileSystemUtils.writeIsoLatin1(outputBaseReadmeFile, "WORKSPACE: " + getWorkspace(), "", + "The first line of this file is intentionally easy to parse for various", + "interactive scripting and debugging purposes. But please DO NOT write programs", + "that exploit it, as they will be broken by design: it is not possible to", + "reverse engineer the set of source trees or the --package_path from the output", + "tree, and if you attempt it, you will fail, creating subtle and", + "hard-to-diagnose bugs, that will no doubt get blamed on changes made by the", + "Blaze team.", "", "This directory was generated by Blaze.", + "Do not attempt to modify or delete any files in this directory.", + "Among other issues, Blaze's file system caching assumes that", + "only Blaze will modify this directory and the files in it,", + "so if you change anything here you may mess up Blaze's cache."); + } catch (IOException e) { + LOG.warning("Couldn't write to '" + outputBaseReadmeFile + "': " + e.getMessage()); + } + } + + private void writeOutputBaseDoNotBuildHereFile() { + Preconditions.checkNotNull(getWorkspace()); + Path filePath = getOutputBase().getRelative(DO_NOT_BUILD_FILE_NAME); + try { + FileSystemUtils.writeContent(filePath, ISO_8859_1, getWorkspace().toString()); + } catch (IOException e) { + LOG.warning("Couldn't write to '" + filePath + "': " + e.getMessage()); + } + } + + /** + * Creates the execRoot dir under outputBase. + */ + private void setupExecRoot() { + try { + FileSystemUtils.createDirectoryAndParents(directories.getExecRoot()); + } catch (IOException e) { + LOG.warning("failed to create execution root '" + directories.getExecRoot() + "': " + + e.getMessage()); + } + } + + public void recordCommandStartTime(long commandStartTime) { + this.commandStartTime = commandStartTime; + } + + public long getCommandStartTime() { + return commandStartTime; + } + + public String getWorkspaceName() { + Path workspace = directories.getWorkspace(); + if (workspace == null) { + return ""; + } + return workspace.getBaseName(); + } + + /** + * Returns any prefix to be inserted between relative source paths and the runfiles directory. + */ + public PathFragment getRunfilesPrefix() { + return runfilesPrefix; + } + + /** + * Returns the Blaze directories object for this runtime. + */ + public BlazeDirectories getDirectories() { + return directories; + } + + /** + * Returns the working directory of the server. + * + * <p>This is often the first entry on the {@code --package_path}, but not always. + * Callers should certainly not make this assumption. The Path returned may be null. + * + * @see #getWorkingDirectory() + */ + public Path getWorkspace() { + return directories.getWorkspace(); + } + + /** + * Returns the working directory of the {@code blaze} client process. + * + * <p>This may be equal to {@code getWorkspace()}, or beneath it. + * + * @see #getWorkspace() + */ + public Path getWorkingDirectory() { + return workingDirectory; + } + + /** + * Returns if the client passed a valid workspace to be used for the build. + */ + public boolean inWorkspace() { + return directories.inWorkspace(); + } + + /** + * Returns the output base directory associated with this Blaze server + * process. This is the base directory for shared Blaze state as well as tool + * and strategy specific subdirectories. + */ + public Path getOutputBase() { + return directories.getOutputBase(); + } + + /** + * Returns the output path associated with this Blaze server process.. + */ + public Path getOutputPath() { + return directories.getOutputPath(); + } + + /** + * The directory in which blaze stores the server state - that is, the socket + * file and a log. + */ + public Path getServerDirectory() { + return getOutputBase().getChild("server"); + } + + /** + * Returns the execution root directory associated with this Blaze server + * process. This is where all input and output files visible to the actual + * build reside. + */ + public Path getExecRoot() { + return directories.getExecRoot(); + } + + /** + * Returns the reporter for events. + */ + public Reporter getReporter() { + return reporter; + } + + /** + * Returns the current event bus. Only valid within the scope of a single Blaze command. + */ + public EventBus getEventBus() { + return eventBus; + } + + public BinTools getBinTools() { + return binTools; + } + + /** + * Returns the skyframe executor. + */ + public SkyframeExecutor getSkyframeExecutor() { + return skyframeExecutor; + } + + /** + * Returns the package factory. + */ + public PackageFactory getPackageFactory() { + return packageFactory; + } + + /** + * Returns the build tool. + */ + public BuildTool getBuildTool() { + return buildTool; + } + + public ImmutableList<OutputFormatter> getQueryOutputFormatters() { + ImmutableList.Builder<OutputFormatter> result = ImmutableList.builder(); + result.addAll(OutputFormatter.getDefaultFormatters()); + for (BlazeModule module : blazeModules) { + result.addAll(module.getQueryOutputFormatters()); + } + + return result.build(); + } + + /** + * Returns the package manager. + */ + public PackageManager getPackageManager() { + return skyframeExecutor.getPackageManager(); + } + + public WorkspaceStatusAction.Factory getworkspaceStatusActionFactory() { + return workspaceStatusActionFactory; + } + + public BlazeModule.ModuleEnvironment getBlazeModuleEnvironment() { + return blazeModuleEnvironment; + } + + /** + * Returns the rule class provider. + */ + public ConfiguredRuleClassProvider getRuleClassProvider() { + return ruleClassProvider; + } + + public LoadingPhaseRunner getLoadingPhaseRunner() { + return loadingPhaseRunner; + } + + /** + * Returns the build view. + */ + public BuildView getView() { + return view; + } + + public Iterable<BlazeModule> getBlazeModules() { + return blazeModules; + } + + @SuppressWarnings("unchecked") + public <T extends BlazeModule> T getBlazeModule(Class<T> moduleClass) { + for (BlazeModule module : blazeModules) { + if (module.getClass() == moduleClass) { + return (T) module; + } + } + + return null; + } + + public ConfigurationFactory getConfigurationFactory() { + return configurationFactory; + } + + /** + * Returns the target pattern parser. + */ + public TargetPatternEvaluator getTargetPatternEvaluator() { + return loadingPhaseRunner.getTargetPatternEvaluator(); + } + + /** + * Returns reference to the lazily instantiated persistent action cache + * instance. Note, that method may recreate instance between different build + * requests, so return value should not be cached. + */ + public ActionCache getPersistentActionCache() throws IOException { + if (actionCache == null) { + if (OS.getCurrent() == OS.WINDOWS) { + // TODO(bazel-team): Add support for a persistent action cache on Windows. + actionCache = new NullActionCache(); + return actionCache; + } + long startTime = Profiler.nanoTimeMaybe(); + try { + actionCache = new CompactPersistentActionCache(getCacheDirectory(), clock); + } catch (IOException e) { + LOG.log(Level.WARNING, "Failed to load action cache: " + e.getMessage(), e); + LoggingUtil.logToRemote(Level.WARNING, "Failed to load action cache: " + + e.getMessage(), e); + getReporter().handle( + Event.error("Error during action cache initialization: " + e.getMessage() + + ". Corrupted files were renamed to '" + getCacheDirectory() + "/*.bad'. " + + "Blaze will now reset action cache data, causing a full rebuild")); + actionCache = new CompactPersistentActionCache(getCacheDirectory(), clock); + } finally { + Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO, "Loading action cache"); + } + } + return actionCache; + } + + /** + * Removes in-memory caches. + */ + public void clearCaches() throws IOException { + clearSkyframeRelevantCaches(); + actionCache = null; + FileSystemUtils.deleteTree(getCacheDirectory()); + } + + /** Removes skyframe cache and other caches that must be kept synchronized with skyframe. */ + private void clearSkyframeRelevantCaches() { + skyframeExecutor.resetEvaluator(); + view.clear(); + } + + /** + * Returns the TimestampGranularityMonitor. The same monitor object is used + * across multiple Blaze commands, but it doesn't hold any persistent state + * across different commands. + */ + public TimestampGranularityMonitor getTimestampGranularityMonitor() { + return timestampGranularityMonitor; + } + + /** + * Returns path to the cache directory. Path must be inside output base to + * ensure that users can run concurrent instances of blaze in different + * clients without attempting to concurrently write to the same action cache + * on disk, which might not be safe. + */ + private Path getCacheDirectory() { + return getOutputBase().getChild("action_cache"); + } + + /** + * Returns a provider for project file objects. Can be null if no such provider was set by any of + * the modules. + */ + @Nullable + public ProjectFile.Provider getProjectFileProvider() { + return projectFileProvider; + } + + /** + * Hook method called by the BlazeCommandDispatcher prior to the dispatch of + * each command. + * + * @param options The CommonCommandOptions used by every command. + * @throws AbruptExitException if this command is unsuitable to be run as specified + */ + void beforeCommand(String commandName, OptionsParser optionsParser, + CommonCommandOptions options, long execStartTimeNanos) + throws AbruptExitException { + commandStartTime -= options.startupTime; + + eventBus.post(new GotOptionsEvent(startupOptionsProvider, + optionsParser)); + throwPendingException(); + + outputService = null; + BlazeModule outputModule = null; + for (BlazeModule module : blazeModules) { + OutputService moduleService = module.getOutputService(); + if (moduleService != null) { + if (outputService != null) { + throw new IllegalStateException(String.format( + "More than one module (%s and %s) returns an output service", + module.getClass(), outputModule.getClass())); + } + outputService = moduleService; + outputModule = module; + } + } + + skyframeExecutor.setBatchStatter(outputService == null + ? null + : outputService.getBatchStatter()); + + outputFileSystem = determineOutputFileSystem(); + + // Ensure that the working directory will be under the workspace directory. + Path workspace = getWorkspace(); + if (inWorkspace()) { + workingDirectory = workspace.getRelative(options.clientCwd); + } else { + workspace = FileSystemUtils.getWorkingDirectory(directories.getFileSystem()); + workingDirectory = workspace; + } + updateClientEnv(options.clientEnv, options.ignoreClientEnv); + loadingPhaseRunner.updatePatternEvaluator(workingDirectory.relativeTo(workspace)); + + // Fail fast in the case where a Blaze command forgets to install the package path correctly. + skyframeExecutor.setActive(false); + // Let skyframe figure out if it needs to store graph edges for this build. + skyframeExecutor.decideKeepIncrementalState( + startupOptionsProvider.getOptions(BlazeServerStartupOptions.class).batch, + optionsParser.getOptions(BuildView.Options.class)); + + // Conditionally enable profiling + // We need to compensate for launchTimeNanos (measurements taken outside of the jvm). + long startupTimeNanos = options.startupTime * 1000000L; + if (initProfiler(options, this.getCommandId(), execStartTimeNanos - startupTimeNanos)) { + Profiler profiler = Profiler.instance(); + + // Instead of logEvent() we're calling the low level function to pass the timings we took in + // the launcher. We're setting the INIT phase marker so that it follows immediately the LAUNCH + // phase. + profiler.logSimpleTaskDuration(execStartTimeNanos - startupTimeNanos, 0, ProfilerTask.PHASE, + ProfilePhase.LAUNCH.description); + profiler.logSimpleTaskDuration(execStartTimeNanos, 0, ProfilerTask.PHASE, + ProfilePhase.INIT.description); + } + + if (options.memoryProfilePath != null) { + Path memoryProfilePath = getWorkingDirectory().getRelative(options.memoryProfilePath); + try { + MemoryProfiler.instance().start(memoryProfilePath.getOutputStream()); + } catch (IOException e) { + getReporter().handle( + Event.error("Error while creating memory profile file: " + e.getMessage())); + } + } + + eventBus.post(new CommandStartEvent(commandName, commandId, clientEnv, workingDirectory)); + // Initialize exit code to dummy value for afterCommand. + storedExitCode.set(ExitCode.RESERVED.getNumericExitCode()); + } + + /** + * Hook method called by the BlazeCommandDispatcher right before the dispatch + * of each command ends (while its outcome can still be modified). + */ + ExitCode precompleteCommand(ExitCode originalExit) { + eventBus.post(new CommandPrecompleteEvent(originalExit)); + // If Blaze did not suffer an infrastructure failure, check for errors in modules. + ExitCode exitCode = originalExit; + if (!originalExit.isInfrastructureFailure()) { + if (pendingException != null) { + exitCode = pendingException.getExitCode(); + } + } + pendingException = null; + return exitCode; + } + + /** + * Posts the {@link CommandCompleteEvent}, so that listeners can tidy up. Called by {@link + * #afterCommand}, and by BugReport when crashing from an exception in an async thread. + */ + public void notifyCommandComplete(int exitCode) { + if (!storedExitCode.compareAndSet(ExitCode.RESERVED.getNumericExitCode(), exitCode)) { + // This command has already been called, presumably because there is a race between the main + // thread and a worker thread that crashed. Don't try to arbitrate the dispute. If the main + // thread won the race (unlikely, but possible), this may be incorrectly logged as a success. + return; + } + eventBus.post(new CommandCompleteEvent(exitCode)); + } + + /** + * Hook method called by the BlazeCommandDispatcher after the dispatch of each + * command. + */ + @VisibleForTesting + public void afterCommand(int exitCode) { + // Remove any filters that the command might have added to the reporter. + getReporter().setOutputFilter(OutputFilter.OUTPUT_EVERYTHING); + + notifyCommandComplete(exitCode); + + for (BlazeModule module : blazeModules) { + module.afterCommand(); + } + + clearEventBus(); + + try { + Profiler.instance().stop(); + MemoryProfiler.instance().stop(); + } catch (IOException e) { + getReporter().handle(Event.error("Error while writing profile file: " + e.getMessage())); + } + } + + // Make sure we keep a strong reference to this logger, so that the + // configuration isn't lost when the gc kicks in. + private static Logger templateLogger = Logger.getLogger("com.google.devtools.build"); + + /** + * Configures "com.google.devtools.build.*" loggers to the given + * {@code level}. Note: This code relies on static state. + */ + public static void setupLogging(Level level) { + templateLogger.setLevel(level); + templateLogger.info("Log level: " + templateLogger.getLevel()); + } + + /** + * Return an unmodifiable view of the blaze client's environment when it + * invoked the most recent command. Updates from future requests will be + * accessible from this view. + */ + public Map<String, String> getClientEnv() { + return Collections.unmodifiableMap(clientEnv); + } + + @VisibleForTesting + void updateClientEnv(List<Map.Entry<String, String>> clientEnvList, boolean ignoreClientEnv) { + clientEnv.clear(); + + Collection<Map.Entry<String, String>> env = + ignoreClientEnv ? System.getenv().entrySet() : clientEnvList; + for (Map.Entry<String, String> entry : env) { + clientEnv.put(entry.getKey(), entry.getValue()); + } + } + + /** + * Returns the Clock-instance used for the entire build. Before, + * individual classes (such as Profiler) used to specify the type + * of clock (e.g. EpochClock) they wanted to use. This made it + * difficult to get Blaze working on Windows as some of the clocks + * available for Linux aren't (directly) available on Windows. + * Setting the Blaze-wide clock upon construction of BlazeRuntime + * allows injecting whatever Clock instance should be used from + * BlazeMain. + * + * @return The Blaze-wide clock + */ + public Clock getClock() { + return clock; + } + + public OptionsProvider getStartupOptionsProvider() { + return startupOptionsProvider; + } + + /** + * An array of String values useful if Blaze crashes. + * For now, just returns the size of the action cache and the build id. + */ + public String[] getCrashData() { + return new String[]{ + getFileSizeString(CompactPersistentActionCache.cacheFile(getCacheDirectory()), + "action cache"), + commandIdString(), + }; + } + + private String commandIdString() { + UUID uuid = getCommandId(); + return (uuid == null) + ? "no build id" + : uuid + " (build id)"; + } + + /** + * @return the OutputService in use, or null if none. + */ + public OutputService getOutputService() { + return outputService; + } + + private String getFileSizeString(Path path, String type) { + try { + return String.format("%d bytes (%s)", path.getFileSize(), type); + } catch (IOException e) { + return String.format("unknown file size (%s)", type); + } + } + + /** + * Returns the UUID that Blaze uses to identify everything + * logged from the current build command. + */ + public UUID getCommandId() { + return commandId; + } + + void setCommandMap(Map<String, BlazeCommand> commandMap) { + this.commandMap = ImmutableMap.copyOf(commandMap); + } + + public Map<String, BlazeCommand> getCommandMap() { + return commandMap; + } + + /** + * Sets the UUID that Blaze uses to identify everything + * logged from the current build command. + */ + @VisibleForTesting + public void setCommandId(UUID runId) { + commandId = runId; + } + + /** + * Constructs a build configuration key for the given options. + */ + public BuildConfigurationKey getBuildConfigurationKey(BuildOptions buildOptions, + ImmutableSortedSet<String> multiCpu) { + return new BuildConfigurationKey(buildOptions, directories, clientEnv, multiCpu); + } + + /** + * This method only exists for the benefit of InfoCommand, which needs to construct a {@link + * BuildConfigurationCollection} without running a full loading phase. Don't add any more clients; + * instead, we should change info so that it doesn't need the configuration. + */ + public BuildConfigurationCollection getConfigurations(OptionsProvider optionsProvider) + throws InvalidConfigurationException, InterruptedException { + BuildConfigurationKey configurationKey = getBuildConfigurationKey( + createBuildOptions(optionsProvider), ImmutableSortedSet.<String>of()); + boolean keepGoing = optionsProvider.getOptions(BuildView.Options.class).keepGoing; + LoadedPackageProvider loadedPackageProvider = + loadingPhaseRunner.loadForConfigurations(reporter, + ImmutableSet.copyOf(configurationKey.getLabelsToLoadUnconditionally().values()), + keepGoing); + if (loadedPackageProvider == null) { + throw new InvalidConfigurationException("Configuration creation failed"); + } + return skyframeExecutor.createConfigurations(keepGoing, configurationFactory, + configurationKey); + } + + /** + * Initializes the package cache using the given options, and syncs the package cache. Also + * injects a defaults package using the options for the {@link BuildConfiguration}. + * + * @see DefaultsPackage + */ + public void setupPackageCache(PackageCacheOptions packageCacheOptions, + String defaultsPackageContents) throws InterruptedException, AbruptExitException { + if (!skyframeExecutor.hasIncrementalState()) { + clearSkyframeRelevantCaches(); + } + skyframeExecutor.sync(packageCacheOptions, getWorkingDirectory(), + defaultsPackageContents, getCommandId()); + } + + public void shutdown() { + for (BlazeModule module : blazeModules) { + module.blazeShutdown(); + } + } + + /** + * Throws the exception currently queued by a Blaze module. + * + * <p>This should be called as often as is practical so that errors are reported as soon as + * possible. Ideally, we'd not need this, but the event bus swallows exceptions so we raise + * the exception this way. + */ + public void throwPendingException() throws AbruptExitException { + if (pendingException != null) { + AbruptExitException exception = pendingException; + pendingException = null; + throw exception; + } + } + + /** + * Returns the defaults package for the default settings. Should only be called by commands that + * do <i>not</i> process {@link BuildOptions}, since build options can alter the contents of the + * defaults package, which will not be reflected here. + */ + public String getDefaultsPackageContent() { + return ruleClassProvider.getDefaultsPackageContent(); + } + + /** + * Returns the defaults package for the given options taken from an optionsProvider. + */ + public String getDefaultsPackageContent(OptionsClassProvider optionsProvider) { + return ruleClassProvider.getDefaultsPackageContent(optionsProvider); + } + + /** + * Creates a BuildOptions class for the given options taken from an optionsProvider. + */ + public BuildOptions createBuildOptions(OptionsClassProvider optionsProvider) { + return ruleClassProvider.createBuildOptions(optionsProvider); + } + + /** + * An EventBus exception handler that will report the exception to a remote server, if a + * handler is registered. + */ + public static final class RemoteExceptionHandler implements SubscriberExceptionHandler { + @Override + public void handleException(Throwable exception, SubscriberExceptionContext context) { + LoggingUtil.logToRemote(Level.SEVERE, "Failure in EventBus subscriber.", exception); + } + } + + /** + * An EventBus exception handler that will call BugReport.handleCrash exiting + * the current thread. + */ + public static final class BugReportingExceptionHandler implements SubscriberExceptionHandler { + @Override + public void handleException(Throwable exception, SubscriberExceptionContext context) { + BugReport.handleCrash(exception); + } + } + + /** + * Main method for the Blaze server startup. Note: This method logs + * exceptions to remote servers. Do not add this to a unittest. + */ + public static void main(Iterable<Class<? extends BlazeModule>> moduleClasses, String[] args) { + setupUncaughtHandler(args); + List<BlazeModule> modules = createModules(moduleClasses); + if (args.length >= 1 && args[0].equals("--batch")) { + // Run Blaze in batch mode. + System.exit(batchMain(modules, args)); + } + LOG.info("Starting Blaze server with args " + Arrays.toString(args)); + try { + // Run Blaze in server mode. + System.exit(serverMain(modules, OutErr.SYSTEM_OUT_ERR, args)); + } catch (RuntimeException | Error e) { // A definite bug... + BugReport.printBug(OutErr.SYSTEM_OUT_ERR, e); + BugReport.sendBugReport(e, Arrays.asList(args)); + System.exit(ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode()); + throw e; // Shouldn't get here. + } + } + + @VisibleForTesting + public static List<BlazeModule> createModules( + Iterable<Class<? extends BlazeModule>> moduleClasses) { + ImmutableList.Builder<BlazeModule> result = ImmutableList.builder(); + for (Class<? extends BlazeModule> moduleClass : moduleClasses) { + try { + BlazeModule module = moduleClass.newInstance(); + result.add(module); + } catch (Throwable e) { + throw new IllegalStateException("Cannot instantiate module " + moduleClass.getName(), e); + } + } + + return result.build(); + } + + /** + * Generates a string form of a request to be written to the logs, + * filtering the user environment to remove anything that looks private. + * The current filter criteria removes any variable whose name includes + * "auth", "pass", or "cookie". + * + * @param requestStrings + * @return the filtered request to write to the log. + */ + @VisibleForTesting + public static String getRequestLogString(List<String> requestStrings) { + StringBuilder buf = new StringBuilder(); + buf.append('['); + String sep = ""; + for (String s : requestStrings) { + buf.append(sep); + if (s.startsWith("--client_env")) { + int varStart = "--client_env=".length(); + int varEnd = s.indexOf('=', varStart); + String varName = s.substring(varStart, varEnd); + if (suppressFromLog.matcher(varName).matches()) { + buf.append("--client_env="); + buf.append(varName); + buf.append("=__private_value_removed__"); + } else { + buf.append(s); + } + } else { + buf.append(s); + } + sep = ", "; + } + buf.append(']'); + return buf.toString(); + } + + /** + * Command line options split in to two parts: startup options and everything else. + */ + @VisibleForTesting + static class CommandLineOptions { + private final List<String> startupArgs; + private final List<String> otherArgs; + + CommandLineOptions(List<String> startupArgs, List<String> otherArgs) { + this.startupArgs = ImmutableList.copyOf(startupArgs); + this.otherArgs = ImmutableList.copyOf(otherArgs); + } + + public List<String> getStartupArgs() { + return startupArgs; + } + + public List<String> getOtherArgs() { + return otherArgs; + } + } + + /** + * Splits given arguments into two lists - arguments matching options defined in this class + * and everything else, while preserving order in each list. + */ + static CommandLineOptions splitStartupOptions( + Iterable<BlazeModule> modules, String... args) { + List<String> prefixes = new ArrayList<>(); + List<Field> startupFields = Lists.newArrayList(); + for (Class<? extends OptionsBase> defaultOptions + : BlazeCommandUtils.getStartupOptions(modules)) { + startupFields.addAll(ImmutableList.copyOf(defaultOptions.getFields())); + } + + for (Field field : startupFields) { + if (field.isAnnotationPresent(Option.class)) { + prefixes.add("--" + field.getAnnotation(Option.class).name()); + if (field.getType() == boolean.class || field.getType() == TriState.class) { + prefixes.add("--no" + field.getAnnotation(Option.class).name()); + } + } + } + + List<String> startupArgs = new ArrayList<>(); + List<String> otherArgs = Lists.newArrayList(args); + + for (Iterator<String> argi = otherArgs.iterator(); argi.hasNext(); ) { + String arg = argi.next(); + if (!arg.startsWith("--")) { + break; // stop at command - all startup options would be specified before it. + } + for (String prefix : prefixes) { + if (arg.startsWith(prefix)) { + startupArgs.add(arg); + argi.remove(); + break; + } + } + } + return new CommandLineOptions(startupArgs, otherArgs); + } + + private static void captureSigint() { + final Thread mainThread = Thread.currentThread(); + final AtomicInteger numInterrupts = new AtomicInteger(); + + final Runnable interruptWatcher = new Runnable() { + @Override + public void run() { + int count = 0; + // Not an actual infinite loop because it's run in a daemon thread. + while (true) { + count++; + Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS); + LOG.warning("Slow interrupt number " + count + " in batch mode"); + ThreadUtils.warnAboutSlowInterrupt(); + } + } + }; + + new InterruptSignalHandler() { + @Override + public void run() { + LOG.info("User interrupt"); + OutErr.SYSTEM_OUT_ERR.printErrLn("Blaze received an interrupt"); + mainThread.interrupt(); + + int curNumInterrupts = numInterrupts.incrementAndGet(); + if (curNumInterrupts == 1) { + Thread interruptWatcherThread = new Thread(interruptWatcher, "interrupt-watcher"); + interruptWatcherThread.setDaemon(true); + interruptWatcherThread.start(); + } else if (curNumInterrupts == 2) { + LOG.warning("Second --batch interrupt: Reverting to JVM SIGINT handler"); + uninstall(); + } + } + }; + } + + /** + * A main method that runs blaze commands in batch mode. The return value indicates the desired + * exit status of the program. + */ + private static int batchMain(Iterable<BlazeModule> modules, String[] args) { + captureSigint(); + CommandLineOptions commandLineOptions = splitStartupOptions(modules, args); + LOG.info("Running Blaze in batch mode with startup args " + + commandLineOptions.getStartupArgs()); + + String memoryWarning = validateJvmMemorySettings(); + if (memoryWarning != null) { + OutErr.SYSTEM_OUT_ERR.printErrLn(memoryWarning); + } + + BlazeRuntime runtime; + try { + runtime = newRuntime(modules, parseOptions(modules, commandLineOptions.getStartupArgs())); + } catch (OptionsParsingException e) { + OutErr.SYSTEM_OUT_ERR.printErr(e.getMessage()); + return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode(); + } catch (AbruptExitException e) { + OutErr.SYSTEM_OUT_ERR.printErr(e.getMessage()); + return e.getExitCode().getNumericExitCode(); + } + + BlazeCommandDispatcher dispatcher = + new BlazeCommandDispatcher(runtime, getBuiltinCommandList()); + + try { + LOG.info(getRequestLogString(commandLineOptions.getOtherArgs())); + return dispatcher.exec(commandLineOptions.getOtherArgs(), OutErr.SYSTEM_OUT_ERR, + runtime.getClock().currentTimeMillis()); + } catch (BlazeCommandDispatcher.ShutdownBlazeServerException e) { + return e.getExitStatus(); + } finally { + runtime.shutdown(); + dispatcher.shutdown(); + } + } + + /** + * A main method that does not send email. The return value indicates the desired exit status of + * the program. + */ + private static int serverMain(Iterable<BlazeModule> modules, OutErr outErr, String[] args) { + try { + createBlazeRPCServer(modules, Arrays.asList(args)).serve(); + return ExitCode.SUCCESS.getNumericExitCode(); + } catch (OptionsParsingException e) { + outErr.printErr(e.getMessage()); + return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode(); + } catch (IOException e) { + outErr.printErr("I/O Error: " + e.getMessage()); + return ExitCode.BUILD_FAILURE.getNumericExitCode(); + } catch (AbruptExitException e) { + outErr.printErr(e.getMessage()); + return e.getExitCode().getNumericExitCode(); + } + } + + private static FileSystem fileSystemImplementation() { + // The JNI-based UnixFileSystem is faster, but on Windows it is not available. + return OS.getCurrent() == OS.WINDOWS ? new JavaIoFileSystem() : new UnixFileSystem(); + } + + /** + * Creates and returns a new Blaze RPCServer. Call {@link RPCServer#serve()} to start the server. + */ + private static RPCServer createBlazeRPCServer(Iterable<BlazeModule> modules, List<String> args) + throws IOException, OptionsParsingException, AbruptExitException { + OptionsProvider options = parseOptions(modules, args); + BlazeServerStartupOptions startupOptions = options.getOptions(BlazeServerStartupOptions.class); + + final BlazeRuntime runtime = newRuntime(modules, options); + final BlazeCommandDispatcher dispatcher = + new BlazeCommandDispatcher(runtime, getBuiltinCommandList()); + final String memoryWarning = validateJvmMemorySettings(); + + final ServerCommand blazeCommand; + + // Adaptor from RPC mechanism to BlazeCommandDispatcher: + blazeCommand = new ServerCommand() { + private boolean shutdown = false; + + @Override + public int exec(List<String> args, OutErr outErr, long firstContactTime) { + LOG.info(getRequestLogString(args)); + if (memoryWarning != null) { + outErr.printErrLn(memoryWarning); + } + + try { + return dispatcher.exec(args, outErr, firstContactTime); + } catch (BlazeCommandDispatcher.ShutdownBlazeServerException e) { + if (e.getCause() != null) { + StringWriter message = new StringWriter(); + message.write("Shutting down due to exception:\n"); + PrintWriter writer = new PrintWriter(message, true); + e.printStackTrace(writer); + writer.flush(); + LOG.severe(message.toString()); + } + shutdown = true; + runtime.shutdown(); + dispatcher.shutdown(); + return e.getExitStatus(); + } + } + + @Override + public boolean shutdown() { + return shutdown; + } + }; + + RPCServer server = RPCServer.newServerWith(runtime.getClock(), blazeCommand, + runtime.getServerDirectory(), runtime.getWorkspace(), startupOptions.maxIdleSeconds); + return server; + } + + private static Function<String, String> sourceFunctionForMap(final Map<String, String> map) { + return new Function<String, String>() { + @Override + public String apply(String input) { + if (!map.containsKey(input)) { + return "default"; + } + + if (map.get(input).isEmpty()) { + return "command line"; + } + + return map.get(input); + } + }; + } + + /** + * Parses the command line arguments into a {@link OptionsParser} object. + * + * <p>This function needs to parse the --option_sources option manually so that the real option + * parser can set the source for every option correctly. If that cannot be parsed or is missing, + * we just report an unknown source for every startup option. + */ + private static OptionsProvider parseOptions( + Iterable<BlazeModule> modules, List<String> args) throws OptionsParsingException { + Set<Class<? extends OptionsBase>> optionClasses = Sets.newHashSet(); + optionClasses.addAll(BlazeCommandUtils.getStartupOptions(modules)); + // First parse the command line so that we get the option_sources argument + OptionsParser parser = OptionsParser.newOptionsParser(optionClasses); + parser.setAllowResidue(false); + parser.parse(OptionPriority.COMMAND_LINE, null, args); + Function<? super String, String> sourceFunction = + sourceFunctionForMap(parser.getOptions(BlazeServerStartupOptions.class).optionSources); + + // Then parse the command line again, this time with the correct option sources + parser = OptionsParser.newOptionsParser(optionClasses); + parser.setAllowResidue(false); + parser.parseWithSourceFunction(OptionPriority.COMMAND_LINE, sourceFunction, args); + return parser; + } + + /** + * Creates a new blaze runtime, given the install and output base directories. + * + * <p>Note: This method can and should only be called once per startup, as it also creates the + * filesystem object that will be used for the runtime. So it should only ever be called from the + * main method of the Blaze program. + * + * @param options Blaze startup options. + * + * @return a new BlazeRuntime instance initialized with the given filesystem and directories, and + * an error string that, if not null, describes a fatal initialization failure that makes + * this runtime unsuitable for real commands + */ + private static BlazeRuntime newRuntime( + Iterable<BlazeModule> blazeModules, OptionsProvider options) throws AbruptExitException { + for (BlazeModule module : blazeModules) { + module.globalInit(options); + } + + BlazeServerStartupOptions startupOptions = options.getOptions(BlazeServerStartupOptions.class); + PathFragment workspaceDirectory = startupOptions.workspaceDirectory; + PathFragment installBase = startupOptions.installBase; + PathFragment outputBase = startupOptions.outputBase; + + OsUtils.maybeForceJNI(installBase); // Must be before first use of JNI. + + // From the point of view of the Java program --install_base and --output_base + // are mandatory options, despite the comment in their declarations. + if (installBase == null || !installBase.isAbsolute()) { // (includes "" default case) + throw new IllegalArgumentException( + "Bad --install_base option specified: '" + installBase + "'"); + } + if (outputBase != null && !outputBase.isAbsolute()) { // (includes "" default case) + throw new IllegalArgumentException( + "Bad --output_base option specified: '" + outputBase + "'"); + } + + PathFragment outputPathFragment = BlazeDirectories.outputPathFromOutputBase( + outputBase, workspaceDirectory); + FileSystem fs = null; + for (BlazeModule module : blazeModules) { + FileSystem moduleFs = module.getFileSystem(options, outputPathFragment); + if (moduleFs != null) { + Preconditions.checkState(fs == null, "more than one module returns a file system"); + fs = moduleFs; + } + } + + if (fs == null) { + fs = fileSystemImplementation(); + } + Path.setFileSystemForSerialization(fs); + + Path installBasePath = fs.getPath(installBase); + Path outputBasePath = fs.getPath(outputBase); + Path workspaceDirectoryPath = null; + if (!workspaceDirectory.equals(PathFragment.EMPTY_FRAGMENT)) { + workspaceDirectoryPath = fs.getPath(workspaceDirectory); + } + + BlazeDirectories directories = + new BlazeDirectories(installBasePath, outputBasePath, workspaceDirectoryPath); + + Clock clock = BlazeClock.instance(); + + BinTools binTools; + try { + binTools = BinTools.forProduction(directories); + } catch (IOException e) { + throw new AbruptExitException( + "Cannot enumerate embedded binaries: " + e.getMessage(), + ExitCode.LOCAL_ENVIRONMENTAL_ERROR); + } + + BlazeRuntime.Builder runtimeBuilder = new BlazeRuntime.Builder().setDirectories(directories) + .setStartupOptionsProvider(options) + .setBinTools(binTools) + .setClock(clock) + // TODO(bazel-team): Make BugReportingExceptionHandler the default. + // See bug "Make exceptions in EventBus subscribers fatal" + .setEventBusExceptionHandler( + startupOptions.fatalEventBusExceptions || !BlazeVersionInfo.instance().isReleasedBlaze() + ? new BlazeRuntime.BugReportingExceptionHandler() + : new BlazeRuntime.RemoteExceptionHandler()); + + runtimeBuilder.setRunfilesPrefix(new PathFragment(Constants.RUNFILES_PREFIX)); + for (BlazeModule blazeModule : blazeModules) { + runtimeBuilder.addBlazeModule(blazeModule); + } + + BlazeRuntime runtime = runtimeBuilder.build(); + BugReport.setRuntime(runtime); + return runtime; + } + + /** + * Returns null if JVM memory settings are considered safe, and an error string otherwise. + */ + private static String validateJvmMemorySettings() { + boolean is64BitVM = "64".equals(System.getProperty("sun.arch.data.model")); + if (is64BitVM) { + return null; + } + MemoryMXBean mem = ManagementFactory.getMemoryMXBean(); + long heapSize = mem.getHeapMemoryUsage().getMax(); + long nonHeapSize = mem.getNonHeapMemoryUsage().getMax(); + if (heapSize == -1 || nonHeapSize == -1) { + return null; + } + + if (heapSize + nonHeapSize > MAX_BLAZE32_RESERVED_MEMORY) { + return String.format( + "WARNING: JVM reserved %d MB of virtual memory (above threshold of %d MB). " + + "This may result in OOMs at runtime. Use lower values of MaxPermSize " + + "or switch to blaze64.", + (heapSize + nonHeapSize) >> 20, MAX_BLAZE32_RESERVED_MEMORY >> 20); + } else if (heapSize < MIN_BLAZE32_HEAP_SIZE) { + return String.format( + "WARNING: JVM heap size is %d MB. You probably have a custom -Xmx setting in your " + + "local Blaze configuration. This may result in OOMs. Removing overrides of -Xmx " + + "settings is advised.", + heapSize >> 20); + } else { + return null; + } + } + + /** + * Make sure async threads cannot be orphaned. This method makes sure bugs are reported to + * telemetry and the proper exit code is reported. + */ + private static void setupUncaughtHandler(final String[] args) { + Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread thread, Throwable throwable) { + BugReport.handleCrash(throwable, args); + } + }); + } + + + /** + * Returns an immutable list containing new instances of each Blaze command. + */ + @VisibleForTesting + public static List<BlazeCommand> getBuiltinCommandList() { + return ImmutableList.of( + new BuildCommand(), + new CanonicalizeCommand(), + new CleanCommand(), + new HelpCommand(), + new SkylarkCommand(), + new InfoCommand(), + new ProfileCommand(), + new QueryCommand(), + new RunCommand(), + new ShutdownCommand(), + new TestCommand(), + new VersionCommand()); + } + + /** + * A builder for {@link BlazeRuntime} objects. The only required fields are the {@link + * BlazeDirectories}, and the {@link RuleClassProvider} (except for testing). All other fields + * have safe default values. + * + * <p>If a {@link ConfigurationFactory} is set, then the builder ignores the host system flag. + * <p>The default behavior of the BlazeRuntime's EventBus is to exit when a subscriber throws + * an exception. Please plan appropriately. + */ + public static class Builder { + + private PathFragment runfilesPrefix = PathFragment.EMPTY_FRAGMENT; + private BlazeDirectories directories; + private Reporter reporter; + private ConfigurationFactory configurationFactory; + private Clock clock; + private OptionsProvider startupOptionsProvider; + private final List<BlazeModule> blazeModules = Lists.newArrayList(); + private SubscriberExceptionHandler eventBusExceptionHandler = + new RemoteExceptionHandler(); + private BinTools binTools; + private UUID instanceId; + + public BlazeRuntime build() throws AbruptExitException { + Preconditions.checkNotNull(directories); + Preconditions.checkNotNull(startupOptionsProvider); + Reporter reporter = (this.reporter == null) ? new Reporter() : this.reporter; + + Clock clock = (this.clock == null) ? BlazeClock.instance() : this.clock; + UUID instanceId = (this.instanceId == null) ? UUID.randomUUID() : this.instanceId; + + Preconditions.checkNotNull(clock); + Map<String, String> clientEnv = new HashMap<>(); + TimestampGranularityMonitor timestampMonitor = new TimestampGranularityMonitor(clock); + + Preprocessor.Factory.Supplier preprocessorFactorySupplier = null; + SkyframeExecutorFactory skyframeExecutorFactory = null; + for (BlazeModule module : blazeModules) { + module.blazeStartup(startupOptionsProvider, + BlazeVersionInfo.instance(), instanceId, directories, clock); + Preprocessor.Factory.Supplier modulePreprocessorFactorySupplier = + module.getPreprocessorFactorySupplier(); + if (modulePreprocessorFactorySupplier != null) { + Preconditions.checkState(preprocessorFactorySupplier == null, + "more than one module defines a preprocessor factory supplier"); + preprocessorFactorySupplier = modulePreprocessorFactorySupplier; + } + SkyframeExecutorFactory skyFactory = module.getSkyframeExecutorFactory(); + if (skyFactory != null) { + Preconditions.checkState(skyframeExecutorFactory == null, + "At most one skyframe factory supported. But found two: %s and %s", skyFactory, + skyframeExecutorFactory); + skyframeExecutorFactory = skyFactory; + } + } + if (skyframeExecutorFactory == null) { + skyframeExecutorFactory = new SequencedSkyframeExecutorFactory(); + } + if (preprocessorFactorySupplier == null) { + preprocessorFactorySupplier = Preprocessor.Factory.Supplier.NullSupplier.INSTANCE; + } + + ConfiguredRuleClassProvider.Builder ruleClassBuilder = + new ConfiguredRuleClassProvider.Builder(); + for (BlazeModule module : blazeModules) { + module.initializeRuleClasses(ruleClassBuilder); + } + + Map<String, String> platformRegexps = null; + { + ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>(); + for (BlazeModule module : blazeModules) { + builder.putAll(module.getPlatformSetRegexps()); + } + platformRegexps = builder.build(); + if (platformRegexps.isEmpty()) { + platformRegexps = null; // Use the default. + } + } + + Set<Path> immutableDirectories = null; + { + ImmutableSet.Builder<Path> builder = new ImmutableSet.Builder<>(); + for (BlazeModule module : blazeModules) { + builder.addAll(module.getImmutableDirectories()); + } + immutableDirectories = builder.build(); + } + + Iterable<DiffAwareness.Factory> diffAwarenessFactories = null; + { + ImmutableList.Builder<DiffAwareness.Factory> builder = new ImmutableList.Builder<>(); + boolean watchFS = startupOptionsProvider != null + && startupOptionsProvider.getOptions(BlazeServerStartupOptions.class).watchFS; + for (BlazeModule module : blazeModules) { + builder.addAll(module.getDiffAwarenessFactories(watchFS)); + } + diffAwarenessFactories = builder.build(); + } + + // Merge filters from Blaze modules that allow some action inputs to be missing. + Predicate<PathFragment> allowedMissingInputs = null; + for (BlazeModule module : blazeModules) { + Predicate<PathFragment> modulePredicate = module.getAllowedMissingInputs(); + if (modulePredicate != null) { + Preconditions.checkArgument(allowedMissingInputs == null, + "More than one Blaze module allows missing inputs."); + allowedMissingInputs = modulePredicate; + } + } + if (allowedMissingInputs == null) { + allowedMissingInputs = Predicates.alwaysFalse(); + } + + ConfiguredRuleClassProvider ruleClassProvider = ruleClassBuilder.build(); + WorkspaceStatusAction.Factory workspaceStatusActionFactory = null; + for (BlazeModule module : blazeModules) { + WorkspaceStatusAction.Factory candidate = module.getWorkspaceStatusActionFactory(); + if (candidate != null) { + Preconditions.checkState(workspaceStatusActionFactory == null, + "more than one module defines a workspace status action factory"); + workspaceStatusActionFactory = candidate; + } + } + + List<PackageFactory.EnvironmentExtension> extensions = new ArrayList<>(); + for (BlazeModule module : blazeModules) { + extensions.add(module.getPackageEnvironmentExtension()); + } + + // We use an immutable map builder for the nice side effect that it throws if a duplicate key + // is inserted. + ImmutableMap.Builder<SkyFunctionName, SkyFunction> skyFunctions = ImmutableMap.builder(); + for (BlazeModule module : blazeModules) { + skyFunctions.putAll(module.getSkyFunctions(directories)); + } + + ImmutableList.Builder<PrecomputedValue.Injected> precomputedValues = ImmutableList.builder(); + for (BlazeModule module : blazeModules) { + precomputedValues.addAll(module.getPrecomputedSkyframeValues()); + } + + final PackageFactory pkgFactory = + new PackageFactory(ruleClassProvider, platformRegexps, extensions); + SkyframeExecutor skyframeExecutor = skyframeExecutorFactory.create(reporter, pkgFactory, + timestampMonitor, directories, workspaceStatusActionFactory, + ruleClassProvider.getBuildInfoFactories(), immutableDirectories, diffAwarenessFactories, + allowedMissingInputs, preprocessorFactorySupplier, skyFunctions.build(), + precomputedValues.build()); + + if (configurationFactory == null) { + configurationFactory = new ConfigurationFactory( + ruleClassProvider.getConfigurationCollectionFactory(), + ruleClassProvider.getConfigurationFragments()); + } + + ProjectFile.Provider projectFileProvider = null; + for (BlazeModule module : blazeModules) { + ProjectFile.Provider candidate = module.createProjectFileProvider(); + if (candidate != null) { + Preconditions.checkState(projectFileProvider == null, + "more than one module defines a project file provider"); + projectFileProvider = candidate; + } + } + + return new BlazeRuntime(directories, reporter, workspaceStatusActionFactory, skyframeExecutor, + pkgFactory, ruleClassProvider, configurationFactory, + runfilesPrefix == null ? PathFragment.EMPTY_FRAGMENT : runfilesPrefix, + clock, startupOptionsProvider, ImmutableList.copyOf(blazeModules), + clientEnv, timestampMonitor, + eventBusExceptionHandler, binTools, projectFileProvider); + } + + public Builder setRunfilesPrefix(PathFragment prefix) { + this.runfilesPrefix = prefix; + return this; + } + + public Builder setBinTools(BinTools binTools) { + this.binTools = binTools; + return this; + } + + public Builder setDirectories(BlazeDirectories directories) { + this.directories = directories; + return this; + } + + /** + * Creates and sets a new {@link BlazeDirectories} instance with the given + * parameters. + */ + public Builder setDirectories(Path installBase, Path outputBase, + Path workspace) { + this.directories = new BlazeDirectories(installBase, outputBase, workspace); + return this; + } + + public Builder setReporter(Reporter reporter) { + this.reporter = reporter; + return this; + } + + public Builder setConfigurationFactory(ConfigurationFactory configurationFactory) { + this.configurationFactory = configurationFactory; + return this; + } + + public Builder setClock(Clock clock) { + this.clock = clock; + return this; + } + + public Builder setStartupOptionsProvider(OptionsProvider startupOptionsProvider) { + this.startupOptionsProvider = startupOptionsProvider; + return this; + } + + public Builder addBlazeModule(BlazeModule blazeModule) { + blazeModules.add(blazeModule); + return this; + } + + public Builder setInstanceId(UUID id) { + instanceId = id; + return this; + } + + @VisibleForTesting + public Builder setEventBusExceptionHandler( + SubscriberExceptionHandler eventBusExceptionHandler) { + this.eventBusExceptionHandler = eventBusExceptionHandler; + return this; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java new file mode 100644 index 0000000..1f9bcea --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java
@@ -0,0 +1,225 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +import java.util.Map; + +/** + * Options that will be evaluated by the blaze client startup code and passed + * to the blaze server upon startup. + * + * <h4>IMPORTANT</h4> These options and their defaults must be kept in sync with those in the + * source of the launcher. The latter define the actual default values; this class exists only to + * provide the help message, which displays the default values. + * + * The same relationship holds between {@link HostJvmStartupOptions} and the launcher. + */ +public class BlazeServerStartupOptions extends OptionsBase { + /** + * Converter for the <code>option_sources</code> option. Takes a string in the form of + * "option_name1:source1:option_name2:source2:.." and converts it into an option name to + * source map. + */ + public static class OptionSourcesConverter implements Converter<Map<String, String>> { + private String unescape(String input) { + return input.replace("_C", ":").replace("_U", "_"); + } + + @Override + public Map<String, String> convert(String input) { + ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); + if (input.isEmpty()) { + return builder.build(); + } + + String[] elements = input.split(":"); + for (int i = 0; i < (elements.length + 1) / 2; i++) { + String name = elements[i * 2]; + String value = ""; + if (elements.length > i * 2 + 1) { + value = elements[i * 2 + 1]; + } + builder.put(unescape(name), unescape(value)); + } + return builder.build(); + } + + @Override + public String getTypeDescription() { + return "a list of option-source pairs"; + } + } + + /* Passed from the client to the server, specifies the installation + * location. The location should be of the form: + * $OUTPUT_BASE/_blaze_${USER}/install/${MD5_OF_INSTALL_MANIFEST}. + * The server code will only accept a non-empty path; it's the + * responsibility of the client to compute a proper default if + * necessary. + */ + @Option(name = "install_base", + defaultValue = "", // NOTE: purely decorative! See class docstring. + category = "hidden", + converter = OptionsUtils.PathFragmentConverter.class, + help = "This launcher option is intended for use only by tests.") + public PathFragment installBase; + + /* Note: The help string in this option applies to the client code; not + * the server code. The server code will only accept a non-empty path; it's + * the responsibility of the client to compute a proper default if + * necessary. + */ + @Option(name = "output_base", + defaultValue = "null", // NOTE: purely decorative! See class docstring. + category = "server startup", + converter = OptionsUtils.PathFragmentConverter.class, + help = "If set, specifies the output location to which all build output will be written. " + + "Otherwise, the location will be " + + "${OUTPUT_ROOT}/_blaze_${USER}/${MD5_OF_WORKSPACE_ROOT}. Note: If you specify a " + + "different option from one to the next Blaze invocation for this value, you'll likely " + + "start up a new, additional Blaze server. Blaze starts exactly one server per " + + "specified output base. Typically there is one output base per workspace--however, " + + "with this option you may have multiple output bases per workspace and thereby run " + + "multiple builds for the same client on the same machine concurrently. See " + + "'blaze help shutdown' on how to shutdown a Blaze server.") + public PathFragment outputBase; + + /* Note: This option is only used by the C++ client, never by the Java server. + * It is included here to make sure that the option is documented in the help + * output, which is auto-generated by Java code. + */ + @Option(name = "output_user_root", + defaultValue = "null", // NOTE: purely decorative! See class docstring. + category = "server startup", + converter = OptionsUtils.PathFragmentConverter.class, + help = "The user-specific directory beneath which all build outputs are written; " + + "by default, this is a function of $USER, but by specifying a constant, build outputs " + + "can be shared between collaborating users.") + public PathFragment outputUserRoot; + + @Option(name = "workspace_directory", + defaultValue = "", + category = "hidden", + converter = OptionsUtils.PathFragmentConverter.class, + help = "The root of the workspace, that is, the directory that Blaze uses as the root of the " + + "build. This flag is only to be set by the blaze client.") + public PathFragment workspaceDirectory; + + @Option(name = "max_idle_secs", + defaultValue = "" + (3 * 3600), // NOTE: purely decorative! See class docstring. + category = "server startup", + help = "The number of seconds the build server will wait idling " + + "before shutting down. Note: Blaze will ignore this option " + + "unless you are starting a new instance. See also 'blaze help " + + "shutdown'.") + public int maxIdleSeconds; + + @Option(name = "batch", + defaultValue = "false", // NOTE: purely decorative! See class docstring. + category = "server startup", + help = "If set, Blaze will be run in batch mode, instead of " + + "the standard client/server. Doing so may provide " + + "more predictable semantics with respect to signal handling and job control, " + + "Batch mode retains proper queueing semantics within the same output_base. " + + "That is, simultaneous invocations will be processed in order, without overlap. " + + "If a batch mode Blaze is run on a client with a running server, it first kills " + + "the server before processing the command." + + "Blaze will run slower in batch mode, compared to client/server mode. " + + "Among other things, the build file cache is memory-resident, so it is not " + + "preserved between sequential batch invocations. Therefore, using batch mode " + + "often makes more sense in cases where performance is less critical, " + + "such as continuous builds.") + public boolean batch; + + @Option(name = "block_for_lock", + defaultValue = "true", // NOTE: purely decorative! See class docstring. + category = "server startup", + help = "If set, Blaze will exit immediately instead of waiting for other " + + "Blaze commands holding the server lock to complete.") + public boolean noblock_for_lock; + + @Option(name = "io_nice_level", + defaultValue = "-1", // NOTE: purely decorative! + category = "server startup", + help = "Set a level from 0-7 for best-effort IO scheduling. 0 is highest priority, " + + "7 is lowest. The anticipatory scheduler may only honor up to priority 4. " + + "Negative values are ignored.") + public int ioNiceLevel; + + @Option(name = "batch_cpu_scheduling", + defaultValue = "false", // NOTE: purely decorative! + category = "server startup", + help = "Use 'batch' CPU scheduling for Blaze. This policy is useful for workloads that " + + "are non-interactive, but do not want to lower their nice value. " + + "See 'man 2 sched_setscheduler'.") + public boolean batchCpuScheduling; + + @Option(name = "blazerc", + // NOTE: purely decorative! + defaultValue = "In the current directory, then in the user's home directory, the file named " + + ".$(basename $0)rc (i.e. .bazelrc for Bazel or .blazerc for Blaze)", + category = "misc", + help = "The location of the .bazelrc/.blazerc file containing default values of " + + "Blaze command options. Use /dev/null to disable the search for a " + + "blazerc file, e.g. in release builds.") + public String blazerc; + + @Option(name = "master_blazerc", + defaultValue = "true", // NOTE: purely decorative! + category = "misc", + help = "If this option is false, the master blazerc/bazelrc next to the binary " + + "is not read.") + public boolean masterBlazerc; + + @Option(name = "skyframe", + defaultValue = "full", + category = "undocumented", + help = "Unused.") + public String unusedSkyframe; + + @Option(name = "fatal_event_bus_exceptions", + defaultValue = "false", // NOTE: purely decorative! + category = "undocumented", + help = "Whether or not to allow EventBus exceptions to be fatal. Experimental.") + public boolean fatalEventBusExceptions; + + @Option(name = "option_sources", + converter = OptionSourcesConverter.class, + defaultValue = "", + category = "hidden", + help = "") + public Map<String, String> optionSources; + + // TODO(bazel-team): In order to make it easier to have local watchers in open source Bazel, + // turn this into a non-startup option. + @Option(name = "watchfs", + defaultValue = "false", + category = "undocumented", + help = "If true, Blaze tries to use the operating system's file watch service for local " + + "changes instead of scanning every file for a change.") + public boolean watchFS; + + @Option(name = "use_webstatusserver", + defaultValue = "0", + category = "server startup", + help = "Specifies port to run web status server on (0 to disable, which is default).") + public int useWebStatusServer; +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java b/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java new file mode 100644 index 0000000..ee1e429 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java
@@ -0,0 +1,141 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.io.OutErr; + +import java.io.PrintStream; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Utility methods for sending bug reports. + * + * <p> Note, code in this class must be extremely robust. There's nothing + * worse than a crash-handler that itself crashes! + */ +public abstract class BugReport { + + private BugReport() {} + + private static Logger LOG = Logger.getLogger(BugReport.class.getName()); + + private static BlazeVersionInfo versionInfo = BlazeVersionInfo.instance(); + + private static BlazeRuntime runtime = null; + + public static void setRuntime(BlazeRuntime newRuntime) { + Preconditions.checkNotNull(newRuntime); + Preconditions.checkState(runtime == null, "runtime already set: %s, %s", runtime, newRuntime); + runtime = newRuntime; + } + + /** + * Logs the unhandled exception with a special prefix signifying that this was a crash. + * + * @param exception the unhandled exception to display. + * @param args additional values to record in the message. + * @param values Additional string values to clarify the exception. + */ + public static void sendBugReport(Throwable exception, List<String> args, String... values) { + if (!versionInfo.isReleasedBlaze()) { + LOG.info("(Not a released binary; not logged.)"); + return; + } + + logException(exception, filterClientEnv(args), values); + } + + /** + * Print and send a bug report, and exit with the proper Blaze code. + */ + public static void handleCrash(Throwable throwable, String... args) { + BugReport.sendBugReport(throwable, Arrays.asList(args)); + BugReport.printBug(OutErr.SYSTEM_OUT_ERR, throwable); + System.err.println("Blaze crash in async thread:"); + throwable.printStackTrace(); + int exitCode = + (throwable instanceof OutOfMemoryError) ? ExitCode.OOM_ERROR.getNumericExitCode() + : ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode(); + if (runtime != null) { + runtime.notifyCommandComplete(exitCode); + // We don't call runtime#shutDown() here because all it does is shut down the modules, and who + // knows if they can be trusted. + } + System.exit(exitCode); + } + + private static void printThrowableTo(OutErr outErr, Throwable e) { + PrintStream err = new PrintStream(outErr.getErrorStream()); + e.printStackTrace(err); + err.flush(); + LOG.log(Level.SEVERE, "Blaze crashed", e); + } + + /** + * Print user-helpful information about the bug/crash to the output. + * + * @param outErr where to write the output + * @param e the exception thrown + */ + public static void printBug(OutErr outErr, Throwable e) { + if (e instanceof OutOfMemoryError) { + outErr.printErr(e.getMessage() + "\n\n" + + "Blaze ran out of memory and crashed.\n"); + } else { + printThrowableTo(outErr, e); + } + } + + /** + * Filters {@code args} by removing any item that starts with "--client_env", + * then returns this as an immutable list. + * + * <p>The client's environment variables may contain sensitive data, so we filter it out. + */ + private static List<String> filterClientEnv(Iterable<String> args) { + if (args == null) { + return null; + } + + ImmutableList.Builder<String> filteredArgs = ImmutableList.builder(); + for (String arg : args) { + if (arg != null && !arg.startsWith("--client_env")) { + filteredArgs.add(arg); + } + } + return filteredArgs.build(); + } + + // Log the exception. Because this method is only called in a blaze release, + // this will result in a report being sent to a remote logging service. + private static void logException(Throwable exception, List<String> args, String... values) { + // The preamble is used in the crash watcher, so don't change it + // unless you know what you're doing. + String preamble = exception instanceof OutOfMemoryError + ? "Blaze OOMError: " + : "Blaze crashed with args: "; + + LoggingUtil.logToRemote(Level.SEVERE, preamble + Joiner.on(' ').join(args), exception, + values); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BuildPhase.java b/src/main/java/com/google/devtools/build/lib/runtime/BuildPhase.java new file mode 100644 index 0000000..5175a15 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BuildPhase.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +/** + * Represents how far into the build a given target has gone. + * Used primarily for master log status reporting and representation. + */ +public enum BuildPhase { + PARSING("parsing-failed", false), + LOADING("loading-failed", false), + ANALYSIS("analysis-failed", false), + TEST_FILTERING("test-filtered", true), + TARGET_FILTERING("target-filtered", true), + NOT_BUILT("not-built", false), + NOT_ANALYZED("not-analyzed", false), + EXECUTION("build-failed", false), + BLAZE_HALTED("blaze-halted", false), + COMPLETE("built", true); + + private final String msg; + private final boolean success; + + BuildPhase(String msg, boolean success) { + this.msg = msg; + this.success = success; + } + + public String getMessage() { + return msg; + } + + public boolean getSuccess() { + return success; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BuildSummaryStatsModule.java b/src/main/java/com/google/devtools/build/lib/runtime/BuildSummaryStatsModule.java new file mode 100644 index 0000000..8b072c7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BuildSummaryStatsModule.java
@@ -0,0 +1,88 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Joiner; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent; +import com.google.devtools.build.lib.buildtool.buildevent.ExecutionStartingEvent; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.util.BlazeClock; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * Blaze module for the build summary message that reports various stats to the user. + */ +public class BuildSummaryStatsModule extends BlazeModule { + + private static final Logger LOG = Logger.getLogger(BuildSummaryStatsModule.class.getName()); + + private SimpleCriticalPathComputer criticalPathComputer; + private EventBus eventBus; + private Reporter reporter; + + @Override + public void beforeCommand(BlazeRuntime runtime, Command command) { + this.reporter = runtime.getReporter(); + this.eventBus = runtime.getEventBus(); + eventBus.register(this); + } + + @Subscribe + public void executionPhaseStarting(ExecutionStartingEvent event) { + criticalPathComputer = new SimpleCriticalPathComputer(BlazeClock.instance()); + eventBus.register(criticalPathComputer); + } + + @Subscribe + public void buildComplete(BuildCompleteEvent event) { + try { + // We might want to make this conditional on a flag; it can sometimes be a bit of a nuisance. + List<String> items = new ArrayList<>(); + items.add(String.format("Elapsed time: %.3fs", event.getResult().getElapsedSeconds())); + + if (criticalPathComputer != null) { + Profiler.instance().startTask(ProfilerTask.CRITICAL_PATH, "Critical path"); + AggregatedCriticalPath<SimpleCriticalPathComponent> criticalPath = + criticalPathComputer.aggregate(); + items.add(criticalPath.toStringSummary()); + LOG.info(criticalPath.toString()); + LOG.info("Slowest actions:\n " + Joiner.on("\n ") + .join(criticalPathComputer.getSlowestComponents())); + // We reverse the critical path because the profiler expect events ordered by the time + // when the actions were executed while critical path computation is stored in the reverse + // way. + for (SimpleCriticalPathComponent stat : criticalPath.components().reverse()) { + Profiler.instance().logSimpleTaskDuration( + TimeUnit.MILLISECONDS.toNanos(stat.getStartTime()), + TimeUnit.MILLISECONDS.toNanos(stat.getActionWallTime()), + ProfilerTask.CRITICAL_PATH_COMPONENT, stat.getAction()); + } + Profiler.instance().completeTask(ProfilerTask.CRITICAL_PATH); + } + + reporter.handle(Event.info(Joiner.on(", ").join(items))); + } finally { + criticalPathComputer = null; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/Command.java b/src/main/java/com/google/devtools/build/lib/runtime/Command.java new file mode 100644 index 0000000..1797cd3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/Command.java
@@ -0,0 +1,108 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * An annotation that lets blaze commands specify their options and their help. + * The annotations are processed by {@link BlazeCommand}. + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface Command { + /** + * The name of the command, as the user would type it. + */ + String name(); + + /** + * Options processed by the command, indicated by options interfaces. + * These interfaces must contain methods annotated with {@link Option}. + */ + Class<? extends OptionsBase>[] options() default {}; + + /** + * The set of other Blaze commands that this annotation's command "inherits" + * options from. These classes must be annotated with {@link Command}. + */ + Class<? extends BlazeCommand>[] inherits() default {}; + + /** + * A short description, which appears in 'blaze help'. + */ + String shortDescription(); + + /** + * True if the configuration-specific options should be available for this command. + */ + boolean usesConfigurationOptions() default false; + + /** + * True if the command runs a build. + */ + boolean builds() default false; + + /** + * True if the command should not be shown in the output of 'blaze help'. + */ + boolean hidden() default false; + + /** + * Specifies whether this command allows a residue after the parsed options. + * For example, a command might expect a list of targets to build in the + * residue. + */ + boolean allowResidue() default false; + + /** + * Returns true if this command wants to write binary data to stdout. + * Enabling this flag will disable ANSI escape stripping for this command. + */ + boolean binaryStdOut() default false; + + /** + * Returns true if this command wants to write binary data to stderr. + * Enabling this flag will disable ANSI escape stripping for this command. + */ + boolean binaryStdErr() default false; + + /** + * The help message for this command. If the value starts with "resource:", + * the remainder is interpreted as the name of a text file resource (in the + * .jar file that provides the Command implementation class). + */ + String help(); + + /** + * Returns true iff this command may only be run from within a Blaze workspace. Broadly, this + * should be true for any command that interprets the package-path, since it's potentially + * confusing otherwise. + */ + boolean mustRunInWorkspace() default true; + + /** + * Returns true iff this command is allowed to run in the output directory, + * i.e. $OUTPUT_BASE/_blaze_$USER/$MD5/... . No command should be allowed to run here, + * but there are some legacy uses of 'blaze query'. + */ + boolean canRunInOutputDirectory() default false; + +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandCompleteEvent.java new file mode 100644 index 0000000..fb92781 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandCompleteEvent.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +/** + * This event is fired when the Blaze command is complete + * (clean, build, test, etc.). + */ +public class CommandCompleteEvent extends CommandEvent { + + private final int exitCode; + + /** + * @param exitCode the exit code of the blaze command + */ + public CommandCompleteEvent(int exitCode) { + this.exitCode = exitCode; + } + + /** + * @return the exit code of the blaze command + */ + public int getExitCode() { + return exitCode; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandEvent.java new file mode 100644 index 0000000..3e59dce --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandEvent.java
@@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.util.BlazeClock; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.util.Date; + +/** + * Base class for Command events that includes some resource fields. + */ +public abstract class CommandEvent { + + private final long eventTimeInNanos; + private final long eventTimeInEpochTime; + private final long gcTimeInMillis; + + protected CommandEvent() { + eventTimeInNanos = BlazeClock.nanoTime(); + eventTimeInEpochTime = new Date().getTime(); + gcTimeInMillis = collectGcTimeInMillis(); + } + + /** + * Returns time spent in garbage collection since the start of the JVM process. + */ + private static long collectGcTimeInMillis() { + long gcTime = 0; + for (GarbageCollectorMXBean gcBean : ManagementFactory.getGarbageCollectorMXBeans()) { + gcTime += gcBean.getCollectionTime(); + } + return gcTime; + } + + /** + * Get the time-stamp in ns for the event. + */ + public long getEventTimeInNanos() { + return eventTimeInNanos; + } + + /** + * Get the time-stamp as epoch-time for the event. + */ + public long getEventTimeInEpochTime() { + return eventTimeInEpochTime; + } + + /** + * Get the cumulative GC time for the event. + */ + public long getGCTimeInMillis() { + return gcTimeInMillis; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandPrecompleteEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandPrecompleteEvent.java new file mode 100644 index 0000000..9a44086 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandPrecompleteEvent.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.util.ExitCode; + +/** + * This message is fired right before the Blaze command completes, + * and can be used to modify the command's exit code. + */ +public class CommandPrecompleteEvent { + private final ExitCode exitCode; + + /** + * @param exitCode the exit code of the blaze command + */ + public CommandPrecompleteEvent(ExitCode exitCode) { + this.exitCode = exitCode; + } + + /** + * @return the exit code of the blaze command + */ + public ExitCode getExitCode() { + return exitCode; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandStartEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandStartEvent.java new file mode 100644 index 0000000..32834a2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandStartEvent.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.vfs.Path; + +import java.util.Map; +import java.util.UUID; + +/** + * This event is fired when the Blaze command is started (clean, build, test, + * etc.). + */ +public class CommandStartEvent extends CommandEvent { + private final String commandName; + private final UUID commandId; + private final Map<String, String> clientEnv; + private final Path workingDirectory; + + /** + * @param commandName the name of the command + */ + public CommandStartEvent(String commandName, UUID commandId, Map<String, String> clientEnv, + Path workingDirectory) { + this.commandName = commandName; + this.commandId = commandId; + this.clientEnv = clientEnv; + this.workingDirectory = workingDirectory; + } + + public String getCommandName() { + return commandName; + } + + public UUID getCommandId() { + return commandId; + } + + public Map<String, String> getClientEnv() { + return clientEnv; + } + + public Path getWorkingDirectory() { + return workingDirectory; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java new file mode 100644 index 0000000..7054975 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java
@@ -0,0 +1,250 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.List; +import java.util.Map; +import java.util.logging.Level; + +/** + * Options common to all commands. + */ +public class CommonCommandOptions extends OptionsBase { + /** + * A class representing a blazerc option. blazeRc is serial number of the rc + * file this option came from, option is the name of the option and value is + * its value (or null if not specified). + */ + public static class OptionOverride { + final int blazeRc; + final String command; + final String option; + + public OptionOverride(int blazeRc, String command, String option) { + this.blazeRc = blazeRc; + this.command = command; + this.option = option; + } + + @Override + public String toString() { + return String.format("%d:%s=%s", blazeRc, command, option); + } + } + + /** + * Converter for --default_override. The format is: + * --default_override=blazerc:command=option. + */ + public static class OptionOverrideConverter implements Converter<OptionOverride> { + static final String ERROR_MESSAGE = "option overrides must be in form " + + " rcfile:command=option, where rcfile is a nonzero integer"; + + public OptionOverrideConverter() {} + + @Override + public OptionOverride convert(String input) throws OptionsParsingException { + int colonPos = input.indexOf(':'); + int assignmentPos = input.indexOf('='); + + if (colonPos < 0) { + throw new OptionsParsingException(ERROR_MESSAGE); + } + + if (assignmentPos <= colonPos + 1) { + throw new OptionsParsingException(ERROR_MESSAGE); + } + + int blazeRc; + try { + blazeRc = Integer.valueOf(input.substring(0, colonPos)); + } catch (NumberFormatException e) { + throw new OptionsParsingException(ERROR_MESSAGE); + } + + if (blazeRc < 0) { + throw new OptionsParsingException(ERROR_MESSAGE); + } + + String command = input.substring(colonPos + 1, assignmentPos); + String option = input.substring(assignmentPos + 1); + + return new OptionOverride(blazeRc, command, option); + } + + @Override + public String getTypeDescription() { + return "blazerc option override"; + } + } + + + @Option(name = "config", + defaultValue = "", + category = "misc", + allowMultiple = true, + help = "Selects additional config sections from the rc files; for every <command>, it " + + "also pulls in the options from <command>:<config> if such a section exists. " + + "Note that it is currently only possible to provide these options on the " + + "command line, not in the rc files. The config sections and flag combinations " + + "they are equivalent to are located in the tools/*.blazerc config files.") + public List<String> configs; + + @Option(name = "logging", + defaultValue = "3", // Level.INFO + category = "verbosity", + converter = Converters.LogLevelConverter.class, + help = "The logging level.") + public Level verbosity; + + @Option(name = "client_env", + defaultValue = "", + category = "hidden", + converter = Converters.AssignmentConverter.class, + allowMultiple = true, + help = "A system-generated parameter which specifies the client's environment") + public List<Map.Entry<String, String>> clientEnv; + + @Option(name = "ignore_client_env", + defaultValue = "false", + category = "hidden", + help = "If true, ignore the '--client_env' flag, and use the JVM environment instead") + public boolean ignoreClientEnv; + + @Option(name = "client_cwd", + defaultValue = "", + category = "hidden", + converter = OptionsUtils.PathFragmentConverter.class, + help = "A system-generated parameter which specifies the client's working directory") + public PathFragment clientCwd; + + @Option(name = "announce_rc", + defaultValue = "false", + category = "verbosity", + help = "Whether to announce rc options.") + public boolean announceRcOptions; + + /** + * These are the actual default overrides. + * Each value is a pair of (command name, value). + * + * For example: "--default_override=build=--cpu=piii" + */ + @Option(name = "default_override", + defaultValue = "", + allowMultiple = true, + category = "hidden", + converter = OptionOverrideConverter.class, + help = "") + public List<OptionOverride> optionsOverrides; + + /** + * This is the filename that the Blaze client parsed. + */ + @Option(name = "rc_source", + defaultValue = "", + allowMultiple = true, + category = "hidden", + help = "") + public List<String> rcSource; + + @Option(name = "always_profile_slow_operations", + defaultValue = "true", + category = "undocumented", + help = "Whether profiling slow operations is always turned on") + public boolean alwaysProfileSlowOperations; + + @Option(name = "profile", + defaultValue = "null", + category = "misc", + converter = OptionsUtils.PathFragmentConverter.class, + help = "If set, profile Blaze and write data to the specified " + + "file. Use blaze analyze-profile to analyze the profile.") + public PathFragment profilePath; + + @Option(name = "record_full_profiler_data", + defaultValue = "false", + category = "undocumented", + help = "By default, Blaze profiler will record only aggregated data for fast but numerous " + + "events (such as statting the file). If this option is enabled, profiler will record " + + "each event - resulting in more precise profiling data but LARGE performance " + + "hit. Option only has effect if --profile used as well.") + public boolean recordFullProfilerData; + + @Option(name = "memory_profile", + defaultValue = "null", + category = "undocumented", + converter = OptionsUtils.PathFragmentConverter.class, + help = "If set, write memory usage data to the specified " + + "file at phase ends.") + public PathFragment memoryProfilePath; + + @Option(name = "gc_watchdog", + defaultValue = "false", + category = "undocumented", + deprecationWarning = "Ignoring: this option is no longer supported", + help = "Deprecated.") + public boolean gcWatchdog; + + @Option(name = "startup_time", + defaultValue = "0", + category = "hidden", + help = "The time in ms the launcher spends before sending the request to the blaze server.") + public long startupTime; + + @Option(name = "extract_data_time", + defaultValue = "0", + category = "hidden", + help = "The time spend on extracting the new blaze version.") + public long extractDataTime; + + @Option(name = "command_wait_time", + defaultValue = "0", + category = "hidden", + help = "The time in ms a command had to wait on a busy Blaze server process.") + public long waitTime; + + @Option(name = "tool_tag", + defaultValue = "", + allowMultiple = true, + category = "misc", + help = "A tool name to attribute this Blaze invocation to.") + public List<String> toolTag; + + @Option(name = "restart_reason", + defaultValue = "no_restart", + category = "hidden", + help = "The reason for the server restart.") + public String restartReason; + + @Option(name = "binary_path", + defaultValue = "", + category = "hidden", + help = "The absolute path of the blaze binary.") + public String binaryPath; + + @Option(name = "experimental_allow_project_files", + defaultValue = "false", + category = "hidden", + help = "Enable processing of +<file> parameters.") + public boolean allowProjectFiles; +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CriticalPathComputer.java b/src/main/java/com/google/devtools/build/lib/runtime/CriticalPathComputer.java new file mode 100644 index 0000000..2546492 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/CriticalPathComputer.java
@@ -0,0 +1,231 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionCompletionEvent; +import com.google.devtools.build.lib.actions.ActionMetadata; +import com.google.devtools.build.lib.actions.ActionMiddlemanEvent; +import com.google.devtools.build.lib.actions.ActionStartedEvent; +import com.google.devtools.build.lib.actions.Actions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.CachedActionEvent; +import com.google.devtools.build.lib.util.Clock; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.PriorityQueue; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +import javax.annotation.concurrent.ThreadSafe; + +/** + * Computes the critical path in the action graph based on events published to the event bus. + * + * <p>After instantiation, this object needs to be registered on the event bus to work. + */ +@ThreadSafe +public abstract class CriticalPathComputer<C extends AbstractCriticalPathComponent<C>, + A extends AggregatedCriticalPath<C>> { + + /** Number of top actions to record. */ + static final int SLOWEST_COMPONENTS_SIZE = 30; + // outputArtifactToComponent is accessed from multiple event handlers. + protected final ConcurrentMap<Artifact, C> outputArtifactToComponent = Maps.newConcurrentMap(); + + /** Maximum critical path found. */ + private C maxCriticalPath; + private final Clock clock; + + /** + * The list of slowest individual components, ignoring the time to build dependencies. + * + * <p>This data is a useful metric when running non highly incremental builds, where multiple + * tasks could run un parallel and critical path would only record the longest path. + */ + private final PriorityQueue<C> slowestComponents = new PriorityQueue<>(SLOWEST_COMPONENTS_SIZE, + new Comparator<C>() { + @Override + public int compare(C o1, C o2) { + return Long.compare(o1.getActionWallTime(), o2.getActionWallTime()); + } + } + ); + + private final Object lock = new Object(); + + protected CriticalPathComputer(Clock clock) { + this.clock = clock; + maxCriticalPath = null; + } + + /** + * Creates a critical path component for an action. + * @param action the action for the critical path component + * @param startTimeMillis time when the action started to run + */ + protected abstract C createComponent(Action action, long startTimeMillis); + + /** + * Return the critical path stats for the current command execution. + * + * <p>This method allows us to calculate lazily the aggregate statistics of the critical path, + * avoiding the memory and cpu penalty for doing it for all the actions executed. + */ + public abstract A aggregate(); + + /** + * Record an action that has started to run. + * + * @param event information about the started action + */ + @Subscribe + public void actionStarted(ActionStartedEvent event) { + Action action = event.getAction(); + C component = createComponent(action, TimeUnit.NANOSECONDS.toMillis(event.getNanoTimeStart())); + for (Artifact output : action.getOutputs()) { + C old = outputArtifactToComponent.put(output, component); + Preconditions.checkState(old == null, "Duplicate output artifact found. This could happen" + + " if a previous event registered the action %s. Artifact: %s", action, output); + } + } + + /** + * Record a middleman action execution. Even if middleman are almost instant, we record them + * because they depend on other actions and we need them for constructing the critical path. + * + * <p>For some rules with incorrect configuration transitions we might get notified several times + * for the same middleman. This should only happen if the actions are shared. + */ + @Subscribe + public void middlemanAction(ActionMiddlemanEvent event) { + Action action = event.getAction(); + C component = createComponent(action, TimeUnit.NANOSECONDS.toMillis(event.getNanoTimeStart())); + boolean duplicate = false; + for (Artifact output : action.getOutputs()) { + C old = outputArtifactToComponent.putIfAbsent(output, component); + if (old != null) { + if (!Actions.canBeShared(action, old.getAction())) { + throw new IllegalStateException("Duplicate output artifact found for middleman." + + "This could happen if a previous event registered the action.\n" + + "Old action: " + old.getAction() + "\n\n" + + "New action: " + action + "\n\n" + + "Artifact: " + output + "\n"); + } + duplicate = true; + } + } + if (!duplicate) { + finalizeActionStat(action, component); + } + } + + /** + * Record an action that was not executed because it was in the (disk) cache. This is needed so + * that we can calculate correctly the dependencies tree if we have some cached actions in the + * middle of the critical path. + */ + @Subscribe + public void actionCached(CachedActionEvent event) { + Action action = event.getAction(); + C component = createComponent(action, TimeUnit.NANOSECONDS.toMillis(event.getNanoTimeStart())); + for (Artifact output : action.getOutputs()) { + outputArtifactToComponent.put(output, component); + } + finalizeActionStat(action, component); + } + + /** + * Records the elapsed time stats for the action. For each input artifact, it finds the real + * dependent artifacts and records the critical path stats. + */ + @Subscribe + public void actionComplete(ActionCompletionEvent event) { + ActionMetadata action = event.getActionMetadata(); + C component = Preconditions.checkNotNull( + outputArtifactToComponent.get(action.getPrimaryOutput())); + finalizeActionStat(action, component); + } + + /** Maximum critical path component found during the build. */ + protected C getMaxCriticalPath() { + synchronized (lock) { + return maxCriticalPath; + } + } + + /** + * The list of slowest individual components, ignoring the time to build dependencies. + */ + public ImmutableList<C> getSlowestComponents() { + ArrayList<C> list; + synchronized (lock) { + list = new ArrayList<>(slowestComponents); + Collections.sort(list, slowestComponents.comparator()); + } + return ImmutableList.copyOf(list).reverse(); + } + + private void finalizeActionStat(ActionMetadata action, C component) { + component.setFinishTimeMillis(getTime()); + for (Artifact input : action.getInputs()) { + addArtifactDependency(component, input); + } + + synchronized (lock) { + if (isBiggestCriticalPath(component)) { + maxCriticalPath = component; + } + + if (slowestComponents.size() == SLOWEST_COMPONENTS_SIZE) { + // The new component is faster than any of the slow components, avoid insertion. + if (slowestComponents.peek().getActionWallTime() >= component.getActionWallTime()) { + return; + } + // Remove the head element to make space (The fastest component in the queue). + slowestComponents.remove(); + } + slowestComponents.add(component); + } + } + + private long getTime() { + return TimeUnit.NANOSECONDS.toMillis(clock.nanoTime()); + } + + private boolean isBiggestCriticalPath(C newCriticalPath) { + synchronized (lock) { + return maxCriticalPath == null + || maxCriticalPath.getAggregatedWallTime() < newCriticalPath.getAggregatedWallTime(); + } + } + + /** + * If "input" is a generated artifact, link its critical path to the one we're building. + */ + private void addArtifactDependency(C actionStats, Artifact input) { + C depComponent = outputArtifactToComponent.get(input); + if (depComponent != null) { + actionStats.addDepInfo(depComponent); + } + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/EventHandlerPreconditions.java b/src/main/java/com/google/devtools/build/lib/runtime/EventHandlerPreconditions.java new file mode 100644 index 0000000..f4ef8e3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/EventHandlerPreconditions.java
@@ -0,0 +1,143 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.events.ExceptionListener; +import com.google.devtools.build.lib.util.LoggingUtil; + +import java.util.logging.Level; + +/** + * Reports precondition failures from within an event handler. + * Necessary because the EventBus silently ignores exceptions thrown from within a handler. + * This class logs the exceptions and creates some noise when a precondition check fails. + */ +public class EventHandlerPreconditions { + + private final ExceptionListener listener; + + /** + * Creates a new precondition helper which outputs errors to the given reporter. + */ + public EventHandlerPreconditions(ExceptionListener listener) { + this.listener = listener; + } + + /** + * Verifies that the given condition (a check on an argument) is true, + * throwing an IllegalArgumentException if not. + * + * @param condition a condition to check for truth. + * @throws IllegalArgumentException if the condition is false. + */ + @SuppressWarnings("unused") + public void checkArgument(boolean condition) { + checkArgument(condition, null); + } + + /** + * Verifies that the given condition (a check on an argument) is true, + * throwing an IllegalArgumentException with the given message if not. + * + * @param condition a condition to check for truth. + * @param message extra information to output if the condition is false. + * @throws IllegalArgumentException if the condition is false. + */ + public void checkArgument(boolean condition, String message) { + try { + Preconditions.checkArgument(condition, message); + } catch (IllegalArgumentException iae) { + String error = "Event handler argument check failed"; + LoggingUtil.logToRemote(Level.SEVERE, error, iae); + listener.error(null, error, iae); + throw iae; // Still terminate the handler. + } + } + + /** + * Verifies that the given condition (a check against the program's current state) is true, + * throwing an IllegalStateException if not. + * + * @param condition a condition to check for truth. + * @throws IllegalStateException if the condition is false. + */ + public void checkState(boolean condition) { + checkState(condition, null); + } + + /** + * Verifies that the given condition (a check against the program's current state) is true, + * throwing an IllegalStateException with the given message if not. + * + * @param condition a condition to check for truth. + * @param message extra information to output if the condition is false. + * @throws IllegalStateException if the condition is false. + */ + public void checkState(boolean condition, String message) { + try { + Preconditions.checkState(condition, message); + } catch (IllegalStateException ise) { + String error = "Event handler state check failed"; + LoggingUtil.logToRemote(Level.SEVERE, error, ise); + listener.error(null, error, ise); + throw ise; // Still terminate the handler. + } + } + + /** + * Fails with an IllegalStateException when invoked. + */ + public void fail(String message) { + String error = "Event handler failed: " + message; + IllegalStateException ise = new IllegalStateException(message); + LoggingUtil.logToRemote(Level.SEVERE, error, ise); + listener.error(null, error, ise); + throw ise; + } + + /** + * Verifies that the given argument is not null, throwing a NullPointerException if it is null. + * Returns the original argument or throws. + * + * @param object an object to test for null. + * @return the reference which was checked. + * @throws NullPointerException if the object is null. + */ + public <T> T checkNotNull(T object) { + return checkNotNull(object, null); + } + + /** + * Verifies that the given argument is not null, throwing a + * NullPointerException with the given message if it is null. + * Returns the original argument or throws. + * + * @param object an object to test for null. + * @param message extra information to output if the object is null. + * @return the reference which was checked. + * @throws NullPointerException if the object is null. + */ + public <T> T checkNotNull(T object, String message) { + try { + return Preconditions.checkNotNull(object, message); + } catch (NullPointerException npe) { + String error = "Event handler not-null check failed"; + LoggingUtil.logToRemote(Level.SEVERE, error, npe); + listener.error(null, error, npe); + throw npe; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/FancyTerminalEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/FancyTerminalEventHandler.java new file mode 100644 index 0000000..e55ad2f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/FancyTerminalEventHandler.java
@@ -0,0 +1,355 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Splitter; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.io.AnsiTerminal; +import com.google.devtools.build.lib.util.io.OutErr; + +import java.io.IOException; +import java.util.Iterator; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * An event handler for ANSI terminals which uses control characters to + * provide eye-candy, reduce scrolling, and generally improve usability + * for users running directly from the shell. + * + * <p/> + * This event handler differs from a normal terminal because it only adds + * control characters to stderr, not stdout. All blaze status feedback + * is sent to stderr, so adding control characters just to that stream gives + * the benefits described above without modifying the normal output stream. + * For commands like build that don't generate stdout output this doesn't + * matter, but for commands like query and ide_build_info, inserting these + * control characters in stdout invalidated their output. + * + * <p/> + * The underlying streams may be either line-bufferred or unbuffered. + * Normally each event will write out a sequence of output to a single + * stream, and will end with a newline, which ensures a flush. + * But care is required when outputting incomplete lines, or when mixing + * output between the two different streams (stdout and stderr): + * it may be necessary to explicitly flush the output in those cases. + * However, we also don't want to flush too often; that can lead to + * a choppy UI experience. + */ +public class FancyTerminalEventHandler extends BlazeCommandEventHandler { + private static Logger LOG = Logger.getLogger(FancyTerminalEventHandler.class.getName()); + private static final Pattern progressPattern = Pattern.compile( + // Match strings that look like they start with progress info: + // [42%] Compiling base/base.cc + // [1,442 / 23,476] Compiling base/base.cc + "^\\[(?:(?:\\d\\d?\\d?%)|(?:[\\d+,]+ / [\\d,]+))\\] "); + private static final Splitter LINEBREAK_SPLITTER = Splitter.on('\n'); + + private final AnsiTerminal terminal; + + private final boolean useColor; + private final boolean useCursorControls; + private final boolean progressInTermTitle; + public final int terminalWidth; + + private boolean terminalClosed = false; + private boolean previousLineErasable = false; + private int numLinesPreviousErasable = 0; + + public FancyTerminalEventHandler(OutErr outErr, BlazeCommandEventHandler.Options options) { + super(outErr, options); + this.terminal = new AnsiTerminal(outErr.getErrorStream()); + this.terminalWidth = (options.terminalColumns > 0 ? options.terminalColumns : 80); + useColor = options.useColor(); + useCursorControls = options.useCursorControl(); + progressInTermTitle = options.progressInTermTitle; + } + + @Override + public void handle(Event event) { + if (terminalClosed) { + return; + } + if (!eventMask.contains(event.getKind())) { + return; + } + + try { + boolean previousLineErased = false; + if (previousLineErasable) { + previousLineErased = maybeOverwritePreviousMessage(); + } + switch (event.getKind()) { + case PROGRESS: + case START: + { + String message = event.getMessage(); + Pair<String,String> progressPair = matchProgress(message); + if (progressPair != null) { + progress(progressPair.getFirst(), progressPair.getSecond()); + } else { + progress("INFO: ", message); + } + break; + } + case FINISH: + { + String message = event.getMessage(); + Pair<String,String> progressPair = matchProgress(message); + if (progressPair != null) { + String percentage = progressPair.getFirst(); + String rest = progressPair.getSecond(); + progress(percentage, rest + " DONE"); + } else { + progress("INFO: ", message + " DONE"); + } + break; + } + case PASS: + progress("PASS: ", event.getMessage()); + break; + case INFO: + info(event); + break; + case ERROR: + case FAIL: + case TIMEOUT: + // For errors, scroll the message, so it appears above the status + // line, and highlight the word "ERROR" or "FAIL" in boldface red. + errorOrFail(event); + break; + case WARNING: + // For warnings, highlight the word "Warning" in boldface magenta, + // and scroll it. + warning(event); + break; + case SUBCOMMAND: + subcmd(event); + break; + case STDOUT: + if (previousLineErased) { + terminal.flush(); + } + previousLineErasable = false; + super.handle(event); + // We don't need to flush stdout here, because + // super.handle(event) will take care of that. + break; + case STDERR: + putOutput(event); + break; + default: + // Ignore all other event types. + break; + } + } catch (IOException e) { + // The terminal shouldn't have IO errors, unless the shell is killed, which + // should also kill the blaze client. So this isn't something that should + // occur here; it will show up in the client/server interface as a broken + // pipe. + LOG.warning("Terminal was closed during build: " + e); + terminalClosed = true; + } + } + + /** + * Displays a progress message that may be erased by subsequent messages. + * + * @param prefix a short string such as "[99%] " or "INFO: ", which will be highlighted + * @param rest the remainder of the message; may be multiple lines + */ + private void progress(String prefix, String rest) throws IOException { + previousLineErasable = true; + + if (progressInTermTitle) { + int newlinePos = rest.indexOf('\n'); + if (newlinePos == -1) { + terminal.setTitle(prefix + rest); + } else { + terminal.setTitle(prefix + rest.substring(0, newlinePos)); + } + } + + if (useColor) { + terminal.textGreen(); + } + int prefixWidth = prefix.length(); + terminal.writeString(prefix); + terminal.resetTerminal(); + if (showTimestamp) { + String timestamp = timestamp(); + prefixWidth += timestamp.length(); + terminal.writeString(timestamp); + } + int numLines = 0; + Iterator<String> lines = LINEBREAK_SPLITTER.split(rest).iterator(); + String firstLine = lines.next(); + terminal.writeString(firstLine); + // Subtract one, because when the line length is the same as the terminal + // width, the terminal doesn't line-advance, so we don't want to erase + // two lines. + numLines += (prefixWidth + firstLine.length() - 1) / terminalWidth + 1; + crlf(); + while (lines.hasNext()) { + String line = lines.next(); + terminal.writeString(line); + crlf(); + numLines += (line.length() - 1) / terminalWidth + 1; + } + numLinesPreviousErasable = numLines; + } + + /** + * Try to match a message against the "progress message" pattern. If it + * matches, return the progress percentage, and the rest of the message. + * @param message the message to match + * @return a pair containing the progress percentage, and the rest of the + * progress message, or null if the message isn't a progress message. + */ + private Pair<String,String> matchProgress(String message) { + Matcher m = progressPattern.matcher(message); + if (m.find()) { + return Pair.of(message.substring(0, m.end()), message.substring(m.end())); + } else { + return null; + } + } + + /** + * Send the terminal controls that will put the cursor on the beginning + * of the same line if cursor control is on, or the next line if not. + * @returns True if it did any output; if so, caller is responsible for + * flushing the terminal if needed. + */ + private boolean maybeOverwritePreviousMessage() throws IOException { + if (useCursorControls && numLinesPreviousErasable != 0) { + for (int i = 0; i < numLinesPreviousErasable; i++) { + terminal.cr(); + terminal.cursorUp(1); + terminal.clearLine(); + } + return true; + } else { + return false; + } + } + + private void errorOrFail(Event event) throws IOException { + previousLineErasable = false; + if (useColor) { + terminal.textRed(); + terminal.textBold(); + } + terminal.writeString(event.getKind().toString() + ": "); + if (useColor) { + terminal.resetTerminal(); + } + writeTimestampAndLocation(event); + terminal.writeString(event.getMessage()); + terminal.writeString("."); + crlf(); + } + + private void warning(Event warning) throws IOException { + previousLineErasable = false; + if (useColor) { + terminal.textMagenta(); + } + terminal.writeString("WARNING: "); + terminal.resetTerminal(); + writeTimestampAndLocation(warning); + terminal.writeString(warning.getMessage()); + terminal.writeString("."); + crlf(); + } + + private void info(Event event) throws IOException { + previousLineErasable = false; + if (useColor) { + terminal.textGreen(); + } + terminal.writeString(event.getKind().toString() + ": "); + terminal.resetTerminal(); + writeTimestampAndLocation(event); + terminal.writeString(event.getMessage()); + // No period; info messages often end in '...'. + crlf(); + } + + private void subcmd(Event subcmd) throws IOException { + previousLineErasable = false; + if (useColor) { + terminal.textBlue(); + } + terminal.writeString(">>>>> "); + terminal.resetTerminal(); + writeTimestampAndLocation(subcmd); + terminal.writeString(subcmd.getMessage()); + crlf(); + } + + /* Handle STDERR events. */ + private void putOutput(Event event) throws IOException { + previousLineErasable = false; + terminal.writeBytes(event.getMessageBytes()); +/* + * The following code doesn't work because buildtool.TerminalTestNotifier + * writes ANSI-formatted text via this mechanism, one character at a time, + * and if we try to insert additional ANSI sequences in between the characters + * of another ANSI escape sequence, we screw things up. (?) + * TODO(bazel-team): (2009) fix this. TerminalTestNotifier should go via the Reporter + * rather than via an AnsiTerminalWriter. + */ +// terminal.resetTerminal(); +// writeTimestampAndLocation(event); +// if (useColor) { +// terminal.textNormal(); +// } +// terminal.writeBytes(event.getMessageBytes()); +// terminal.resetTerminal(); + } + + /** + * Add a carriage return, shifting to the next line on the terminal, while + * guaranteeing that the terminal control codes don't cause any strange + * effects. Without the CR before the "\n", the "\n" can cause a line-break + * moving text to the next line, where the new message will be generated. + * Emitting a "CR" before means that the actual terminal controls generated + * here are CR+CR+LF; the double-CR resets the terminal line state, which + * prevents the potentially ugly formatting issue. + */ + private void crlf() throws IOException { + terminal.cr(); + terminal.writeString("\n"); + } + + private void writeTimestampAndLocation(Event event) throws IOException { + if (showTimestamp) { + terminal.writeString(timestamp()); + } + if (event.getLocation() != null) { + terminal.writeString(event.getLocation() + ": "); + } + } + + public void resetTerminal() { + try { + terminal.resetTerminal(); + } catch (IOException e) { + LOG.warning("IO Error writing to user terminal: " + e); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/GCStatsRecorder.java b/src/main/java/com/google/devtools/build/lib/runtime/GCStatsRecorder.java new file mode 100644 index 0000000..48e366d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/GCStatsRecorder.java
@@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; + +import java.lang.management.GarbageCollectorMXBean; +import java.util.ArrayList; +import java.util.List; + +/** + * Record GC stats for a build. + */ +public class GCStatsRecorder { + + private final Iterable<GarbageCollectorMXBean> mxBeans; + private final ImmutableMap<String, GCStat> initialData; + + public GCStatsRecorder(Iterable<GarbageCollectorMXBean> mxBeans) { + this.mxBeans = mxBeans; + ImmutableMap.Builder<String, GCStat> initialData = ImmutableMap.builder(); + for (GarbageCollectorMXBean mxBean : mxBeans) { + String name = mxBean.getName(); + initialData.put(name, new GCStat(name, mxBean.getCollectionCount(), + mxBean.getCollectionTime())); + } + this.initialData = initialData.build(); + } + + public Iterable<GCStat> getCurrentGcStats() { + List<GCStat> stats = new ArrayList<>(); + for (GarbageCollectorMXBean mxBean : mxBeans) { + String name = mxBean.getName(); + GCStat initStat = Preconditions.checkNotNull(initialData.get(name)); + stats.add(new GCStat(name, + mxBean.getCollectionCount() - initStat.getNumCollections(), + mxBean.getCollectionTime() - initStat.getTotalTimeInMs())); + } + return stats; + } + + /** Represents the garbage collections statistics for one collector (For example CMS). */ + public static class GCStat { + + private final String name; + private final long numCollections; + private final long totalTimeInMs; + + public GCStat(String name, long numCollections, long totalTimeInMs) { + this.name = name; + this.numCollections = numCollections; + this.totalTimeInMs = totalTimeInMs; + } + + /** Name of the Collector. For example CMS. */ + public String getName() { return name; } + + /** Number of invocations for a build. */ + public long getNumCollections() { return numCollections; } + + /** + * Total time spend in GC for the collector. Note that the time does need to be exclusive (aka a + * stop-the-world GC). + */ + public long getTotalTimeInMs() { return totalTimeInMs; } + + @Override + public String toString() { + return "GC time for '" + name + "' collector: " + numCollections + + " collections using " + totalTimeInMs + "ms"; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/GotOptionsEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/GotOptionsEvent.java new file mode 100644 index 0000000..622d112 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/GotOptionsEvent.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.common.options.OptionsProvider; + +/** + * An event in which the command line options + * are discovered. + */ +public class GotOptionsEvent { + + private final OptionsProvider startupOptions; + private final OptionsProvider options; + + /** + * Construct the options event. + * + * @param startupOptions the parsed startup options + * @param options the parsed options + */ + public GotOptionsEvent(OptionsProvider startupOptions, OptionsProvider options) { + this.startupOptions = startupOptions; + this.options = options; + } + + /** + * @return the parsed startup options + */ + public OptionsProvider getStartupOptions() { + return startupOptions; + } + + /** + * @return the parsed options. + */ + public OptionsProvider getOptions() { + return options; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/HostJvmStartupOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/HostJvmStartupOptions.java new file mode 100644 index 0000000..305c048 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/HostJvmStartupOptions.java
@@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +/** + * Options that will be evaluated by the blaze client startup code only. + * + * The only reason we have this interface is that we'd like to print a nice + * help page for the client startup options. These options do not affect the + * server's behavior in any way. + */ +public class HostJvmStartupOptions extends OptionsBase { + + @Option(name = "host_jvm_args", + defaultValue = "", // NOTE: purely decorative! See BlazeServerStartupOptions. + category = "host jvm startup", + help = "Flags to pass to the JVM executing Blaze. Note: Blaze " + + "will ignore this option unless you are starting a new " + + "instance. See also 'blaze help shutdown'.") + public String hostJvmArgs; + + @Option(name = "host_jvm_profile", + defaultValue = "", // NOTE: purely decorative! See BlazeServerStartupOptions. + category = "host jvm startup", + help = "Run the JVM executing Blaze in the given profiler. " + + "Blaze will search for hardcoded paths based on the " + + "profiler. Note: Blaze will ignore this option unless you " + + "are starting a new instance. See also 'blaze help shutdown'.") + public String hostJvmProfile; + + @Option(name = "host_jvm_debug", + defaultValue = "false", // NOTE: purely decorative! See BlazeServerStartupOptions. + category = "host jvm startup", + help = "Run the JVM executing Blaze so that it listens for a " + + "connection from a JDWP-compliant debugger. Note: Blaze " + + "will ignore this option unless you are starting a new " + + "instance. See also 'blaze help shutdown'.") + public boolean hostJvmDebug; +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/ProjectFile.java b/src/main/java/com/google/devtools/build/lib/runtime/ProjectFile.java new file mode 100644 index 0000000..56747d8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/ProjectFile.java
@@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.List; + +/** + * A file that describes a project - for large source trees that are worked on by multiple + * independent teams, it is useful to have a larger unit than a package which combines a set of + * target patterns and a set of corresponding options. + */ +public interface ProjectFile { + + /** + * A provider for a project file - we generally expect the provider to cache parsed files + * internally and return a cached version if it can ascertain that that is still correct. + * + * <p>Note in particular that packages may be moved between different package path entries, which + * should lead to cache invalidation. + */ + public interface Provider { + /** + * Returns an (optionally cached) project file instance. If there is no such file, or if the + * file cannot be parsed, then it throws an exception. + */ + ProjectFile getProjectFile(List<Path> packagePath, PathFragment path) + throws AbruptExitException; + } + + /** + * A string name of the project file that is reported to the user. It should be in such a format + * that passing it back in on the command line works. + */ + String getName(); + + /** + * A list of strings that are parsed into the options for the command. + * + * @param command An action from the command line, e.g. "build" or "test". + * @throws UnsupportedOperationException if an unknown command is passed. + */ + List<String> getCommandLineFor(String command); +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/RateLimitingEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/RateLimitingEventHandler.java new file mode 100644 index 0000000..5e90f2e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/RateLimitingEventHandler.java
@@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.Clock; + +/** + * An event handler that rate limits events. + */ +public class RateLimitingEventHandler implements EventHandler { + + private final EventHandler outputHandler; + private final double intervalMillis; + private final Clock clock; + private long lastEventMillis = -1; + + /** + * Creates a new Event handler that rate limits the events of type PROGRESS + * to one per event "rateLimitation" seconds. Events that arrive too quickly are dropped; + * all others are are forwarded to the handler "delegateTo". + * + * @param delegateTo The event handler that ultimately handles the events + * @param rateLimitation The minimum number of seconds between events that will be forwarded + * to the delegateTo-handler. + * If less than zero (or NaN), all events will be forwarded. + */ + public static EventHandler create(EventHandler delegateTo, double rateLimitation) { + if (rateLimitation < 0.0 || Double.isNaN(rateLimitation)) { + return delegateTo; + } + return new RateLimitingEventHandler(delegateTo, rateLimitation); + } + + private RateLimitingEventHandler(EventHandler delegateTo, double rateLimitation) { + clock = BlazeClock.instance(); + outputHandler = delegateTo; + this.intervalMillis = rateLimitation * 1000; + } + + @Override + public void handle(Event event) { + switch (event.getKind()) { + case PROGRESS: + case START: + case FINISH: + long currentTime = clock.currentTimeMillis(); + if (lastEventMillis + intervalMillis <= currentTime) { + lastEventMillis = currentTime; + outputHandler.handle(event); + } + break; + default: + outputHandler.handle(event); + break; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComponent.java b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComponent.java new file mode 100644 index 0000000..b8d5d45 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComponent.java
@@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.actions.Action; + +/** + * This class records the critical path for the graph of actions executed. + */ +public class SimpleCriticalPathComponent + extends AbstractCriticalPathComponent<SimpleCriticalPathComponent> { + + public SimpleCriticalPathComponent(Action action, long startTime) { super(action, startTime); } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComputer.java b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComputer.java new file mode 100644 index 0000000..65a9c95 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComputer.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.util.Clock; + +/** + * Computes the critical path during a build. + */ +public class SimpleCriticalPathComputer + extends CriticalPathComputer<SimpleCriticalPathComponent, + AggregatedCriticalPath<SimpleCriticalPathComponent>> { + + public SimpleCriticalPathComputer(Clock clock) { + super(clock); + } + + @Override + public SimpleCriticalPathComponent createComponent(Action action, long startTimeMillis) { + return new SimpleCriticalPathComponent(action, startTimeMillis); + } + + /** + * Return the critical path stats for the current command execution. + * + * <p>This method allow us to calculate lazily the aggregate statistics of the critical path, + * avoiding the memory and cpu penalty for doing it for all the actions executed. + */ + @Override + public AggregatedCriticalPath<SimpleCriticalPathComponent> aggregate() { + ImmutableList.Builder<SimpleCriticalPathComponent> components = ImmutableList.builder(); + SimpleCriticalPathComponent maxCriticalPath = getMaxCriticalPath(); + if (maxCriticalPath == null) { + return new AggregatedCriticalPath<>(0, components.build()); + } + SimpleCriticalPathComponent child = maxCriticalPath; + while (child != null) { + components.add(child); + child = child.getChild(); + } + return new AggregatedCriticalPath<>(maxCriticalPath.getAggregatedWallTime(), + components.build()); + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java b/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java new file mode 100644 index 0000000..0134f55 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java
@@ -0,0 +1,220 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.rules.test.TestLogHelper; +import com.google.devtools.build.lib.rules.test.TestResult; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestSummaryFormat; +import com.google.devtools.build.lib.util.StringUtil; +import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * Prints the test results to a terminal. + */ +public class TerminalTestResultNotifier implements TestResultNotifier { + private static class TestResultStats { + int numberOfTargets; + int passCount; + int failedToBuildCount; + int failedCount; + int failedRemotelyCount; + int failedLocallyCount; + int noStatusCount; + int numberOfExecutedTargets; + boolean wasUnreportedWrongSize; + } + + /** + * Flags specific to test summary reporting. + */ + public static class TestSummaryOptions extends OptionsBase { + @Option(name = "verbose_test_summary", + defaultValue = "true", + category = "verbosity", + help = "If true, print additional information (timing, number of failed runs, etc) in the" + + " test summary.") + public boolean verboseSummary; + + @Option(name = "test_verbose_timeout_warnings", + defaultValue = "false", + category = "verbosity", + help = "If true, print additional warnings when the actual test execution time does not " + + "match the timeout defined by the test (whether implied or explicit).") + public boolean testVerboseTimeoutWarnings; + } + + private final AnsiTerminalPrinter printer; + private final OptionsProvider options; + private final TestSummaryOptions summaryOptions; + + /** + * @param printer The terminal to print to + */ + public TerminalTestResultNotifier(AnsiTerminalPrinter printer, OptionsProvider options) { + this.printer = printer; + this.options = options; + this.summaryOptions = options.getOptions(TestSummaryOptions.class); + } + + /** + * Prints a test result summary that contains only failed tests. + */ + private void printDetailedTestResultSummary(Set<TestSummary> summaries) { + for (TestSummary entry : summaries) { + if (entry.getStatus() != BlazeTestStatus.PASSED) { + TestSummaryPrinter.print(entry, printer, summaryOptions.verboseSummary, true); + } + } + } + + /** + * Prints a full test result summary. + */ + private void printShortSummary(Set<TestSummary> summaries, boolean showPassingTests) { + for (TestSummary entry : summaries) { + if (entry.getStatus() != BlazeTestStatus.PASSED || showPassingTests) { + TestSummaryPrinter.print(entry, printer, summaryOptions.verboseSummary, false); + } + } + } + + /** + * Returns true iff the --check_tests_up_to_date option is enabled. + */ + private boolean optionCheckTestsUpToDate() { + return options.getOptions(ExecutionOptions.class).testCheckUpToDate; + } + + + /** + * Prints a test summary information for all tests to the terminal. + * + * @param summaries Summary of all targets that were ran + * @param numberOfExecutedTargets the number of targets that were actually ran + */ + @Override + public void notify(Set<TestSummary> summaries, int numberOfExecutedTargets) { + TestResultStats stats = new TestResultStats(); + stats.numberOfTargets = summaries.size(); + stats.numberOfExecutedTargets = numberOfExecutedTargets; + + TestOutputFormat testOutput = options.getOptions(ExecutionOptions.class).testOutput; + + for (TestSummary summary : summaries) { + if (summary.isLocalActionCached() + && TestLogHelper.shouldOutputTestLog(testOutput, + TestResult.isBlazeTestStatusPassed(summary.getStatus()))) { + TestSummaryPrinter.printCachedOutput(summary, testOutput, printer); + } + } + + for (TestSummary summary : summaries) { + if (TestResult.isBlazeTestStatusPassed(summary.getStatus())) { + stats.passCount++; + } else if (summary.getStatus() == BlazeTestStatus.FAILED_TO_BUILD) { + stats.failedToBuildCount++; + } else if (summary.ranRemotely()) { + stats.failedRemotelyCount++; + } else { + stats.failedLocallyCount++; + } + + if (summary.getStatus() == BlazeTestStatus.NO_STATUS) { + stats.noStatusCount++; + } + + if (summary.wasUnreportedWrongSize()) { + stats.wasUnreportedWrongSize = true; + } + } + + stats.failedCount = summaries.size() - stats.passCount; + + TestSummaryFormat testSummaryFormat = options.getOptions(ExecutionOptions.class).testSummary; + switch (testSummaryFormat) { + case DETAILED: + printDetailedTestResultSummary(summaries); + break; + + case SHORT: + printShortSummary(summaries, /*printSuccess=*/true); + break; + + case TERSE: + printShortSummary(summaries, /*printSuccess=*/false); + break; + + case NONE: + break; + } + + printStats(stats); + } + + private void addToErrorList(List<String> list, String failureDescription, int count) { + if (count > 0) { + list.add(String.format("%s%d %s %s%s", + AnsiTerminalPrinter.Mode.ERROR, + count, + count == 1 ? "fails" : "fail", + failureDescription, + AnsiTerminalPrinter.Mode.DEFAULT)); + } + } + + private void printStats(TestResultStats stats) { + if (!optionCheckTestsUpToDate()) { + List<String> results = new ArrayList<>(); + if (stats.passCount == 1) { + results.add(stats.passCount + " test passes"); + } else if (stats.passCount > 0) { + results.add(stats.passCount + " tests pass"); + } + addToErrorList(results, "to build", stats.failedToBuildCount); + addToErrorList(results, "locally", stats.failedLocallyCount); + addToErrorList(results, "remotely", stats.failedRemotelyCount); + printer.print(String.format("\nExecuted %d out of %d tests: %s.\n", + stats.numberOfExecutedTargets, + stats.numberOfTargets, + StringUtil.joinEnglishList(results, "and"))); + } else { + int failingUpToDateCount = stats.failedCount - stats.noStatusCount; + printer.print(String.format( + "\nFinished with %d passing and %s%d failing%s tests up to date, %s%d out of date.%s\n", + stats.passCount, + failingUpToDateCount > 0 ? AnsiTerminalPrinter.Mode.ERROR : "", + failingUpToDateCount, + AnsiTerminalPrinter.Mode.DEFAULT, + stats.noStatusCount > 0 ? AnsiTerminalPrinter.Mode.ERROR : "", + stats.noStatusCount, + AnsiTerminalPrinter.Mode.DEFAULT)); + } + + if (stats.wasUnreportedWrongSize) { + printer.print("There were tests whose specified size is too big. Use the " + + "--test_verbose_timeout_warnings command line option to see which " + + "ones these are.\n"); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java b/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java new file mode 100644 index 0000000..ed9120b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java
@@ -0,0 +1,349 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.packages.TestSize; +import com.google.devtools.build.lib.packages.TestTimeout; +import com.google.devtools.build.lib.rules.test.TestProvider; +import com.google.devtools.build.lib.rules.test.TestResult; +import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Prints results to the terminal, showing the results of each test target. + */ +@ThreadCompatible +public class TestResultAnalyzer { + private final Path execRoot; + private final TestSummaryOptions summaryOptions; + private final ExecutionOptions executionOptions; + private final EventBus eventBus; + + /** + * @param summaryOptions Parsed test summarization options. + * @param executionOptions Parsed build/test execution options. + * @param eventBus For reporting failed to build and cached tests. + */ + public TestResultAnalyzer(Path execRoot, + TestSummaryOptions summaryOptions, + ExecutionOptions executionOptions, + EventBus eventBus) { + this.execRoot = execRoot; + this.summaryOptions = summaryOptions; + this.executionOptions = executionOptions; + this.eventBus = eventBus; + } + + /** + * Prints out the results of the given tests, and returns true if they all passed. + * Posts any targets which weren't already completed by the listener to the EventBus. + * Reports all targets on the console via the given notifier. + * Run at the end of the build, run only once. + * + * @param testTargets The list of targets being run + * @param listener An aggregating listener with intermediate results + * @param notifier A console notifier to echo results to. + * @return true if all the tests passed, else false + */ + public boolean differentialAnalyzeAndReport( + Collection<ConfiguredTarget> testTargets, + AggregatingTestListener listener, + TestResultNotifier notifier) { + + Preconditions.checkNotNull(testTargets); + Preconditions.checkNotNull(listener); + Preconditions.checkNotNull(notifier); + + // The natural ordering of the summaries defines their output order. + Set<TestSummary> summaries = Sets.newTreeSet(); + + int totalRun = 0; // Number of targets running at least one non-cached test. + int passCount = 0; + + for (ConfiguredTarget testTarget : testTargets) { + TestSummary summary = aggregateAndReportSummary(testTarget, listener).build(); + summaries.add(summary); + + // Finished aggregating; build the final console output. + if (summary.actionRan()) { + totalRun++; + } + + if (TestResult.isBlazeTestStatusPassed(summary.getStatus())) { + passCount++; + } + } + + Preconditions.checkState(summaries.size() == testTargets.size()); + + notifier.notify(summaries, totalRun); + return passCount == testTargets.size(); + } + + private static BlazeTestStatus aggregateStatus(BlazeTestStatus status, BlazeTestStatus other) { + return status.ordinal() > other.ordinal() ? status : other; + } + + /** + * Helper for differential analysis which aggregates the TestSummary + * for an individual target, reporting runs on the EventBus if necessary. + */ + private TestSummary.Builder aggregateAndReportSummary( + ConfiguredTarget testTarget, + AggregatingTestListener listener) { + + // If already reported by the listener, no work remains for this target. + TestSummary.Builder summary = listener.getCurrentSummary(testTarget); + Label testLabel = testTarget.getLabel(); + Preconditions.checkNotNull(summary, + "%s did not complete test filtering, but has a test result", testLabel); + if (listener.targetReported(testTarget)) { + return summary; + } + + Collection<Artifact> incompleteRuns = listener.getIncompleteRuns(testTarget); + Map<Artifact, TestResult> statusMap = listener.getStatusMap(); + + // We will get back multiple TestResult instances if test had to be retried several + // times before passing. Sharding and multiple runs of the same test without retries + // will be represented by separate artifacts and will produce exactly one TestResult. + for (Artifact testStatus : TestProvider.getTestStatusArtifacts(testTarget)) { + // When a build is interrupted ( eg. a broken target with --nokeep_going ) runResult could + // be null for an unrelated test because we were not able to even try to execute the test. + // In that case, for tests that were previously passing we return null ( == NO STATUS), + // because checking if the cached test target is up-to-date would require running the + // dependency checker transitively. + TestResult runResult = statusMap.get(testStatus); + boolean isIncompleteRun = incompleteRuns.contains(testStatus); + if (runResult == null) { + summary = markIncomplete(summary); + } else if (isIncompleteRun) { + // Only process results which were not recorded by the listener. + + boolean newlyFetched = !statusMap.containsKey(testStatus); + summary = incrementalAnalyze(summary, runResult); + if (newlyFetched) { + eventBus.post(runResult); + } + Preconditions.checkState( + listener.getIncompleteRuns(testTarget).contains(testStatus) == isIncompleteRun, + "TestListener changed in differential analysis. Ensure it isn't still registered."); + } + } + + // The target was not posted by the listener and must be posted now. + eventBus.post(summary.build()); + return summary; + } + + /** + * Incrementally updates a TestSummary given an existing summary + * and a new TestResult. Only call on built targets. + * + * @param summaryBuilder Existing unbuilt test summary associated with a target. + * @param result New test result to aggregate into the summary. + * @return The updated TestSummary. + */ + public TestSummary.Builder incrementalAnalyze(TestSummary.Builder summaryBuilder, + TestResult result) { + // Cache retrieval should have been performed already. + Preconditions.checkNotNull(result); + Preconditions.checkNotNull(summaryBuilder); + TestSummary existingSummary = Preconditions.checkNotNull(summaryBuilder.peek()); + + TransitiveInfoCollection target = existingSummary.getTarget(); + Preconditions.checkNotNull( + target, "The existing TestSummary must be associated with a target"); + + BlazeTestStatus status = existingSummary.getStatus(); + int numCached = existingSummary.numCached(); + int numLocalActionCached = existingSummary.numLocalActionCached(); + + if (!existingSummary.actionRan() && !result.isCached()) { + // At least one run of the test actually ran uncached. + summaryBuilder.setActionRan(true); + + // Coverage data artifact will be identical for all test results - it is provided by the + // TestRunnerAction and all results in this collection associate with the same action. + PathFragment coverageData = result.getCoverageData(); + if (coverageData != null) { + summaryBuilder.addCoverageFiles( + Collections.singletonList(execRoot.getRelative(coverageData))); + } + } + + if (result.isCached() || result.getData().getRemotelyCached()) { + numCached++; + } + if (result.isCached()) { + numLocalActionCached++; + } + + if (!executionOptions.runsPerTestDetectsFlakes) { + status = aggregateStatus(status, result.getData().getStatus()); + } else { + int shardNumber = result.getShardNum(); + int runsPerTestForLabel = target.getProvider(TestProvider.class).getTestParams().getRuns(); + List<BlazeTestStatus> singleShardStatuses = summaryBuilder.addShardStatus( + shardNumber, result.getData().getStatus()); + if (singleShardStatuses.size() == runsPerTestForLabel) { + BlazeTestStatus shardStatus = BlazeTestStatus.NO_STATUS; + int passes = 0; + for (BlazeTestStatus runStatusForShard : singleShardStatuses) { + shardStatus = aggregateStatus(shardStatus, runStatusForShard); + if (TestResult.isBlazeTestStatusPassed(shardStatus)) { + passes++; + } + } + // Under the RunsPerTestDetectsFlakes option, return flaky if 1 <= p < n shards pass. + // If all results pass or fail, aggregate the passing/failing shardStatus. + if (passes == 0 || passes == runsPerTestForLabel) { + status = aggregateStatus(status, shardStatus); + } else { + status = aggregateStatus(status, BlazeTestStatus.FLAKY); + } + } + } + + List<String> filtered = new ArrayList<>(); + warningLoop: for (String warning : result.getData().getWarningList()) { + for (String ignoredPrefix : Constants.IGNORED_TEST_WARNING_PREFIXES) { + if (warning.startsWith(ignoredPrefix)) { + continue warningLoop; + } + } + + filtered.add(warning); + } + + List<Path> passed = new ArrayList<>(); + if (result.getData().hasPassedLog()) { + passed.add(result.getTestAction().getTestLog().getPath().getRelative( + result.getData().getPassedLog())); + } + + List<Path> failed = new ArrayList<>(); + for (String path : result.getData().getFailedLogsList()) { + failed.add(result.getTestAction().getTestLog().getPath().getRelative(path)); + } + + summaryBuilder + .addTestTimes(result.getData().getTestTimesList()) + .addPassedLogs(passed) + .addFailedLogs(failed) + .addWarnings(filtered) + .collectFailedTests(result.getData().getTestCase()) + .setRanRemotely(result.getData().getIsRemoteStrategy()); + + List<String> warnings = new ArrayList<>(); + if (status == BlazeTestStatus.PASSED) { + if (shouldEmitTestSizeWarningInSummary( + summaryOptions.testVerboseTimeoutWarnings, + warnings, result.getData().getTestProcessTimesList(), target)) { + summaryBuilder.setWasUnreportedWrongSize(true); + } + } + + return summaryBuilder + .setStatus(status) + .setNumCached(numCached) + .setNumLocalActionCached(numLocalActionCached) + .addWarnings(warnings); + } + + private TestSummary.Builder markIncomplete(TestSummary.Builder summaryBuilder) { + // TODO(bazel-team): (2010) Make NotRunTestResult support both tests failed to built and + // tests with no status and post it here. + TestSummary summary = summaryBuilder.peek(); + BlazeTestStatus status = summary.getStatus(); + if (status != BlazeTestStatus.NO_STATUS) { + status = aggregateStatus(status, BlazeTestStatus.INCOMPLETE); + } + + return summaryBuilder.setStatus(status); + } + + TestSummary.Builder markUnbuilt(TestSummary.Builder summary, boolean blazeHalted) { + BlazeTestStatus runStatus = blazeHalted ? BlazeTestStatus.BLAZE_HALTED_BEFORE_TESTING + : (executionOptions.testCheckUpToDate + ? BlazeTestStatus.NO_STATUS + : BlazeTestStatus.FAILED_TO_BUILD); + + return summary.setStatus(runStatus); + } + + /** + * Checks whether the specified test timeout could have been smaller and adds + * a warning message if verbose is true. + * + * <p>Returns true if there was a test with the wrong timeout, but if was not + * reported. + */ + private static boolean shouldEmitTestSizeWarningInSummary(boolean verbose, + List<String> warnings, List<Long> testTimes, TransitiveInfoCollection target) { + + TestTimeout specifiedTimeout = + target.getProvider(TestProvider.class).getTestParams().getTimeout(); + long maxTimeOfShard = 0; + + for (Long shardTime : testTimes) { + if (shardTime != null) { + maxTimeOfShard = Math.max(maxTimeOfShard, shardTime); + } + } + + int maxTimeInSeconds = (int) (maxTimeOfShard / 1000); + + if (!specifiedTimeout.isInRangeFuzzy(maxTimeInSeconds)) { + TestTimeout expectedTimeout = TestTimeout.getSuggestedTestTimeout(maxTimeInSeconds); + TestSize expectedSize = TestSize.getTestSize(expectedTimeout); + if (verbose) { + StringBuilder builder = new StringBuilder(String.format( + "Test execution time (%.1fs excluding execution overhead) outside of " + + "range for %s tests. Consider setting timeout=\"%s\"", + maxTimeOfShard / 1000.0, + specifiedTimeout.prettyPrint(), + expectedTimeout)); + if (expectedSize != null) { + builder.append(" or size=\"").append(expectedSize).append("\""); + } + builder.append(". You need not modify the size if you think it is correct."); + warnings.add(builder.toString()); + return false; + } + return true; + } else { + return false; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestResultNotifier.java b/src/main/java/com/google/devtools/build/lib/runtime/TestResultNotifier.java new file mode 100644 index 0000000..d7dbebb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/TestResultNotifier.java
@@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import java.util.Set; + +/** + * Used to notify interested parties of test results. + */ +public interface TestResultNotifier { + + /** + * @param summaries Summary of all targets that were supposed to be tested + * (regardless whether they actually were executed). + * @param numberOfExecutedTargets the number of targets that were actually run. + * Must not exceed summaries.size(). + */ + void notify(Set<TestSummary> summaries, int numberOfExecutedTargets); +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java b/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java new file mode 100644 index 0000000..171f150 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java
@@ -0,0 +1,428 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter.Mode; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; +import com.google.devtools.build.lib.view.test.TestStatus.FailedTestCasesStatus; +import com.google.devtools.build.lib.view.test.TestStatus.TestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Test summary entry. Stores summary information for a single test rule. + * Also used to sort summary output by status. + * + * <p>Invariant: + * All TestSummary mutations should be performed through the Builder. + * No direct TestSummary methods (except the constructor) may mutate the object. + */ +@VisibleForTesting // Ideally package-scoped. +public class TestSummary implements Comparable<TestSummary> { + /** + * Builder class responsible for creating and altering TestSummary objects. + */ + public static class Builder { + private TestSummary summary; + private boolean built; + + private Builder() { + summary = new TestSummary(); + built = false; + } + + private void mergeFrom(TestSummary existingSummary) { + // Yuck, manually fill in fields. + summary.shardRunStatuses = ArrayListMultimap.create(existingSummary.shardRunStatuses); + setTarget(existingSummary.target); + setStatus(existingSummary.status); + addCoverageFiles(existingSummary.coverageFiles); + addPassedLogs(existingSummary.passedLogs); + addFailedLogs(existingSummary.failedLogs); + + if (existingSummary.failedTestCasesStatus != null) { + addFailedTestCases(existingSummary.getFailedTestCases(), + existingSummary.getFailedTestCasesStatus()); + } + + addTestTimes(existingSummary.testTimes); + addWarnings(existingSummary.warnings); + setActionRan(existingSummary.actionRan); + setNumCached(existingSummary.numCached); + setRanRemotely(existingSummary.ranRemotely); + setWasUnreportedWrongSize(existingSummary.wasUnreportedWrongSize); + } + + // Implements copy on write logic, allowing reuse of the same builder. + private void checkMutation() { + // If mutating the builder after an object was built, create another copy. + if (built) { + built = false; + TestSummary lastSummary = summary; + summary = new TestSummary(); + mergeFrom(lastSummary); + } + } + + // This used to return a reference to the value on success. + // However, since it can alter the summary member, inlining it in an + // assignment to a property of summary was unsafe. + private void checkMutation(Object value) { + Preconditions.checkNotNull(value); + checkMutation(); + } + + public Builder setTarget(ConfiguredTarget target) { + checkMutation(target); + summary.target = target; + return this; + } + + public Builder setStatus(BlazeTestStatus status) { + checkMutation(status); + summary.status = status; + return this; + } + + public Builder addCoverageFiles(List<Path> coverageFiles) { + checkMutation(coverageFiles); + summary.coverageFiles.addAll(coverageFiles); + return this; + } + + public Builder addPassedLogs(List<Path> passedLogs) { + checkMutation(passedLogs); + summary.passedLogs.addAll(passedLogs); + return this; + } + + public Builder addFailedLogs(List<Path> failedLogs) { + checkMutation(failedLogs); + summary.failedLogs.addAll(failedLogs); + return this; + } + + public Builder collectFailedTests(TestCase testCase) { + if (testCase == null) { + summary.failedTestCasesStatus = FailedTestCasesStatus.NOT_AVAILABLE; + return this; + } + summary.failedTestCasesStatus = FailedTestCasesStatus.FULL; + return collectFailedTestCases(testCase); + } + + private Builder collectFailedTestCases(TestCase testCase) { + if (testCase.getChildCount() > 0) { + // This is a non-leaf result. Traverse its children, but do not add its + // name to the output list. It should not contain any 'failure' or + // 'error' tags, but we want to be lax here, because the syntax of the + // test.xml file is also lax. + for (TestCase child : testCase.getChildList()) { + collectFailedTestCases(child); + } + } else { + // This is a leaf result. If it passed, don't add it. + if (testCase.getStatus() == TestCase.Status.PASSED) { + return this; + } + + String name = testCase.getName(); + String className = testCase.getClassName(); + if (name == null || className == null) { + // A test case detail is not really interesting if we cannot tell which + // one it is. + this.summary.failedTestCasesStatus = FailedTestCasesStatus.PARTIAL; + return this; + } + + this.summary.failedTestCases.add(testCase); + } + return this; + } + + public Builder addFailedTestCases(List<TestCase> testCases, FailedTestCasesStatus status) { + checkMutation(status); + checkMutation(testCases); + + if (summary.failedTestCasesStatus == null) { + summary.failedTestCasesStatus = status; + } else if (summary.failedTestCasesStatus != status) { + summary.failedTestCasesStatus = FailedTestCasesStatus.PARTIAL; + } + + if (testCases.isEmpty()) { + return this; + } + + // union of summary.failedTestCases, testCases + Map<String, TestCase> allCases = new TreeMap<>(); + if (summary.failedTestCases != null) { + for (TestCase detail : summary.failedTestCases) { + allCases.put(detail.getClassName() + "." + detail.getName(), detail); + } + } + for (TestCase detail : testCases) { + allCases.put(detail.getClassName() + "." + detail.getName(), detail); + } + + summary.failedTestCases = new ArrayList<TestCase>(allCases.values()); + return this; + } + + public Builder addTestTimes(List<Long> testTimes) { + checkMutation(testTimes); + summary.testTimes.addAll(testTimes); + return this; + } + + public Builder addWarnings(List<String> warnings) { + checkMutation(warnings); + summary.warnings.addAll(warnings); + return this; + } + + public Builder setActionRan(boolean actionRan) { + checkMutation(); + summary.actionRan = actionRan; + return this; + } + + public Builder setNumCached(int numCached) { + checkMutation(); + summary.numCached = numCached; + return this; + } + + public Builder setNumLocalActionCached(int numLocalActionCached) { + checkMutation(); + summary.numLocalActionCached = numLocalActionCached; + return this; + } + + public Builder setRanRemotely(boolean ranRemotely) { + checkMutation(); + summary.ranRemotely = ranRemotely; + return this; + } + + public Builder setWasUnreportedWrongSize(boolean wasUnreportedWrongSize) { + checkMutation(); + summary.wasUnreportedWrongSize = wasUnreportedWrongSize; + return this; + } + + /** + * Records a new result for the given shard of the test. + * + * @return an immutable view of the statuses associated with the shard, with the new element. + */ + public List<BlazeTestStatus> addShardStatus(int shardNumber, BlazeTestStatus status) { + Preconditions.checkState(summary.shardRunStatuses.put(shardNumber, status), + "shardRunStatuses must allow duplicate statuses"); + return ImmutableList.copyOf(summary.shardRunStatuses.get(shardNumber)); + } + + /** + * Returns the created TestSummary object. + * Any actions following a build() will create another copy of the same values. + * Since no mutators are provided directly by TestSummary, a copy will not + * be produced if two builds are invoked in a row without calling a setter. + */ + public TestSummary build() { + peek(); + if (!built) { + makeSummaryImmutable(); + // else: it is already immutable. + } + Preconditions.checkState(built, "Built flag was not set"); + return summary; + } + + /** + * Within-package, it is possible to read directly from an + * incompletely-built TestSummary. Used to pass Builders around directly. + */ + TestSummary peek() { + Preconditions.checkNotNull(summary.target, "Target cannot be null"); + Preconditions.checkNotNull(summary.status, "Status cannot be null"); + return summary; + } + + private void makeSummaryImmutable() { + // Once finalized, the list types are immutable. + summary.passedLogs = Collections.unmodifiableList(summary.passedLogs); + summary.failedLogs = Collections.unmodifiableList(summary.failedLogs); + summary.warnings = Collections.unmodifiableList(summary.warnings); + summary.coverageFiles = Collections.unmodifiableList(summary.coverageFiles); + summary.testTimes = Collections.unmodifiableList(summary.testTimes); + + built = true; + } + } + + private ConfiguredTarget target; + private BlazeTestStatus status; + // Currently only populated if --runs_per_test_detects_flakes is enabled. + private Multimap<Integer, BlazeTestStatus> shardRunStatuses = ArrayListMultimap.create(); + private int numCached; + private int numLocalActionCached; + private boolean actionRan; + private boolean ranRemotely; + private boolean wasUnreportedWrongSize; + private List<TestCase> failedTestCases = new ArrayList<>(); + private List<Path> passedLogs = new ArrayList<>(); + private List<Path> failedLogs = new ArrayList<>(); + private List<String> warnings = new ArrayList<>(); + private List<Path> coverageFiles = new ArrayList<>(); + private List<Long> testTimes = new ArrayList<>(); + private FailedTestCasesStatus failedTestCasesStatus = null; + + // Don't allow public instantiation; go through the Builder. + private TestSummary() { + } + + /** + * Creates a new Builder allowing construction of a new TestSummary object. + */ + public static Builder newBuilder() { + return new Builder(); + } + + /** + * Creates a new Builder initialized with a copy of the existing object's values. + */ + public static Builder newBuilderFromExisting(TestSummary existing) { + Builder builder = new Builder(); + builder.mergeFrom(existing); + return builder; + } + + public ConfiguredTarget getTarget() { + return target; + } + + public BlazeTestStatus getStatus() { + return status; + } + + public boolean isCached() { + return numCached > 0; + } + + public boolean isLocalActionCached() { + return numLocalActionCached > 0; + } + + public int numLocalActionCached() { + return numLocalActionCached; + } + + public int numCached() { + return numCached; + } + + private int numUncached() { + return totalRuns() - numCached; + } + + public boolean actionRan() { + return actionRan; + } + + public boolean ranRemotely() { + return ranRemotely; + } + + public boolean wasUnreportedWrongSize() { + return wasUnreportedWrongSize; + } + + public List<TestCase> getFailedTestCases() { + return failedTestCases; + } + + public List<Path> getCoverageFiles() { + return coverageFiles; + } + + public List<Path> getPassedLogs() { + return passedLogs; + } + + public List<Path> getFailedLogs() { + return failedLogs; + } + + public FailedTestCasesStatus getFailedTestCasesStatus() { + return failedTestCasesStatus; + } + + /** + * Returns an immutable view of the warnings associated with this test. + */ + public List<String> getWarnings() { + return Collections.unmodifiableList(warnings); + } + + private static int getSortKey(BlazeTestStatus status) { + return status == BlazeTestStatus.PASSED ? -1 : status.ordinal(); + } + + @Override + public int compareTo(TestSummary that) { + if (this.isCached() != that.isCached()) { + return this.isCached() ? -1 : 1; + } else if ((this.isCached() && that.isCached()) && (this.numUncached() != that.numUncached())) { + return this.numUncached() - that.numUncached(); + } else if (this.status != that.status) { + return getSortKey(this.status) - getSortKey(that.status); + } else { + Artifact thisExecutable = this.target.getProvider(FilesToRunProvider.class).getExecutable(); + Artifact thatExecutable = that.target.getProvider(FilesToRunProvider.class).getExecutable(); + return thisExecutable.getPath().compareTo(thatExecutable.getPath()); + } + } + + public List<Long> getTestTimes() { + // The return result is unmodifiable (UnmodifiableList instance) + return testTimes; + } + + public int getNumCached() { + return numCached; + } + + public int totalRuns() { + return testTimes.size(); + } + + static Mode getStatusMode(BlazeTestStatus status) { + return status == BlazeTestStatus.PASSED + ? Mode.INFO + : (status == BlazeTestStatus.FLAKY ? Mode.WARNING : Mode.ERROR); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestSummaryPrinter.java b/src/main/java/com/google/devtools/build/lib/runtime/TestSummaryPrinter.java new file mode 100644 index 0000000..91c1488 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/TestSummaryPrinter.java
@@ -0,0 +1,255 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Joiner; +import com.google.common.base.Strings; +import com.google.devtools.build.lib.rules.test.TestLogHelper; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter; +import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter.Mode; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; +import com.google.devtools.build.lib.view.test.TestStatus.FailedTestCasesStatus; +import com.google.devtools.build.lib.view.test.TestStatus.TestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; + +/** + * Print test statistics in human readable form. + */ +public class TestSummaryPrinter { + + /** + * Print the cached test log to the given printer. + */ + public static void printCachedOutput(TestSummary summary, + TestOutputFormat testOutput, + AnsiTerminalPrinter printer) { + + String testName = summary.getTarget().getLabel().toString(); + List<String> allLogs = new ArrayList<>(); + for (Path path : summary.getFailedLogs()) { + allLogs.add(path.getPathString()); + } + for (Path path : summary.getPassedLogs()) { + allLogs.add(path.getPathString()); + } + printer.printLn("" + TestSummary.getStatusMode(summary.getStatus()) + summary.getStatus() + ": " + + Mode.DEFAULT + testName + " (see " + Joiner.on(' ').join(allLogs) + ")"); + printer.printLn(Mode.INFO + "INFO: " + Mode.DEFAULT + "From Testing " + testName); + + // Whether to output the target at all was checked by the caller. + // Now check whether to output failing shards. + if (TestLogHelper.shouldOutputTestLog(testOutput, false)) { + for (Path path : summary.getFailedLogs()) { + try { + TestLogHelper.writeTestLog(path, testName, printer.getOutputStream()); + } catch (IOException e) { + printer.printLn("==================== Could not read test output for " + testName); + LoggingUtil.logToRemote(Level.WARNING, "Error while reading test log", e); + } + } + } + + // And passing shards, independently. + if (TestLogHelper.shouldOutputTestLog(testOutput, true)) { + for (Path path : summary.getPassedLogs()) { + try { + TestLogHelper.writeTestLog(path, testName, printer.getOutputStream()); + } catch (Exception e) { + printer.printLn("==================== Could not read test output for " + testName); + LoggingUtil.logToRemote(Level.WARNING, "Error while reading test log", e); + } + } + } + } + + private static String statusString(BlazeTestStatus status) { + return status.toString().replace('_', ' '); + } + + /** + * Prints summary status for a single test. + * @param terminalPrinter The printer to print to + */ + public static void print( + TestSummary summary, + AnsiTerminalPrinter terminalPrinter, + boolean verboseSummary, boolean printFailedTestCases) { + // Skip output for tests that failed to build. + if (summary.getStatus() == BlazeTestStatus.FAILED_TO_BUILD) { + return; + } + String message = getCacheMessage(summary) + statusString(summary.getStatus()); + terminalPrinter.print( + Strings.padEnd(summary.getTarget().getLabel().toString(), 78 - message.length(), ' ') + + " " + TestSummary.getStatusMode(summary.getStatus()) + message + Mode.DEFAULT + + (verboseSummary ? getAttemptSummary(summary) + getTimeSummary(summary) : "") + "\n"); + + if (printFailedTestCases && summary.getStatus() == BlazeTestStatus.FAILED) { + if (summary.getFailedTestCasesStatus() == FailedTestCasesStatus.NOT_AVAILABLE) { + terminalPrinter.print( + Mode.WARNING + " (individual test case information not available) " + + Mode.DEFAULT + "\n"); + } else { + for (TestCase testCase : summary.getFailedTestCases()) { + if (testCase.getStatus() != TestCase.Status.PASSED) { + TestSummaryPrinter.printTestCase(terminalPrinter, testCase); + } + } + + if (summary.getFailedTestCasesStatus() != FailedTestCasesStatus.FULL) { + terminalPrinter.print( + Mode.WARNING + + " (some shards did not report details, list of failed test" + + " cases incomplete)\n" + + Mode.DEFAULT); + } + } + } + + if (printFailedTestCases) { + // In this mode, test output and coverage files would just clutter up + // the output. + return; + } + + for (String warning : summary.getWarnings()) { + terminalPrinter.print(" " + AnsiTerminalPrinter.Mode.WARNING + "WARNING: " + + AnsiTerminalPrinter.Mode.DEFAULT + warning + "\n"); + } + + for (Path path : summary.getFailedLogs()) { + if (path.exists()) { + // Don't use getPrettyPath() here - we want to print the absolute path, + // so that it cut and paste into a different terminal, and we don't + // want to use the blaze-bin etc. symbolic links because they could be changed + // by a subsequent build with different options. + terminalPrinter.print(" " + path.getPathString() + "\n"); + } + } + for (Path path : summary.getCoverageFiles()) { + // Print only non-trivial coverage files. + try { + if (path.exists() && path.getFileSize() > 0) { + terminalPrinter.print(" " + path.getPathString() + "\n"); + } + } catch (IOException e) { + LoggingUtil.logToRemote(Level.WARNING, "Error while reading coverage data file size", + e); + } + } + } + + /** + * Prints the result of an individual test case. It is assumed not to have + * passed, since passed test cases are not reported. + */ + static void printTestCase( + AnsiTerminalPrinter terminalPrinter, TestCase testCase) { + String timeSummary; + if (testCase.hasRunDurationMillis()) { + timeSummary = " (" + + timeInSec(testCase.getRunDurationMillis(), TimeUnit.MILLISECONDS) + + ")"; + } else { + timeSummary = ""; + } + + terminalPrinter.print( + " " + + Mode.ERROR + + Strings.padEnd(testCase.getStatus().toString(), 8, ' ') + + Mode.DEFAULT + + testCase.getClassName() + + "." + + testCase.getName() + + timeSummary + + "\n"); + } + + /** + * Return the given time in seconds, to 1 decimal place, + * i.e. "32.1s". + */ + static String timeInSec(long time, TimeUnit unit) { + double ms = TimeUnit.MILLISECONDS.convert(time, unit); + return String.format("%.1fs", ms / 1000.0); + } + + static String getAttemptSummary(TestSummary summary) { + int attempts = summary.getPassedLogs().size() + summary.getFailedLogs().size(); + if (attempts > 1) { + // Print number of failed runs for failed tests if testing was completed. + if (summary.getStatus() == BlazeTestStatus.FLAKY) { + return ", failed in " + summary.getFailedLogs().size() + " out of " + attempts; + } + if (summary.getStatus() == BlazeTestStatus.TIMEOUT + || summary.getStatus() == BlazeTestStatus.FAILED) { + return " in " + summary.getFailedLogs().size() + " out of " + attempts; + } + } + return ""; + } + + static String getCacheMessage(TestSummary summary) { + if (summary.getNumCached() == 0 || summary.getStatus() == BlazeTestStatus.INCOMPLETE) { + return ""; + } else if (summary.getNumCached() == summary.totalRuns()) { + return "(cached) "; + } else { + return String.format("(%d/%d cached) ", summary.getNumCached(), summary.totalRuns()); + } + } + + static String getTimeSummary(TestSummary summary) { + if (summary.getTestTimes().isEmpty()) { + return ""; + } else if (summary.getTestTimes().size() == 1) { + return " in " + timeInSec(summary.getTestTimes().get(0), TimeUnit.MILLISECONDS); + } else { + // We previously used com.google.math for this, which added about 1 MB of deps to the total + // size. If we re-introduce a dependency on that package, we could revert this change. + long min = summary.getTestTimes().get(0).longValue(), max = min, sum = 0; + double sumOfSquares = 0.0; + for (Long l : summary.getTestTimes()) { + long value = l.longValue(); + min = value < min ? value : min; + max = value > max ? value : max; + sum += value; + sumOfSquares += ((double) value) * (double) value; + } + double mean = ((double) sum) / summary.getTestTimes().size(); + double stddev = Math.sqrt((sumOfSquares - sum * mean) / summary.getTestTimes().size()); + // For sharded tests, we print the max time on the same line as + // the test, and then print more detailed info about the + // distribution of times on the next line. + String maxTime = timeInSec(max, TimeUnit.MILLISECONDS); + return String.format( + " in %s\n Stats over %d runs: max = %s, min = %s, avg = %s, dev = %s", + maxTime, + summary.getTestTimes().size(), + maxTime, + timeInSec(min, TimeUnit.MILLISECONDS), + timeInSec((long) mean, TimeUnit.MILLISECONDS), + timeInSec((long) stddev, TimeUnit.MILLISECONDS)); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java new file mode 100644 index 0000000..d6f61eb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java
@@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.analysis.BuildView; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.List; + +/** + * Handles the 'build' command on the Blaze command line, including targets + * named by arguments passed to Blaze. + */ +@Command(name = "build", + builds = true, + options = { BuildRequestOptions.class, + ExecutionOptions.class, + PackageCacheOptions.class, + BuildView.Options.class, + LoadingPhaseRunner.Options.class, + BuildConfiguration.Options.class, + }, + usesConfigurationOptions = true, + shortDescription = "Builds the specified targets.", + allowResidue = true, + help = "resource:build.txt") +public final class BuildCommand implements BlazeCommand { + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) + throws AbruptExitException { + ProjectFileSupport.handleProjectFiles(runtime, optionsParser, "build"); + } + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + List<String> targets = ProjectFileSupport.getTargets(runtime, options); + + BuildRequest request = BuildRequest.create( + getClass().getAnnotation(Command.class).name(), options, + runtime.getStartupOptionsProvider(), + targets, + runtime.getReporter().getOutErr(), runtime.getCommandId(), runtime.getCommandStartTime()); + return runtime.getBuildTool().processRequest(request, null).getExitCondition(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java new file mode 100644 index 0000000..0bb5a0e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java
@@ -0,0 +1,95 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandUtils; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.Collection; +import java.util.List; + +/** + * The 'blaze canonicalize-flags' command. + */ +@Command(name = "canonicalize-flags", + options = { CanonicalizeCommand.Options.class }, + allowResidue = true, + mustRunInWorkspace = false, + shortDescription = "Canonicalizes a list of Blaze options.", + help = "This command canonicalizes a list of Blaze options. Don't forget to prepend '--' " + + "to end option parsing before the flags to canonicalize.\n" + + "%{options}") +public final class CanonicalizeCommand implements BlazeCommand { + + public static class CommandConverter implements Converter<String> { + + @Override + public String convert(String input) throws OptionsParsingException { + if (input.equals("build")) { + return input; + } else if (input.equals("test")) { + return input; + } + throw new OptionsParsingException("Not a valid command: '" + input + "' (should be " + + getTypeDescription() + ")"); + } + + @Override + public String getTypeDescription() { + return "build or test"; + } + } + + public static class Options extends OptionsBase { + + @Option(name = "for_command", + defaultValue = "build", + category = "misc", + converter = CommandConverter.class, + help = "The command for which the options should be canonicalized.") + public String forCommand; + } + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + BlazeCommand command = runtime.getCommandMap().get( + options.getOptions(Options.class).forCommand); + Collection<Class<? extends OptionsBase>> optionsClasses = + BlazeCommandUtils.getOptions( + command.getClass(), runtime.getBlazeModules(), runtime.getRuleClassProvider()); + try { + List<String> result = OptionsParser.canonicalize(optionsClasses, options.getResidue()); + for (String piece : result) { + runtime.getReporter().getOutErr().printOutLn(piece); + } + } catch (OptionsParsingException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.COMMAND_LINE_ERROR; + } + return ExitCode.SUCCESS; + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java new file mode 100644 index 0000000..3fd300e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java
@@ -0,0 +1,185 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.util.CommandBuilder; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.ProcessUtils; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.IOException; +import java.util.logging.Logger; + +/** + * Implements 'blaze clean'. + */ +@Command(name = "clean", + builds = true, // Does not, but people expect build options to be there + options = { CleanCommand.Options.class }, + help = "resource:clean.txt", + shortDescription = "Removes output files and optionally stops the server.", + // TODO(bazel-team): Remove this - we inherit a huge number of unused options. + inherits = { BuildCommand.class }) +public final class CleanCommand implements BlazeCommand { + + /** + * An interface for special options for the clean command. + */ + public static class Options extends OptionsBase { + @Option(name = "clean_style", + defaultValue = "", + category = "clean", + help = "Can be either 'expunge' or 'expunge_async'.") + public String cleanStyle; + + @Option(name = "expunge", + defaultValue = "false", + category = "clean", + expansion = "--clean_style=expunge", + help = "If specified, clean will remove the entire working tree for this Blaze " + + "instance, which includes all Blaze-created temporary and build output " + + "files, and it will stop the Blaze server if it is running.") + public boolean expunge; + + @Option(name = "expunge_async", + defaultValue = "false", + category = "clean", + expansion = "--clean_style=expunge_async", + help = "If specified, clean will asynchronously remove the entire working tree for " + + "this Blaze instance, which includes all Blaze-created temporary and build " + + "output files, and it will stop the Blaze server if it is running. When this " + + "command completes, it will be safe to execute new commands in the same client, " + + "even though the deletion may continue in the background.") + public boolean expunge_async; + } + + private static Logger LOG = Logger.getLogger(CleanCommand.class.getName()); + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) + throws ShutdownBlazeServerException { + Options cleanOptions = options.getOptions(Options.class); + cleanOptions.expunge_async = cleanOptions.cleanStyle.equals("expunge_async"); + cleanOptions.expunge = cleanOptions.cleanStyle.equals("expunge"); + + if (cleanOptions.expunge == false && cleanOptions.expunge_async == false && + !cleanOptions.cleanStyle.isEmpty()) { + runtime.getReporter().handle(Event.error( + null, "Invalid clean_style value '" + cleanOptions.cleanStyle + "'")); + return ExitCode.COMMAND_LINE_ERROR; + } + + String cleanBanner = cleanOptions.expunge_async ? + "Starting clean." : + "Starting clean (this may take a while). " + + "Consider using --expunge_async if the clean takes more than several minutes."; + + runtime.getReporter().handle(Event.info(null/*location*/, cleanBanner)); + try { + String symlinkPrefix = + options.getOptions(BuildRequest.BuildRequestOptions.class).symlinkPrefix; + actuallyClean(runtime, runtime.getOutputBase(), cleanOptions, symlinkPrefix); + return ExitCode.SUCCESS; + } catch (IOException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } catch (CommandException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.RUN_FAILURE; + } catch (ExecException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.RUN_FAILURE; + } catch (InterruptedException e) { + runtime.getReporter().handle(Event.error("clean interrupted")); + return ExitCode.INTERRUPTED; + } + } + + private void actuallyClean(BlazeRuntime runtime, + Path outputBase, Options cleanOptions, String symlinkPrefix) throws IOException, + ShutdownBlazeServerException, CommandException, ExecException, InterruptedException { + if (runtime.getOutputService() != null) { + runtime.getOutputService().clean(); + } + if (cleanOptions.expunge) { + LOG.info("Expunging..."); + // Delete the big subdirectories with the important content first--this + // will take the most time. Then quickly delete the little locks, logs + // and links right before we exit. Once the lock file is gone there will + // be a small possibility of a server race if a client is waiting, but + // all significant files will be gone by then. + FileSystemUtils.deleteTreesBelow(outputBase); + FileSystemUtils.deleteTree(outputBase); + } else if (cleanOptions.expunge_async) { + LOG.info("Expunging asynchronously..."); + String tempBaseName = outputBase.getBaseName() + "_tmp_" + ProcessUtils.getpid(); + + // Keeping tempOutputBase in the same directory ensures it remains in the + // same file system, and therefore the mv will be atomic and fast. + Path tempOutputBase = outputBase.getParentDirectory().getChild(tempBaseName); + outputBase.renameTo(tempOutputBase); + runtime.getReporter().handle(Event.info( + null, "Output base moved to " + tempOutputBase + " for deletion")); + + // Daemonize the shell and use the double-fork idiom to ensure that the shell + // exits even while the "rm -rf" command continues. + String command = String.format("exec >&- 2>&- <&- && (/usr/bin/setsid /bin/rm -rf %s &)&", + ShellEscaper.escapeString(tempOutputBase.getPathString())); + + LOG.info("Executing shell commmand " + ShellEscaper.escapeString(command)); + + // Doesn't throw iff command exited and was successful. + new CommandBuilder().addArg(command).useShell(true) + .setWorkingDir(tempOutputBase.getParentDirectory()) + .build().execute(); + } else { + LOG.info("Output cleaning..."); + runtime.clearCaches(); + for (String directory : new String[] { + BlazeDirectories.RELATIVE_OUTPUT_PATH, runtime.getWorkspaceName() }) { + Path child = outputBase.getChild(directory); + if (child.exists()) { + LOG.finest("Cleaning " + child); + FileSystemUtils.deleteTreesBelow(child); + } + } + } + // remove convenience links + OutputDirectoryLinksUtils.removeOutputDirectoryLinks( + runtime.getWorkspaceName(), runtime.getWorkspace(), runtime.getReporter(), symlinkPrefix); + // shutdown on expunge cleans + if (cleanOptions.expunge || cleanOptions.expunge_async) { + throw new ShutdownBlazeServerException(0); + } + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java new file mode 100644 index 0000000..5267e71 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java
@@ -0,0 +1,248 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.docgen.BlazeRuleHelpPrinter; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandUtils; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * The 'blaze help' command, which prints all available commands as well as + * specific help pages. + */ +@Command(name = "help", + options = { HelpCommand.Options.class }, + allowResidue = true, + mustRunInWorkspace = false, + shortDescription = "Prints help for commands, or the index.", + help = "resource:help.txt") +public final class HelpCommand implements BlazeCommand { + public static class Options extends OptionsBase { + + @Option(name = "help_verbosity", + category = "help", + defaultValue = "medium", + converter = Converters.HelpVerbosityConverter.class, + help = "Select the verbosity of the help command.") + public OptionsParser.HelpVerbosity helpVerbosity; + + @Option(name = "long", + abbrev = 'l', + defaultValue = "null", + category = "help", + expansion = {"--help_verbosity", "long"}, + help = "Show full description of each option, instead of just its name.") + public Void showLongFormOptions; + + @Option(name = "short", + defaultValue = "null", + category = "help", + expansion = {"--help_verbosity", "short"}, + help = "Show only the names of the options, not their types or meanings.") + public Void showShortFormOptions; + } + + /** + * Returns a map that maps option categories to descriptive help strings for categories that + * are not part of the Bazel core. + */ + private ImmutableMap<String, String> getOptionCategories(BlazeRuntime runtime) { + ImmutableMap.Builder<String, String> optionCategoriesBuilder = ImmutableMap.builder(); + optionCategoriesBuilder + .put("checking", + "Checking options, which control Blaze's error checking and/or warnings") + .put("coverage", + "Options that affect how Blaze generates code coverage information") + .put("experimental", + "Experimental options, which control experimental (and potentially risky) features") + .put("flags", + "Flags options, for passing options to other tools") + .put("help", + "Help options") + .put("host jvm startup", + "Options that affect the startup of the Blaze server's JVM") + .put("misc", + "Miscellaneous options") + .put("package loading", + "Options that specify how to locate packages") + .put("query", + "Options affecting the 'blaze query' dependency query command") + .put("run", + "Options specific to 'blaze run'") + .put("semantics", + "Semantics options, which affect the build commands and/or output file contents") + .put("server startup", + "Startup options, which affect the startup of the Blaze server") + .put("strategy", + "Strategy options, which affect how Blaze will execute the build") + .put("testing", + "Options that affect how Blaze runs tests") + .put("verbosity", + "Verbosity options, which control what Blaze prints") + .put("version", + "Version options, for selecting which version of other tools will be used") + .put("what", + "Output selection options, for determining what to build/test"); + for (BlazeModule module : runtime.getBlazeModules()) { + optionCategoriesBuilder.putAll(module.getOptionCategories()); + } + return optionCategoriesBuilder.build(); + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + OutErr outErr = runtime.getReporter().getOutErr(); + Options helpOptions = options.getOptions(Options.class); + if (options.getResidue().isEmpty()) { + emitBlazeVersionInfo(outErr); + emitGenericHelp(runtime, outErr); + return ExitCode.SUCCESS; + } + if (options.getResidue().size() != 1) { + runtime.getReporter().handle(Event.error("You must specify exactly one command")); + return ExitCode.COMMAND_LINE_ERROR; + } + String helpSubject = options.getResidue().get(0); + if (helpSubject.equals("startup_options")) { + emitBlazeVersionInfo(outErr); + emitStartupOptions(outErr, helpOptions.helpVerbosity, runtime, getOptionCategories(runtime)); + return ExitCode.SUCCESS; + } else if (helpSubject.equals("target-syntax")) { + emitBlazeVersionInfo(outErr); + emitTargetSyntaxHelp(outErr, getOptionCategories(runtime)); + return ExitCode.SUCCESS; + } else if (helpSubject.equals("info-keys")) { + emitInfoKeysHelp(runtime, outErr); + return ExitCode.SUCCESS; + } + + BlazeCommand command = runtime.getCommandMap().get(helpSubject); + if (command == null) { + ConfiguredRuleClassProvider provider = runtime.getRuleClassProvider(); + RuleClass ruleClass = provider.getRuleClassMap().get(helpSubject); + if (ruleClass != null && ruleClass.isDocumented()) { + // There is a rule with a corresponding name + outErr.printOut(BlazeRuleHelpPrinter.getRuleDoc(helpSubject, provider)); + return ExitCode.SUCCESS; + } else { + runtime.getReporter().handle(Event.error( + null, "'" + helpSubject + "' is neither a command nor a build rule")); + return ExitCode.COMMAND_LINE_ERROR; + } + } + emitBlazeVersionInfo(outErr); + outErr.printOut(BlazeCommandUtils.getUsage( + command.getClass(), + getOptionCategories(runtime), + helpOptions.helpVerbosity, + runtime.getBlazeModules(), + runtime.getRuleClassProvider())); + return ExitCode.SUCCESS; + } + + private void emitBlazeVersionInfo(OutErr outErr) { + String releaseInfo = BlazeVersionInfo.instance().getReleaseName(); + String line = "[Blaze " + releaseInfo + "]"; + outErr.printOut(String.format("%80s\n", line)); + } + + @SuppressWarnings("unchecked") // varargs generic array creation + private void emitStartupOptions(OutErr outErr, OptionsParser.HelpVerbosity helpVerbosity, + BlazeRuntime runtime, ImmutableMap<String, String> optionCategories) { + outErr.printOut( + BlazeCommandUtils.expandHelpTopic("startup_options", + "resource:startup_options.txt", + getClass(), + BlazeCommandUtils.getStartupOptions(runtime.getBlazeModules()), + optionCategories, + helpVerbosity)); + } + + private void emitTargetSyntaxHelp(OutErr outErr, ImmutableMap<String, String> optionCategories) { + outErr.printOut(BlazeCommandUtils.expandHelpTopic("target-syntax", + "resource:target-syntax.txt", + getClass(), + ImmutableList.<Class<? extends OptionsBase>>of(), + optionCategories, + OptionsParser.HelpVerbosity.MEDIUM)); + } + + private void emitInfoKeysHelp(BlazeRuntime runtime, OutErr outErr) { + for (InfoKey key : InfoKey.values()) { + outErr.printOut(String.format("%-23s %s\n", key.getName(), key.getDescription())); + } + + for (BlazeModule.InfoItem item : InfoCommand.getInfoItemMap(runtime, + OptionsParser.newOptionsParser( + ImmutableList.<Class<? extends OptionsBase>>of())).values()) { + outErr.printOut(String.format("%-23s %s\n", item.getName(), item.getDescription())); + } + } + + private void emitGenericHelp(BlazeRuntime runtime, OutErr outErr) { + outErr.printOut("Usage: blaze <command> <options> ...\n\n"); + + outErr.printOut("Available commands:\n"); + + Map<String, BlazeCommand> commandsByName = runtime.getCommandMap(); + List<String> namesInOrder = new ArrayList<>(commandsByName.keySet()); + Collections.sort(namesInOrder); + + for (String name : namesInOrder) { + BlazeCommand command = commandsByName.get(name); + Command annotation = command.getClass().getAnnotation(Command.class); + if (annotation.hidden()) { + continue; + } + + String shortDescription = annotation.shortDescription(); + outErr.printOut(String.format(" %-19s %s\n", name, shortDescription)); + } + + outErr.printOut("\n"); + outErr.printOut("Getting more help:\n"); + outErr.printOut(" blaze help <command>\n"); + outErr.printOut(" Prints help and options for <command>.\n"); + outErr.printOut(" blaze help startup_options\n"); + outErr.printOut(" Options for the JVM hosting Blaze.\n"); + outErr.printOut(" blaze help target-syntax\n"); + outErr.printOut(" Explains the syntax for specifying targets.\n"); + outErr.printOut(" blaze help info-keys\n"); + outErr.printOut(" Displays a list of keys used by the info command.\n"); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java new file mode 100644 index 0000000..31aaeb1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java
@@ -0,0 +1,448 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.base.Joiner; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.base.Supplier; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.ProtoUtils; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClassProvider; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.AllowedRuleClassInfo; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.AttributeDefinition; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.BuildLanguage; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.RuleDefinition; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.OsUtils; +import com.google.devtools.build.lib.util.StringUtilities; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Implementation of 'blaze info'. + */ +@Command(name = "info", + // TODO(bazel-team): this is not really a build command, but needs access to the + // configuration options to do its job + builds = true, + allowResidue = true, + binaryStdOut = true, + help = "resource:info.txt", + shortDescription = "Displays runtime info about the blaze server.", + options = { InfoCommand.Options.class }, + // We have InfoCommand inherit from {@link BuildCommand} because we want all + // configuration defaults specified in ~/.blazerc for {@code build} to apply to + // {@code info} too, even though it doesn't actually do a build. + // + // (Ideally there would be a way to make {@code info} inherit just the bare + // minimum of relevant options from {@code build}, i.e. those that affect the + // values it prints. But there's no such mechanism.) + inherits = { BuildCommand.class }) +public class InfoCommand implements BlazeCommand { + + public static class Options extends OptionsBase { + @Option(name = "show_make_env", + defaultValue = "false", + category = "misc", + help = "Include the \"Make\" environment in the output.") + public boolean showMakeEnvironment; + } + + /** + * Unchecked variant of ExitCausingException. Below, we need to throw from the Supplier interface, + * which does not allow checked exceptions. + */ + public static class ExitCausingRuntimeException extends RuntimeException { + + private final ExitCode exitCode; + + public ExitCausingRuntimeException(String message, ExitCode exitCode) { + super(message); + this.exitCode = exitCode; + } + + public ExitCausingRuntimeException(ExitCode exitCode) { + this.exitCode = exitCode; + } + + public ExitCode getExitCode() { + return exitCode; + } + } + + private static class HardwiredInfoItem implements BlazeModule.InfoItem { + private final InfoKey key; + private final BlazeRuntime runtime; + private final OptionsProvider commandOptions; + + private HardwiredInfoItem(InfoKey key, BlazeRuntime runtime, OptionsProvider commandOptions) { + this.key = key; + this.runtime = runtime; + this.commandOptions = commandOptions; + } + + @Override + public String getName() { + return key.getName(); + } + + @Override + public String getDescription() { + return key.getDescription(); + } + + @Override + public boolean isHidden() { + return key.isHidden(); + } + + @Override + public byte[] get(Supplier<BuildConfiguration> configurationSupplier) { + return print(getInfoItem(runtime, key, configurationSupplier, commandOptions)); + } + } + + private static class MakeInfoItem implements BlazeModule.InfoItem { + private final String name; + private final String value; + + private MakeInfoItem(String name, String value) { + this.name = name; + this.value = value; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getDescription() { + return "Make environment variable '" + name + "'"; + } + + @Override + public boolean isHidden() { + return false; + } + + @Override + public byte[] get(Supplier<BuildConfiguration> configurationSupplier) { + return print(value); + } + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { } + + @Override + public ExitCode exec(final BlazeRuntime runtime, final OptionsProvider optionsProvider) { + Options infoOptions = optionsProvider.getOptions(Options.class); + + OutErr outErr = runtime.getReporter().getOutErr(); + // Creating a BuildConfiguration is expensive and often unnecessary. Delay the creation until + // it is needed. + Supplier<BuildConfiguration> configurationSupplier = new Supplier<BuildConfiguration>() { + private BuildConfiguration configuration; + @Override + public BuildConfiguration get() { + if (configuration != null) { + return configuration; + } + try { + // In order to be able to answer configuration-specific queries, we need to setup the + // package path. Since info inherits all the build options, all the necessary information + // is available here. + runtime.setupPackageCache( + optionsProvider.getOptions(PackageCacheOptions.class), + runtime.getDefaultsPackageContent(optionsProvider)); + // TODO(bazel-team): What if there are multiple configurations? [multi-config] + configuration = runtime + .getConfigurations(optionsProvider) + .getTargetConfigurations().get(0); + return configuration; + } catch (InvalidConfigurationException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + throw new ExitCausingRuntimeException(ExitCode.COMMAND_LINE_ERROR); + } catch (AbruptExitException e) { + throw new ExitCausingRuntimeException("unknown error: " + e.getMessage(), + e.getExitCode()); + } catch (InterruptedException e) { + runtime.getReporter().handle(Event.error("interrupted")); + throw new ExitCausingRuntimeException(ExitCode.INTERRUPTED); + } + } + }; + + Map<String, BlazeModule.InfoItem> items = getInfoItemMap(runtime, optionsProvider); + + try { + if (infoOptions.showMakeEnvironment) { + Map<String, String> makeEnv = configurationSupplier.get().getMakeEnvironment(); + for (Map.Entry<String, String> entry : makeEnv.entrySet()) { + BlazeModule.InfoItem item = new MakeInfoItem(entry.getKey(), entry.getValue()); + items.put(item.getName(), item); + } + } + + List<String> residue = optionsProvider.getResidue(); + if (residue.size() > 1) { + runtime.getReporter().handle(Event.error("at most one key may be specified")); + return ExitCode.COMMAND_LINE_ERROR; + } + + String key = residue.size() == 1 ? residue.get(0) : null; + if (key != null) { // print just the value for the specified key: + byte[] value; + if (items.containsKey(key)) { + value = items.get(key).get(configurationSupplier); + } else { + runtime.getReporter().handle(Event.error("unknown key: '" + key + "'")); + return ExitCode.COMMAND_LINE_ERROR; + } + try { + outErr.getOutputStream().write(value); + outErr.getOutputStream().flush(); + } catch (IOException e) { + runtime.getReporter().handle(Event.error("Cannot write info block: " + e.getMessage())); + return ExitCode.ANALYSIS_FAILURE; + } + } else { // print them all + configurationSupplier.get(); // We'll need this later anyway + for (BlazeModule.InfoItem infoItem : items.values()) { + if (infoItem.isHidden()) { + continue; + } + outErr.getOutputStream().write( + (infoItem.getName() + ": ").getBytes(StandardCharsets.UTF_8)); + outErr.getOutputStream().write(infoItem.get(configurationSupplier)); + } + } + } catch (AbruptExitException e) { + return e.getExitCode(); + } catch (ExitCausingRuntimeException e) { + return e.getExitCode(); + } catch (IOException e) { + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } + return ExitCode.SUCCESS; + } + + /** + * Compute and return the info for the given key. Only keys that are not hidden are supported + * here. + */ + private static Object getInfoItem(BlazeRuntime runtime, InfoKey key, + Supplier<BuildConfiguration> configurationSupplier, OptionsProvider options) { + switch (key) { + // directories + case WORKSPACE : return runtime.getWorkspace(); + case INSTALL_BASE : return runtime.getDirectories().getInstallBase(); + case OUTPUT_BASE : return runtime.getOutputBase(); + case EXECUTION_ROOT : return runtime.getExecRoot(); + case OUTPUT_PATH : return runtime.getDirectories().getOutputPath(); + // These are the only (non-hidden) info items that require a configuration, because the + // corresponding paths contain the short name. Maybe we should recommend using the symlinks + // or make them hidden by default? + case BLAZE_BIN : return configurationSupplier.get().getBinDirectory().getPath(); + case BLAZE_GENFILES : return configurationSupplier.get().getGenfilesDirectory().getPath(); + case BLAZE_TESTLOGS : return configurationSupplier.get().getTestLogsDirectory().getPath(); + + // logs + case COMMAND_LOG : return BlazeCommandDispatcher.getCommandLogPath(runtime.getOutputBase()); + case MESSAGE_LOG : + // NB: Duplicated in EventLogModule + return runtime.getOutputBase().getRelative("message.log"); + + // misc + case RELEASE : return BlazeVersionInfo.instance().getReleaseName(); + case SERVER_PID : return OsUtils.getpid(); + case PACKAGE_PATH : return getPackagePath(options); + + // memory statistics + case GC_COUNT : + case GC_TIME : + // The documentation is not very clear on what it means to have more than + // one GC MXBean, so we just sum them up. + int gcCount = 0; + int gcTime = 0; + for (GarbageCollectorMXBean gcBean : ManagementFactory.getGarbageCollectorMXBeans()) { + gcCount += gcBean.getCollectionCount(); + gcTime += gcBean.getCollectionTime(); + } + if (key == InfoKey.GC_COUNT) { + return gcCount + ""; + } else { + return gcTime + "ms"; + } + + case MAX_HEAP_SIZE : + return StringUtilities.prettyPrintBytes(getMemoryUsage().getMax()); + case USED_HEAP_SIZE : + case COMMITTED_HEAP_SIZE : + return StringUtilities.prettyPrintBytes(key == InfoKey.USED_HEAP_SIZE ? + getMemoryUsage().getUsed() : getMemoryUsage().getCommitted()); + + case USED_HEAP_SIZE_AFTER_GC : + // Note that this info value is not printed by default, but only when explicitly requested. + System.gc(); + return StringUtilities.prettyPrintBytes(getMemoryUsage().getUsed()); + + case DEFAULTS_PACKAGE: + return runtime.getDefaultsPackageContent(); + + case BUILD_LANGUAGE: + return getBuildLanguageDefinition(runtime.getRuleClassProvider()); + + case DEFAULT_PACKAGE_PATH: + return Joiner.on(":").join(Constants.DEFAULT_PACKAGE_PATH); + + default: + throw new IllegalArgumentException("missing implementation for " + key); + } + } + + private static MemoryUsage getMemoryUsage() { + MemoryMXBean memBean = ManagementFactory.getMemoryMXBean(); + return memBean.getHeapMemoryUsage(); + } + + /** + * Get the package_path variable for the given set of options. + */ + private static String getPackagePath(OptionsProvider options) { + PackageCacheOptions packageCacheOptions = + options.getOptions(PackageCacheOptions.class); + return Joiner.on(":").join(packageCacheOptions.packagePath); + } + + private static AllowedRuleClassInfo getAllowedRuleClasses( + Collection<RuleClass> ruleClasses, Attribute attr) { + AllowedRuleClassInfo.Builder info = AllowedRuleClassInfo.newBuilder(); + info.setPolicy(AllowedRuleClassInfo.AllowedRuleClasses.ANY); + + if (attr.isStrictLabelCheckingEnabled()) { + if (attr.getAllowedRuleClassesPredicate() != Predicates.<RuleClass>alwaysTrue()) { + info.setPolicy(AllowedRuleClassInfo.AllowedRuleClasses.SPECIFIED); + Predicate<RuleClass> filter = attr.getAllowedRuleClassesPredicate(); + for (RuleClass otherClass : Iterables.filter( + ruleClasses, filter)) { + if (otherClass.isDocumented()) { + info.addAllowedRuleClass(otherClass.getName()); + } + } + } + } + + return info.build(); + } + + /** + * Returns a byte array containing a proto-buffer describing the build language. + */ + private static byte[] getBuildLanguageDefinition(RuleClassProvider provider) { + BuildLanguage.Builder resultPb = BuildLanguage.newBuilder(); + Collection<RuleClass> ruleClasses = provider.getRuleClassMap().values(); + for (RuleClass ruleClass : ruleClasses) { + if (!ruleClass.isDocumented()) { + continue; + } + + RuleDefinition.Builder rulePb = RuleDefinition.newBuilder(); + rulePb.setName(ruleClass.getName()); + for (Attribute attr : ruleClass.getAttributes()) { + if (!attr.isDocumented()) { + continue; + } + + AttributeDefinition.Builder attrPb = AttributeDefinition.newBuilder(); + attrPb.setName(attr.getName()); + // The protocol compiler, in its infinite wisdom, generates the field as one of the + // integer type and the getTypeEnum() method is missing. WTF? + attrPb.setType(ProtoUtils.getDiscriminatorFromType(attr.getType())); + attrPb.setMandatory(attr.isMandatory()); + + if (Type.isLabelType(attr.getType())) { + attrPb.setAllowedRuleClasses(getAllowedRuleClasses(ruleClasses, attr)); + } + + rulePb.addAttribute(attrPb); + } + + resultPb.addRule(rulePb); + } + + return resultPb.build().toByteArray(); + } + + private static byte[] print(Object value) { + if (value instanceof byte[]) { + return (byte[]) value; + } + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PrintWriter writer = new PrintWriter(outputStream); + writer.print(value.toString() + "\n"); + writer.flush(); + return outputStream.toByteArray(); + } + + static Map<String, BlazeModule.InfoItem> getInfoItemMap( + BlazeRuntime runtime, OptionsProvider commandOptions) { + Map<String, BlazeModule.InfoItem> result = new TreeMap<>(); // order by key + for (BlazeModule module : runtime.getBlazeModules()) { + for (BlazeModule.InfoItem item : module.getInfoItems()) { + result.put(item.getName(), item); + } + } + + for (InfoKey key : InfoKey.values()) { + BlazeModule.InfoItem item = new HardwiredInfoItem(key, runtime, commandOptions); + result.put(item.getName(), item); + } + + return result; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java new file mode 100644 index 0000000..d2e7bc0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java
@@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + + +/** + * An enumeration of all the valid info keys, excepting the make environment + * variables. + */ +public enum InfoKey { + // directories + WORKSPACE("workspace", "The working directory of the server."), + INSTALL_BASE("install_base", "The installation base directory."), + OUTPUT_BASE("output_base", + "A directory for shared Blaze state as well as tool and strategy specific subdirectories."), + EXECUTION_ROOT("execution_root", + "A directory that makes all input and output files visible to the build."), + OUTPUT_PATH("output_path", "Output directory"), + BLAZE_BIN("blaze-bin", "Configuration dependent directory for binaries."), + BLAZE_GENFILES("blaze-genfiles", "Configuration dependent directory for generated files."), + BLAZE_TESTLOGS("blaze-testlogs", "Configuration dependent directory for logs from a test run."), + + // logs + COMMAND_LOG("command_log", "Location of the log containg the output from the build commands."), + MESSAGE_LOG("message_log" , + "Location of a log containing machine readable message in LogMessage protobuf format."), + + // misc + RELEASE("release", "Blaze release identifier"), + SERVER_PID("server_pid", "Blaze process id"), + PACKAGE_PATH("package_path", "The search path for resolving package labels."), + + // memory statistics + USED_HEAP_SIZE("used-heap-size", "The amount of used memory in bytes. Note that this is not a " + + "good indicator of the actual memory use, as it includes any remaining inaccessible " + + "memory."), + USED_HEAP_SIZE_AFTER_GC("used-heap-size-after-gc", + "The amount of used memory in bytes after a call to System.gc().", true), + COMMITTED_HEAP_SIZE("committed-heap-size", + "The amount of memory in bytes that is committed for the Java virtual machine to use"), + MAX_HEAP_SIZE("max-heap-size", + "The maximum amount of memory in bytes that can be used for memory management."), + GC_COUNT("gc-count", "Number of garbage collection runs."), + GC_TIME("gc-time", "The approximate accumulated time spend on garbage collection."), + + // These are deprecated, they still work, when explicitly requested, but are not shown by default + + // These keys print multi-line messages and thus don't play well with grep. We don't print them + // unless explicitly requested + DEFAULTS_PACKAGE("defaults-package", "Default packages used as implicit dependencies", true), + BUILD_LANGUAGE("build-language", "A protobuffer with the build language structure", true), + DEFAULT_PACKAGE_PATH("default-package-path", "The default package path", true); + + private final String name; + private final String description; + private final boolean hidden; + + private InfoKey(String name, String description) { + this(name, description, false); + } + + private InfoKey(String name, String description, boolean hidden) { + this.name = name; + this.description = description; + this.hidden = hidden; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public boolean isHidden() { + return hidden; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java new file mode 100644 index 0000000..7b91dc7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java
@@ -0,0 +1,771 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.base.Function; +import com.google.common.base.Functions; +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import com.google.common.collect.Ordering; +import com.google.common.collect.TreeMultimap; +import com.google.devtools.build.lib.actions.MiddlemanAction; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.profiler.ProfileInfo; +import com.google.devtools.build.lib.profiler.ProfileInfo.CriticalPathEntry; +import com.google.devtools.build.lib.profiler.ProfileInfo.InfoListener; +import com.google.devtools.build.lib.profiler.ProfilePhase; +import com.google.devtools.build.lib.profiler.ProfilePhaseStatistics; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.profiler.chart.AggregatingChartCreator; +import com.google.devtools.build.lib.profiler.chart.Chart; +import com.google.devtools.build.lib.profiler.chart.ChartCreator; +import com.google.devtools.build.lib.profiler.chart.DetailedChartCreator; +import com.google.devtools.build.lib.profiler.chart.HtmlChartVisitor; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.StringUtil; +import com.google.devtools.build.lib.util.TimeUtilities; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; + +/** + * Command line wrapper for analyzing Blaze build profiles. + */ +@Command(name = "analyze-profile", + options = { ProfileCommand.ProfileOptions.class }, + shortDescription = "Analyzes build profile data.", + help = "resource:analyze-profile.txt", + allowResidue = true, + mustRunInWorkspace = false) +public final class ProfileCommand implements BlazeCommand { + + private final String TWO_COLUMN_FORMAT = "%-37s %10s\n"; + private final String THREE_COLUMN_FORMAT = "%-28s %10s %8s\n"; + + public static class DumpConverter extends Converters.StringSetConverter { + public DumpConverter() { + super("text", "raw", "text-unsorted", "raw-unsorted"); + } + } + + public static class ProfileOptions extends OptionsBase { + @Option(name = "dump", + abbrev='d', + converter = DumpConverter.class, + defaultValue = "null", + help = "output full profile data dump either in human-readable 'text' format or" + + " script-friendly 'raw' format, either sorted or unsorted.") + public String dumpMode; + + @Option(name = "html", + defaultValue = "false", + help = "If present, an HTML file visualizing the tasks of the profiled build is created. " + + "The name of the html file is the name of the profile file plus '.html'.") + public boolean html; + + @Option(name = "html_pixels_per_second", + defaultValue = "50", + help = "Defines the scale of the time axis of the task diagram. The unit is " + + "pixels per second. Default is 50 pixels per second. ") + public int htmlPixelsPerSecond; + + @Option(name = "html_details", + defaultValue = "false", + help = "If --html_details is present, the task diagram contains all tasks of the profile. " + + "If --nohtml_details is present, an aggregated diagram is generated. The default is " + + "to generate an aggregated diagram.") + public boolean htmlDetails; + + @Option(name = "vfs_stats", + defaultValue = "false", + help = "If present, include VFS path statistics.") + public boolean vfsStats; + + @Option(name = "vfs_stats_limit", + defaultValue = "-1", + help = "Maximum number of VFS path statistics to print.") + public int vfsStatsLimit; + } + + private Function<String, String> currentPathMapping = Functions.<String>identity(); + + private InfoListener getInfoListener(final BlazeRuntime runtime) { + return new InfoListener() { + private final EventHandler reporter = runtime.getReporter(); + + @Override + public void info(String text) { + reporter.handle(Event.info(text)); + } + + @Override + public void warn(String text) { + reporter.handle(Event.warn(text)); + } + }; + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} + + @Override + public ExitCode exec(final BlazeRuntime runtime, OptionsProvider options) { + ProfileOptions opts = + options.getOptions(ProfileOptions.class); + + if (!opts.vfsStats) { + opts.vfsStatsLimit = 0; + } + + currentPathMapping = new Function<String, String>() { + @Override + public String apply(String input) { + if (runtime.getWorkspaceName().isEmpty()) { + return input; + } else { + return input.substring(input.lastIndexOf("/" + runtime.getWorkspaceName()) + 1); + } + } + }; + + PrintStream out = new PrintStream(runtime.getReporter().getOutErr().getOutputStream()); + try { + runtime.getReporter().handle(Event.warn( + null, "This information is intended for consumption by Blaze developers" + + " only, and may change at any time. Script against it at your own risk")); + + for (String name : options.getResidue()) { + Path profileFile = runtime.getWorkingDirectory().getRelative(name); + try { + ProfileInfo info = ProfileInfo.loadProfileVerbosely( + profileFile, getInfoListener(runtime)); + if (opts.dumpMode != null) { + dumpProfile(runtime, info, out, opts.dumpMode); + } else if (opts.html) { + createHtml(runtime, info, profileFile, opts); + } else { + createText(runtime, info, out, opts); + } + } catch (IOException e) { + runtime.getReporter().handle(Event.error( + null, "Failed to process file " + name + ": " + e.getMessage())); + } + } + } finally { + out.flush(); + } + return ExitCode.SUCCESS; + } + + private void createText(BlazeRuntime runtime, ProfileInfo info, PrintStream out, + ProfileOptions opts) { + List<ProfilePhaseStatistics> statistics = getStatistics(runtime, info, opts); + + for (ProfilePhaseStatistics stat : statistics) { + String title = stat.getTitle(); + + if (!title.equals("")) { + out.println("\n=== " + title.toUpperCase() + " ===\n"); + } + out.print(stat.getStatistics()); + } + } + + private void createHtml(BlazeRuntime runtime, ProfileInfo info, Path profileFile, + ProfileOptions opts) + throws IOException { + Path htmlFile = + profileFile.getParentDirectory().getChild(profileFile.getBaseName() + ".html"); + List<ProfilePhaseStatistics> statistics = getStatistics(runtime, info, opts); + + runtime.getReporter().handle(Event.info("Creating HTML output in " + htmlFile)); + + ChartCreator chartCreator = + opts.htmlDetails ? new DetailedChartCreator(info, statistics) + : new AggregatingChartCreator(info, statistics); + Chart chart = chartCreator.create(); + OutputStream out = new BufferedOutputStream(htmlFile.getOutputStream()); + try { + chart.accept(new HtmlChartVisitor(new PrintStream(out), opts.htmlPixelsPerSecond)); + } finally { + try { + out.close(); + } catch (IOException e) { + // Ignore + } + } + } + + private List<ProfilePhaseStatistics> getStatistics( + BlazeRuntime runtime, ProfileInfo info, ProfileOptions opts) { + try { + ProfileInfo.aggregateProfile(info, getInfoListener(runtime)); + runtime.getReporter().handle(Event.info("Analyzing relationships")); + + info.analyzeRelationships(); + + List<ProfilePhaseStatistics> statistics = new ArrayList<>(); + + // Print phase durations and total execution time + ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(byteOutput, false, "UTF-8"); + long duration = 0; + for (ProfilePhase phase : ProfilePhase.values()) { + ProfileInfo.Task phaseTask = info.getPhaseTask(phase); + if (phaseTask != null) { + duration += info.getPhaseDuration(phaseTask); + } + } + for (ProfilePhase phase : ProfilePhase.values()) { + ProfileInfo.Task phaseTask = info.getPhaseTask(phase); + if (phaseTask != null) { + long phaseDuration = info.getPhaseDuration(phaseTask); + out.printf(THREE_COLUMN_FORMAT, "Total " + phase.nick + " phase time", + TimeUtilities.prettyTime(phaseDuration), prettyPercentage(phaseDuration, duration)); + } + } + out.printf(THREE_COLUMN_FORMAT, "Total run time", TimeUtilities.prettyTime(duration), + "100.00%"); + statistics.add(new ProfilePhaseStatistics("Phase Summary Information", + new String(byteOutput.toByteArray(), "UTF-8"))); + + // Print details of major phases + if (duration > 0) { + statistics.add(formatInitPhaseStatistics(info, opts)); + statistics.add(formatLoadingPhaseStatistics(info, opts)); + statistics.add(formatAnalysisPhaseStatistics(info, opts)); + ProfilePhaseStatistics stat = formatExecutionPhaseStatistics(info, opts); + if (stat != null) { + statistics.add(stat); + } + } + + return statistics; + } catch (UnsupportedEncodingException e) { + throw new AssertionError("Should not happen since, UTF8 is available on all JVMs"); + } + } + + private void dumpProfile( + BlazeRuntime runtime, ProfileInfo info, PrintStream out, String dumpMode) { + if (!dumpMode.contains("unsorted")) { + ProfileInfo.aggregateProfile(info, getInfoListener(runtime)); + } + if (dumpMode.contains("raw")) { + for (ProfileInfo.Task task : info.allTasksById) { + dumpRaw(task, out); + } + } else if (dumpMode.contains("unsorted")) { + for (ProfileInfo.Task task : info.allTasksById) { + dumpTask(task, out, 0); + } + } else { + for (ProfileInfo.Task task : info.rootTasksById) { + dumpTask(task, out, 0); + } + } + } + + private void dumpTask(ProfileInfo.Task task, PrintStream out, int indent) { + StringBuilder builder = new StringBuilder(String.format( + "\n%s %s\nThread: %-6d Id: %-6d Parent: %d\nStart time: %-12s Duration: %s", + task.type, task.getDescription(), task.threadId, task.id, task.parentId, + TimeUtilities.prettyTime(task.startTime), TimeUtilities.prettyTime(task.duration))); + if (task.hasStats()) { + builder.append("\n"); + ProfileInfo.AggregateAttr[] stats = task.getStatAttrArray(); + for (ProfilerTask type : ProfilerTask.values()) { + ProfileInfo.AggregateAttr attr = stats[type.ordinal()]; + if (attr != null) { + builder.append(type.toString().toLowerCase()).append("=("). + append(attr.count).append(", "). + append(TimeUtilities.prettyTime(attr.totalTime)).append(") "); + } + } + } + out.println(StringUtil.indent(builder.toString(), indent)); + for (ProfileInfo.Task subtask : task.subtasks) { + dumpTask(subtask, out, indent + 1); + } + } + + private void dumpRaw(ProfileInfo.Task task, PrintStream out) { + StringBuilder aggregateString = new StringBuilder(); + ProfileInfo.AggregateAttr[] stats = task.getStatAttrArray(); + for (ProfilerTask type : ProfilerTask.values()) { + ProfileInfo.AggregateAttr attr = stats[type.ordinal()]; + if (attr != null) { + aggregateString.append(type.toString().toLowerCase()).append(","). + append(attr.count).append(",").append(attr.totalTime).append(" "); + } + } + out.println( + task.threadId + "|" + task.id + "|" + task.parentId + "|" + + task.startTime + "|" + task.duration + "|" + + aggregateString.toString().trim() + "|" + + task.type + "|" + task.getDescription()); + } + + /** + * Converts relative duration to the percentage string + * @return formatted percentage string or "N/A" if result is undefined. + */ + private static String prettyPercentage(long duration, long total) { + if (total == 0) { + // Return "not available" string if total is 0 and result is undefined. + return "N/A"; + } + return String.format("%5.2f%%", duration*100.0/total); + } + + private void printCriticalPath(String title, PrintStream out, CriticalPathEntry path) { + out.println(String.format("\n%s (%s):", title, + TimeUtilities.prettyTime(path.cumulativeDuration))); + + boolean lightCriticalPath = isLightCriticalPath(path); + out.println(lightCriticalPath ? + String.format("%6s %11s %8s %s", "Id", "Time", "Percentage", "Description") + : String.format("%6s %11s %8s %8s %s", "Id", "Time", "Share", "Critical", "Description")); + + long totalPathTime = path.cumulativeDuration; + int middlemanCount = 0; + long middlemanDuration = 0L; + long middlemanCritTime = 0L; + + for (; path != null ; path = path.next) { + if (path.task.id < 0) { + // Ignore fake actions. + continue; + } else if (path.task.getDescription().startsWith(MiddlemanAction.MIDDLEMAN_MNEMONIC + " ") + || path.task.getDescription().startsWith("TargetCompletionMiddleman")) { + // Aggregate middleman actions. + middlemanCount++; + middlemanDuration += path.duration; + middlemanCritTime += path.getCriticalTime(); + } else { + String desc = path.task.getDescription().replace(':', ' '); + if (lightCriticalPath) { + out.println(String.format("%6d %11s %8s %s", path.task.id, + TimeUtilities.prettyTime(path.duration), + prettyPercentage(path.duration, totalPathTime), + desc)); + } else { + out.println(String.format("%6d %11s %8s %8s %s", path.task.id, + TimeUtilities.prettyTime(path.duration), + prettyPercentage(path.duration, totalPathTime), + prettyPercentage(path.getCriticalTime(), totalPathTime), desc)); + } + } + } + if (middlemanCount > 0) { + if (lightCriticalPath) { + out.println(String.format(" %11s %8s [%d middleman actions]", + TimeUtilities.prettyTime(middlemanDuration), + prettyPercentage(middlemanDuration, totalPathTime), + middlemanCount)); + } else { + out.println(String.format(" %11s %8s %8s [%d middleman actions]", + TimeUtilities.prettyTime(middlemanDuration), + prettyPercentage(middlemanDuration, totalPathTime), + prettyPercentage(middlemanCritTime, totalPathTime), middlemanCount)); + } + } + } + + private boolean isLightCriticalPath(CriticalPathEntry path) { + return path.task.type == ProfilerTask.CRITICAL_PATH_COMPONENT; + } + + private void printShortPhaseAnalysis(ProfileInfo info, PrintStream out, ProfilePhase phase) { + ProfileInfo.Task phaseTask = info.getPhaseTask(phase); + if (phaseTask != null) { + long phaseDuration = info.getPhaseDuration(phaseTask); + out.printf(TWO_COLUMN_FORMAT, "Total " + phase.nick + " phase time", + TimeUtilities.prettyTime(phaseDuration)); + printTimeDistributionByType(info, out, phaseTask); + } + } + + private void printTimeDistributionByType(ProfileInfo info, PrintStream out, + ProfileInfo.Task phaseTask) { + List<ProfileInfo.Task> taskList = info.getTasksForPhase(phaseTask); + long phaseDuration = info.getPhaseDuration(phaseTask); + long totalDuration = phaseDuration; + for (ProfileInfo.Task task : taskList) { + // Tasks on the phaseTask thread already accounted for in the phaseDuration. + if (task.threadId != phaseTask.threadId) { + totalDuration += task.duration; + } + } + boolean headerNeeded = true; + for (ProfilerTask type : ProfilerTask.values()) { + ProfileInfo.AggregateAttr stats = info.getStatsForType(type, taskList); + if (stats.count > 0 && stats.totalTime > 0) { + if (headerNeeded) { + out.println("\nTotal time (across all threads) spent on:"); + out.println(String.format("%18s %8s %8s %11s", "Type", "Total", "Count", "Average")); + headerNeeded = false; + } + out.println(String.format("%18s %8s %8d %11s", type.toString(), + prettyPercentage(stats.totalTime, totalDuration), stats.count, + TimeUtilities.prettyTime(stats.totalTime / stats.count))); + } + } + } + + static class Stat implements Comparable<Stat> { + public long duration; + public long frequency; + + @Override + public int compareTo(Stat o) { + return this.duration == o.duration ? Long.compare(this.frequency, o.frequency) + : Long.compare(this.duration, o.duration); + } + } + + /** + * Print the time spent on VFS operations on each path. Output is grouped by operation and sorted + * by descending duration. If multiple of the same VFS operation were logged for the same path, + * print the total duration. + * + * @param info profiling data. + * @param out output stream. + * @param phase build phase. + * @param limit maximum number of statistics to print, or -1 for no limit. + */ + private void printVfsStatistics(ProfileInfo info, PrintStream out, + ProfilePhase phase, int limit) { + ProfileInfo.Task phaseTask = info.getPhaseTask(phase); + if (phaseTask == null) { + return; + } + + if (limit == 0) { + return; + } + + // Group into VFS operations and build maps from path to duration. + + List<ProfileInfo.Task> taskList = info.getTasksForPhase(phaseTask); + EnumMap<ProfilerTask, Map<String, Stat>> stats = Maps.newEnumMap(ProfilerTask.class); + + collectVfsEntries(stats, taskList); + + if (!stats.isEmpty()) { + out.printf("\nVFS path statistics:\n"); + out.printf("%15s %10s %10s %s\n", "Type", "Frequency", "Duration", "Path"); + } + + // Reverse the maps to get maps from duration to path. We use a TreeMultimap to sort by duration + // and because durations are not unique. + + for (ProfilerTask type : stats.keySet()) { + Map<String, Stat> statsForType = stats.get(type); + TreeMultimap<Stat, String> sortedStats = + TreeMultimap.create(Ordering.natural().reverse(), Ordering.natural()); + + for (Map.Entry<String, Stat> stat : statsForType.entrySet()) { + sortedStats.put(stat.getValue(), stat.getKey()); + } + + int numPrinted = 0; + for (Map.Entry<Stat, String> stat : sortedStats.entries()) { + if (limit != -1 && numPrinted++ == limit) { + out.printf("... %d more ...\n", sortedStats.size() - limit); + break; + } + out.printf("%15s %10d %10s %s\n", + type.name(), stat.getKey().frequency, TimeUtilities.prettyTime(stat.getKey().duration), + stat.getValue()); + } + } + } + + private void collectVfsEntries(EnumMap<ProfilerTask, Map<String, Stat>> stats, + List<ProfileInfo.Task> taskList) { + for (ProfileInfo.Task task : taskList) { + collectVfsEntries(stats, Arrays.asList(task.subtasks)); + if (!task.type.name().startsWith("VFS_")) { + continue; + } + + Map<String, Stat> statsForType = stats.get(task.type); + if (statsForType == null) { + statsForType = Maps.newHashMap(); + stats.put(task.type, statsForType); + } + + String path = currentPathMapping.apply(task.getDescription()); + + Stat stat = statsForType.get(path); + if (stat == null) { + stat = new Stat(); + } + + stat.duration += task.duration; + stat.frequency++; + statsForType.put(path, stat); + } + } + + /** + * Returns set of profiler tasks to be filtered from critical path. + * Also always filters out ACTION_LOCK and WAIT tasks to simulate + * unlimited resource critical path (see comments inside formatExecutionPhaseStatistics() + * method). + */ + private EnumSet<ProfilerTask> getTypeFilter(ProfilerTask... tasks) { + EnumSet<ProfilerTask> filter = EnumSet.of(ProfilerTask.ACTION_LOCK, ProfilerTask.WAIT); + for (ProfilerTask task : tasks) { + filter.add(task); + } + return filter; + } + + private ProfilePhaseStatistics formatInitPhaseStatistics(ProfileInfo info, ProfileOptions opts) + throws UnsupportedEncodingException { + return formatSimplePhaseStatistics(info, opts, "Init", ProfilePhase.INIT); + } + + private ProfilePhaseStatistics formatLoadingPhaseStatistics(ProfileInfo info, ProfileOptions opts) + throws UnsupportedEncodingException { + return formatSimplePhaseStatistics(info, opts, "Loading", ProfilePhase.LOAD); + } + + private ProfilePhaseStatistics formatAnalysisPhaseStatistics(ProfileInfo info, + ProfileOptions opts) + throws UnsupportedEncodingException { + return formatSimplePhaseStatistics(info, opts, "Analysis", ProfilePhase.ANALYZE); + } + + private ProfilePhaseStatistics formatSimplePhaseStatistics(ProfileInfo info, + ProfileOptions opts, + String name, + ProfilePhase phase) + throws UnsupportedEncodingException { + ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(byteOutput, false, "UTF-8"); + + printShortPhaseAnalysis(info, out, phase); + printVfsStatistics(info, out, phase, opts.vfsStatsLimit); + return new ProfilePhaseStatistics(name + " Phase Information", + new String(byteOutput.toByteArray(), "UTF-8")); + } + + private ProfilePhaseStatistics formatExecutionPhaseStatistics(ProfileInfo info, + ProfileOptions opts) + throws UnsupportedEncodingException { + ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(byteOutput, false, "UTF-8"); + + ProfileInfo.Task prepPhase = info.getPhaseTask(ProfilePhase.PREPARE); + ProfileInfo.Task execPhase = info.getPhaseTask(ProfilePhase.EXECUTE); + ProfileInfo.Task finishPhase = info.getPhaseTask(ProfilePhase.FINISH); + if (execPhase == null) { + return null; + } + + List<ProfileInfo.Task> execTasks = info.getTasksForPhase(execPhase); + long graphTime = info.getStatsForType(ProfilerTask.ACTION_GRAPH, execTasks).totalTime; + long execTime = info.getPhaseDuration(execPhase) - graphTime; + + if (prepPhase != null) { + out.printf(TWO_COLUMN_FORMAT, "Total preparation time", + TimeUtilities.prettyTime(info.getPhaseDuration(prepPhase))); + } + out.printf(TWO_COLUMN_FORMAT, "Total execution phase time", + TimeUtilities.prettyTime(info.getPhaseDuration(execPhase))); + if (finishPhase != null) { + out.printf(TWO_COLUMN_FORMAT, "Total time finalizing build", + TimeUtilities.prettyTime(info.getPhaseDuration(finishPhase))); + } + out.println(""); + out.printf(TWO_COLUMN_FORMAT, "Action dependency map creation", + TimeUtilities.prettyTime(graphTime)); + out.printf(TWO_COLUMN_FORMAT, "Actual execution time", + TimeUtilities.prettyTime(execTime)); + + EnumSet<ProfilerTask> typeFilter = EnumSet.noneOf(ProfilerTask.class); + CriticalPathEntry totalPath = info.getCriticalPath(typeFilter); + info.analyzeCriticalPath(typeFilter, totalPath); + + typeFilter = getTypeFilter(); + CriticalPathEntry optimalPath = info.getCriticalPath(typeFilter); + info.analyzeCriticalPath(typeFilter, optimalPath); + + if (totalPath != null) { + printCriticalPathTimingBreakdown(info, totalPath, optimalPath, execTime, out); + } else { + out.println("\nCritical path not available because no action graph was generated."); + } + + printTimeDistributionByType(info, out, execPhase); + + if (totalPath != null) { + printCriticalPath("Critical path", out, totalPath); + // In light critical path we do not record scheduling delay data so it does not make sense + // to differentiate it. + if (!isLightCriticalPath(totalPath)) { + printCriticalPath("Critical path excluding scheduling delays", out, optimalPath); + } + } + + if (info.getMissingActionsCount() > 0) { + out.println("\n" + info.getMissingActionsCount() + " action(s) are present in the" + + " action graph but missing instrumentation data. Most likely profile file" + + " has been created for the failed or aborted build."); + } + + printVfsStatistics(info, out, ProfilePhase.EXECUTE, opts.vfsStatsLimit); + + return new ProfilePhaseStatistics("Execution Phase Information", + new String(byteOutput.toByteArray(), "UTF-8")); + } + + void printCriticalPathTimingBreakdown(ProfileInfo info, CriticalPathEntry totalPath, + CriticalPathEntry optimalPath, long execTime, PrintStream out) { + Preconditions.checkNotNull(totalPath); + Preconditions.checkNotNull(optimalPath); + // TODO(bazel-team): Print remote vs build stats recorded by CriticalPathStats + if (isLightCriticalPath(totalPath)) { + return; + } + out.println(totalPath.task.type); + // Worker thread pool scheduling delays for the actual critical path. + long workerWaitTime = 0; + long mainThreadWaitTime = 0; + for (ProfileInfo.CriticalPathEntry entry = totalPath; entry != null; entry = entry.next) { + workerWaitTime += info.getActionWaitTime(entry.task); + mainThreadWaitTime += info.getActionQueueTime(entry.task); + } + out.printf(TWO_COLUMN_FORMAT, "Worker thread scheduling delays", + TimeUtilities.prettyTime(workerWaitTime)); + out.printf(TWO_COLUMN_FORMAT, "Main thread scheduling delays", + TimeUtilities.prettyTime(mainThreadWaitTime)); + + out.println("\nCritical path time:"); + // Actual critical path. + long totalTime = totalPath.cumulativeDuration; + out.printf("%-37s %10s (%s of execution time)\n", "Actual time", + TimeUtilities.prettyTime(totalTime), + prettyPercentage(totalTime, execTime)); + // Unlimited resource critical path. Essentially, we assume that if we + // remove all scheduling delays caused by resource semaphore contention, + // each action execution time would not change (even though load now would + // be substantially higher - so this assumption might be incorrect but it is + // still useful for modeling). Given those assumptions we calculate critical + // path excluding scheduling delays. + long optimalTime = optimalPath.cumulativeDuration; + out.printf("%-37s %10s (%s of execution time)\n", "Time excluding scheduling delays", + TimeUtilities.prettyTime(optimalTime), + prettyPercentage(optimalTime, execTime)); + + // Artificial critical path if we ignore all the time spent in all tasks, + // except time directly attributed to the ACTION tasks. + out.println("\nTime related to:"); + + EnumSet<ProfilerTask> typeFilter = EnumSet.allOf(ProfilerTask.class); + ProfileInfo.CriticalPathEntry path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the builder overhead", + prettyPercentage(path.cumulativeDuration, totalTime)); + + typeFilter = getTypeFilter(); + for (ProfilerTask task : ProfilerTask.values()) { + if (task.name().startsWith("VFS_")) { + typeFilter.add(task); + } + } + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the VFS calls", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.ACTION_CHECK); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the dependency checking", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.ACTION_EXECUTE); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the execution setup", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.SPAWN, ProfilerTask.LOCAL_EXECUTION); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "local execution", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.SCANNER); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the include scanner", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.REMOTE_EXECUTION, ProfilerTask.PROCESS_TIME, + ProfilerTask.LOCAL_PARSE, ProfilerTask.UPLOAD_TIME, + ProfilerTask.REMOTE_QUEUE, ProfilerTask.REMOTE_SETUP, ProfilerTask.FETCH); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "Remote execution (cumulative)", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter( ProfilerTask.UPLOAD_TIME, ProfilerTask.REMOTE_SETUP); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " file uploads", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.FETCH); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " file fetching", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.PROCESS_TIME); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " process time", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.REMOTE_QUEUE); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " remote queueing", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.LOCAL_PARSE); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " remote execution parse", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.REMOTE_EXECUTION); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " other remote activities", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java new file mode 100644 index 0000000..2e5faf6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java
@@ -0,0 +1,93 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.CommonCommandOptions; +import com.google.devtools.build.lib.runtime.ProjectFile; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.OptionPriority; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.List; + +/** + * Provides support for implementations for {@link BlazeCommand} to work with {@link ProjectFile}. + */ +public final class ProjectFileSupport { + static final String PROJECT_FILE_PREFIX = "+"; + + private ProjectFileSupport() {} + + /** + * Reads any project files specified on the command line and updates the options parser + * accordingly. If project files cannot be read or if they contain unparsable options, or if they + * are not enabled, then it throws an exception instead. + */ + public static void handleProjectFiles(BlazeRuntime runtime, OptionsParser optionsParser, + String command) throws AbruptExitException { + List<String> targets = optionsParser.getResidue(); + ProjectFile.Provider projectFileProvider = runtime.getProjectFileProvider(); + if (projectFileProvider != null && targets.size() > 0 + && targets.get(0).startsWith(PROJECT_FILE_PREFIX)) { + if (targets.size() > 1) { + throw new AbruptExitException("Cannot handle more than one +<file> argument yet", + ExitCode.COMMAND_LINE_ERROR); + } + if (!optionsParser.getOptions(CommonCommandOptions.class).allowProjectFiles) { + throw new AbruptExitException("project file support is not enabled", + ExitCode.COMMAND_LINE_ERROR); + } + // TODO(bazel-team): This is currently treated as a path relative to the workspace - if the + // cwd is a subdirectory of the workspace, that will be surprising, and we should interpret it + // relative to the cwd instead. + PathFragment projectFilePath = new PathFragment(targets.get(0).substring(1)); + List<Path> packagePath = PathPackageLocator.create( + optionsParser.getOptions(PackageCacheOptions.class).packagePath, runtime.getReporter(), + runtime.getWorkspace(), runtime.getWorkingDirectory()).getPathEntries(); + ProjectFile projectFile = projectFileProvider.getProjectFile(packagePath, projectFilePath); + runtime.getReporter().handle(Event.info("Using " + projectFile.getName())); + + try { + optionsParser.parse( + OptionPriority.RC_FILE, projectFile.getName(), projectFile.getCommandLineFor(command)); + } catch (OptionsParsingException e) { + throw new AbruptExitException(e.getMessage(), ExitCode.COMMAND_LINE_ERROR); + } + } + } + + /** + * Returns a list of targets from the options residue. If a project file is supplied as the first + * argument, it will be ignored, on the assumption that handleProjectFiles() has been called to + * process it. + */ + public static List<String> getTargets(BlazeRuntime runtime, OptionsProvider options) { + List<String> targets = options.getResidue(); + if (runtime.getProjectFileProvider() != null && targets.size() > 0 + && targets.get(0).startsWith(PROJECT_FILE_PREFIX)) { + return targets.subList(1, targets.size()); + } + return targets; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java new file mode 100644 index 0000000..c5120cb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java
@@ -0,0 +1,173 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.query2.BlazeQueryEnvironment; +import com.google.devtools.build.lib.query2.SkyframeQueryEnvironment; +import com.google.devtools.build.lib.query2.engine.BlazeQueryEvalResult; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Setting; +import com.google.devtools.build.lib.query2.engine.QueryException; +import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.output.OutputFormatter; +import com.google.devtools.build.lib.query2.output.OutputFormatter.UnorderedFormatter; +import com.google.devtools.build.lib.query2.output.QueryOptions; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.IOException; +import java.io.PrintStream; +import java.nio.channels.ClosedByInterruptException; +import java.util.Set; + +/** + * Command line wrapper for executing a query with blaze. + */ +@Command(name = "query", + options = { PackageCacheOptions.class, + QueryOptions.class }, + help = "resource:query.txt", + shortDescription = "Executes a dependency graph query.", + allowResidue = true, + binaryStdOut = true, + canRunInOutputDirectory = true) +public final class QueryCommand implements BlazeCommand { + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { } + + /** + * Exit codes: + * 0 on successful evaluation. + * 1 if query evaluation did not complete. + * 2 if query parsing failed. + * 3 if errors were reported but evaluation produced a partial result + * (only when --keep_going is in effect.) + */ + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + QueryOptions queryOptions = options.getOptions(QueryOptions.class); + + try { + runtime.setupPackageCache( + options.getOptions(PackageCacheOptions.class), + runtime.getDefaultsPackageContent()); + } catch (InterruptedException e) { + runtime.getReporter().handle(Event.error("query interrupted")); + return ExitCode.INTERRUPTED; + } catch (AbruptExitException e) { + runtime.getReporter().handle(Event.error(null, "Unknown error: " + e.getMessage())); + return e.getExitCode(); + } + + if (options.getResidue().isEmpty()) { + runtime.getReporter().handle(Event.error( + "missing query expression. Type 'blaze help query' for syntax and help")); + return ExitCode.COMMAND_LINE_ERROR; + } + + Iterable<OutputFormatter> formatters = runtime.getQueryOutputFormatters(); + OutputFormatter formatter = + OutputFormatter.getFormatter(formatters, queryOptions.outputFormat); + if (formatter == null) { + runtime.getReporter().handle(Event.error( + String.format("Invalid output format '%s'. Valid values are: %s", + queryOptions.outputFormat, OutputFormatter.formatterNames(formatters)))); + return ExitCode.COMMAND_LINE_ERROR; + } + + String query = Joiner.on(' ').join(options.getResidue()); + + Set<Setting> settings = queryOptions.toSettings(); + BlazeQueryEnvironment env = newQueryEnvironment( + runtime, + queryOptions.keepGoing, + queryOptions.loadingPhaseThreads, + settings); + + // 1. Parse query: + QueryExpression expr; + try { + expr = QueryExpression.parse(query, env); + } catch (QueryException e) { + runtime.getReporter().handle(Event.error( + null, "Error while parsing '" + query + "': " + e.getMessage())); + return ExitCode.COMMAND_LINE_ERROR; + } + + // 2. Evaluate expression: + BlazeQueryEvalResult<Target> result; + try { + result = env.evaluateQuery(expr); + } catch (QueryException e) { + // Keep consistent with reportBuildFileError() + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.ANALYSIS_FAILURE; + } + + // 3. Output results: + OutputFormatter.UnorderedFormatter unorderedFormatter = null; + if (!queryOptions.orderResults && formatter instanceof UnorderedFormatter) { + unorderedFormatter = (UnorderedFormatter) formatter; + } + + PrintStream output = new PrintStream(runtime.getReporter().getOutErr().getOutputStream()); + try { + if (unorderedFormatter != null) { + unorderedFormatter.outputUnordered(queryOptions, result.getResultSet(), output); + } else { + formatter.output(queryOptions, result.getResultGraph(), output); + } + } catch (ClosedByInterruptException e) { + runtime.getReporter().handle(Event.error("query interrupted")); + return ExitCode.INTERRUPTED; + } catch (IOException e) { + runtime.getReporter().handle(Event.error("I/O error: " + e.getMessage())); + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } finally { + output.flush(); + } + if (result.getResultSet().isEmpty()) { + runtime.getReporter().handle(Event.info("Empty results")); + } + + return result.getSuccess() ? ExitCode.SUCCESS : ExitCode.PARTIAL_ANALYSIS_FAILURE; + } + + @VisibleForTesting // for com.google.devtools.deps.gquery.test.QueryResultTestUtil + public static BlazeQueryEnvironment newQueryEnvironment(BlazeRuntime runtime, + boolean keepGoing, int loadingPhaseThreads, Set<Setting> settings) { + ImmutableList.Builder<QueryFunction> functions = ImmutableList.builder(); + for (BlazeModule module : runtime.getBlazeModules()) { + functions.addAll(module.getQueryFunctions()); + } + return new SkyframeQueryEnvironment( + runtime.getPackageManager().newTransitiveLoader(), + runtime.getPackageManager(), + runtime.getTargetPatternEvaluator(), + keepGoing, loadingPhaseThreads, runtime.getReporter(), settings, functions.build()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java new file mode 100644 index 0000000..b128d37 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java
@@ -0,0 +1,519 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.RunUnder; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions; +import com.google.devtools.build.lib.buildtool.BuildResult; +import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils; +import com.google.devtools.build.lib.buildtool.TargetValidator; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.exec.SymlinkTreeHelper; +import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.pkgcache.LoadingFailedException; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.shell.AbnormalTerminationException; +import com.google.devtools.build.lib.shell.BadExitStatusException; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.util.CommandBuilder; +import com.google.devtools.build.lib.util.CommandDescriptionForm; +import com.google.devtools.build.lib.util.CommandFailureUtils; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Builds and run a target with the given command line arguments. + */ +@Command(name = "run", + builds = true, + options = { RunCommand.RunOptions.class }, + inherits = { BuildCommand.class }, + shortDescription = "Runs the specified target.", + help = "resource:run.txt", + allowResidue = true, + binaryStdOut = true, + binaryStdErr = true) +public class RunCommand implements BlazeCommand { + + public static class RunOptions extends OptionsBase { + @Option(name = "script_path", + category = "run", + defaultValue = "null", + converter = OptionsUtils.PathFragmentConverter.class, + help = "If set, write a shell script to the given file which invokes the " + + "target. If this option is set, the target is not run from Blaze. " + + "Use 'blaze run --script_path=foo //foo && foo' to invoke target '//foo' " + + "This differs from 'blaze run //foo' in that the Blaze lock is released " + + "and the executable is connected to the terminal's stdin.") + public PathFragment scriptPath; + } + + @VisibleForTesting + public static final String SINGLE_TARGET_MESSAGE = "Blaze can only run a single target. " + + "Do not use wildcards that match more than one target"; + @VisibleForTesting + public static final String NO_TARGET_MESSAGE = "No targets found to run"; + + private static final String PROCESS_WRAPPER = "process-wrapper"; + + // Value of --run_under as of the most recent command invocation. + private RunUnder currentRunUnder; + + private static final FileType RUNFILES_MANIFEST = FileType.of(".runfiles_manifest"); + + @VisibleForTesting // productionVisibility = Visibility.PRIVATE + protected BuildResult processRequest(final BlazeRuntime runtime, BuildRequest request) { + return runtime.getBuildTool().processRequest(request, new TargetValidator() { + @Override + public void validateTargets(Collection<Target> targets, boolean keepGoing) + throws LoadingFailedException { + RunCommand.this.validateTargets(runtime.getReporter(), targets, keepGoing); + } + }); + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { } + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + RunOptions runOptions = options.getOptions(RunOptions.class); + // This list should look like: ["//executable:target", "arg1", "arg2"] + List<String> targetAndArgs = options.getResidue(); + + // The user must at the least specify an executable target. + if (targetAndArgs.isEmpty()) { + runtime.getReporter().handle(Event.error("Must specify a target to run")); + return ExitCode.COMMAND_LINE_ERROR; + } + String targetString = targetAndArgs.get(0); + List<String> runTargetArgs = targetAndArgs.subList(1, targetAndArgs.size()); + RunUnder runUnder = options.getOptions(BuildConfiguration.Options.class).runUnder; + + OutErr outErr = runtime.getReporter().getOutErr(); + List<String> targets = (runUnder != null) && (runUnder.getLabel() != null) + ? ImmutableList.of(targetString, runUnder.getLabel().toString()) + : ImmutableList.of(targetString); + BuildRequest request = BuildRequest.create( + this.getClass().getAnnotation(Command.class).name(), options, + runtime.getStartupOptionsProvider(), targets, outErr, + runtime.getCommandId(), runtime.getCommandStartTime()); + if (request.getBuildOptions().compileOnly) { + String message = "The '" + getClass().getAnnotation(Command.class).name() + + "' command is incompatible with the --compile_only option"; + runtime.getReporter().handle(Event.error(message)); + return ExitCode.COMMAND_LINE_ERROR; + } + + currentRunUnder = runUnder; + BuildResult result; + try { + result = processRequest(runtime, request); + } finally { + currentRunUnder = null; + } + + if (!result.getSuccess()) { + runtime.getReporter().handle(Event.error("Build failed. Not running target")); + return result.getExitCondition(); + } + + // Make sure that we have exactly 1 built target (excluding --run_under), + // and that it is executable. + // These checks should only fail if keepGoing is true, because we already did + // validation before the build began. See {@link #validateTargets()}. + Collection<ConfiguredTarget> targetsBuilt = result.getSuccessfulTargets(); + ConfiguredTarget targetToRun = null; + ConfiguredTarget runUnderTarget = null; + + if (targetsBuilt != null) { + int maxTargets = runUnder != null && runUnder.getLabel() != null ? 2 : 1; + if (targetsBuilt.size() > maxTargets) { + runtime.getReporter().handle(Event.error(SINGLE_TARGET_MESSAGE)); + return ExitCode.COMMAND_LINE_ERROR; + } + for (ConfiguredTarget target : targetsBuilt) { + ExitCode targetValidation = fullyValidateTarget(runtime, target); + if (targetValidation != ExitCode.SUCCESS) { + return targetValidation; + } + if (runUnder != null && target.getLabel().equals(runUnder.getLabel())) { + if (runUnderTarget != null) { + runtime.getReporter().handle(Event.error( + null, "Can't identify the run_under target from multiple options?")); + return ExitCode.COMMAND_LINE_ERROR; + } + runUnderTarget = target; + } else if (targetToRun == null) { + targetToRun = target; + } else { + runtime.getReporter().handle(Event.error(SINGLE_TARGET_MESSAGE)); + return ExitCode.COMMAND_LINE_ERROR; + } + } + } + // Handle target & run_under referring to the same target. + if ((targetToRun == null) && (runUnderTarget != null)) { + targetToRun = runUnderTarget; + } + if (targetToRun == null) { + runtime.getReporter().handle(Event.error(NO_TARGET_MESSAGE)); + return ExitCode.COMMAND_LINE_ERROR; + } + + Path executablePath = Preconditions.checkNotNull( + targetToRun.getProvider(FilesToRunProvider.class).getExecutable().getPath()); + BuildConfiguration configuration = targetToRun.getConfiguration(); + if (configuration == null) { + // The target may be an input file, which doesn't have a configuration. In that case, we + // choose any target configuration. + configuration = runtime.getBuildTool().getView().getConfigurationCollection() + .getTargetConfigurations().get(0); + } + Path workingDir; + try { + workingDir = ensureRunfilesBuilt(runtime, targetToRun); + } catch (CommandException e) { + runtime.getReporter().handle(Event.error("Error creating runfiles: " + e.getMessage())); + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } + + List<String> args = runTargetArgs; + + FilesToRunProvider provider = targetToRun.getProvider(FilesToRunProvider.class); + RunfilesSupport runfilesSupport = provider == null ? null : provider.getRunfilesSupport(); + if (runfilesSupport != null && runfilesSupport.getArgs() != null) { + List<String> targetArgs = runfilesSupport.getArgs(); + if (!targetArgs.isEmpty()) { + args = Lists.newArrayListWithCapacity(targetArgs.size() + runTargetArgs.size()); + args.addAll(targetArgs); + args.addAll(runTargetArgs); + } + } + + // + // We now have a unique executable ready to be run. + // + // We build up two different versions of the command to run: one with an absolute path, which + // we'll actually run, and a prettier one with the long absolute path to the executable + // replaced with a shorter relative path that uses the symlinks in the workspace. + PathFragment prettyExecutablePath = + OutputDirectoryLinksUtils.getPrettyPath(executablePath, + runtime.getWorkspaceName(), runtime.getWorkspace(), + options.getOptions(BuildRequestOptions.class).symlinkPrefix); + List<String> cmdLine = new ArrayList<>(); + if (runOptions.scriptPath == null) { + cmdLine.add(runtime.getDirectories().getExecRoot() + .getRelative(runtime.getBinTools().getExecPath(PROCESS_WRAPPER)).getPathString()); + cmdLine.add("-1"); + cmdLine.add("15"); + cmdLine.add("-"); + cmdLine.add("-"); + } + List<String> prettyCmdLine = new ArrayList<>(); + // Insert the command prefix specified by the "--run_under=<command-prefix>" option + // at the start of the command line. + if (runUnder != null) { + String runUnderValue = runUnder.getValue(); + if (runUnderTarget != null) { + // --run_under specifies a target. Get the corresponding executable. + // This must be an absolute path, because the run_under target is only + // in the runfiles of test targets. + runUnderValue = runUnderTarget + .getProvider(FilesToRunProvider.class).getExecutable().getPath().getPathString(); + // If the run_under command contains any options, make sure to add them + // to the command line as well. + List<String> opts = runUnder.getOptions(); + if (!opts.isEmpty()) { + runUnderValue += " " + ShellEscaper.escapeJoinAll(opts); + } + } + cmdLine.add(configuration.getShExecutable().getPathString()); + cmdLine.add("-c"); + cmdLine.add(runUnderValue + " " + executablePath.getPathString() + " " + + ShellEscaper.escapeJoinAll(args)); + prettyCmdLine.add(configuration.getShExecutable().getPathString()); + prettyCmdLine.add("-c"); + prettyCmdLine.add(runUnderValue + " " + prettyExecutablePath.getPathString() + " " + + ShellEscaper.escapeJoinAll(args)); + } else { + cmdLine.add(executablePath.getPathString()); + cmdLine.addAll(args); + prettyCmdLine.add(prettyExecutablePath.getPathString()); + prettyCmdLine.addAll(args); + } + + // Add a newline between the blaze output and the binary's output. + outErr.printErrLn(""); + + if (runOptions.scriptPath != null) { + String unisolatedCommand = CommandFailureUtils.describeCommand( + CommandDescriptionForm.COMPLETE_UNISOLATED, + cmdLine, null, workingDir.getPathString()); + if (writeScript(runtime, runOptions.scriptPath, unisolatedCommand)) { + return ExitCode.SUCCESS; + } else { + return ExitCode.RUN_FAILURE; + } + } + + runtime.getReporter().handle(Event.info( + null, "Running command line: " + ShellEscaper.escapeJoinAll(prettyCmdLine))); + + com.google.devtools.build.lib.shell.Command command = new CommandBuilder() + .addArgs(cmdLine).setEnv(runtime.getClientEnv()).setWorkingDir(workingDir).build(); + + try { + // The command API is a little strange in that the following statement + // will return normally only if the program exits with exit code 0. + // If it ends with any other code, we have to catch BadExitStatusException. + command.execute(com.google.devtools.build.lib.shell.Command.NO_INPUT, + com.google.devtools.build.lib.shell.Command.NO_OBSERVER, + outErr.getOutputStream(), + outErr.getErrorStream(), + true /* interruptible */).getTerminationStatus().getExitCode(); + return ExitCode.SUCCESS; + } catch (BadExitStatusException e) { + String message = "Non-zero return code '" + + e.getResult().getTerminationStatus().getExitCode() + + "' from command: " + e.getMessage(); + runtime.getReporter().handle(Event.error(message)); + return ExitCode.RUN_FAILURE; + } catch (AbnormalTerminationException e) { + // The process was likely terminated by a signal in this case. + return ExitCode.INTERRUPTED; + } catch (CommandException e) { + runtime.getReporter().handle(Event.error("Error running program: " + e.getMessage())); + return ExitCode.RUN_FAILURE; + } + } + + /** + * Ensures that runfiles are built for the specified target. If they already + * are, does nothing, otherwise builds them. + * + * @param target the target to build runfiles for. + * @return the path of the runfiles directory. + * @throws CommandException + */ + private Path ensureRunfilesBuilt(BlazeRuntime runtime, ConfiguredTarget target) + throws CommandException { + FilesToRunProvider provider = target.getProvider(FilesToRunProvider.class); + RunfilesSupport runfilesSupport = provider == null ? null : provider.getRunfilesSupport(); + if (runfilesSupport == null) { + return runtime.getWorkingDirectory(); + } + + Artifact manifest = runfilesSupport.getRunfilesManifest(); + PathFragment runfilesDir = runfilesSupport.getRunfilesDirectoryExecPath(); + Path workingDir = runtime.getExecRoot() + .getRelative(runfilesDir) + .getRelative(runtime.getRunfilesPrefix()); + + // When runfiles are not generated, getManifest() returns the + // .runfiles_manifest file, otherwise it returns the MANIFEST file. This is + // a handy way to check whether runfiles were built or not. + if (!RUNFILES_MANIFEST.matches(manifest.getFilename())) { + // Runfiles already built, nothing to do. + return workingDir; + } + + SymlinkTreeHelper helper = new SymlinkTreeHelper( + manifest.getExecPath(), + runfilesDir, + false); + helper.createSymlinksUsingCommand(runtime.getExecRoot(), target.getConfiguration(), + runtime.getBinTools()); + return workingDir; + } + + private boolean writeScript(BlazeRuntime runtime, PathFragment scriptPathFrag, String cmd) { + final String SH_SHEBANG = "#!/bin/sh"; + Path scriptPath = runtime.getWorkingDirectory().getRelative(scriptPathFrag); + try { + FileSystemUtils.writeContent(scriptPath, StandardCharsets.ISO_8859_1, + SH_SHEBANG + "\n" + cmd + " \"$@\""); + scriptPath.setExecutable(true); + } catch (IOException e) { + runtime.getReporter().handle(Event.error("Error writing run script:" + e.getMessage())); + return false; + } + return true; + } + + // Make sure we are building exactly 1 binary target. + // If keepGoing, we'll build all the targets even if they are non-binary. + private void validateTargets(Reporter reporter, Collection<Target> targets, boolean keepGoing) + throws LoadingFailedException { + Target targetToRun = null; + Target runUnderTarget = null; + + boolean singleTargetWarningWasOutput = false; + int maxTargets = currentRunUnder != null && currentRunUnder.getLabel() != null ? 2 : 1; + if (targets.size() > maxTargets) { + warningOrException(reporter, SINGLE_TARGET_MESSAGE, keepGoing); + singleTargetWarningWasOutput = true; + } + for (Target target : targets) { + String targetError = validateTarget(target); + if (targetError != null) { + warningOrException(reporter, targetError, keepGoing); + } + + if (currentRunUnder != null && target.getLabel().equals(currentRunUnder.getLabel())) { + // It's impossible to have two targets with the same label. + Preconditions.checkState(runUnderTarget == null); + runUnderTarget = target; + } else if (targetToRun == null) { + targetToRun = target; + } else { + if (!singleTargetWarningWasOutput) { + warningOrException(reporter, SINGLE_TARGET_MESSAGE, keepGoing); + } + return; + } + } + // Handle target & run_under referring to the same target. + if ((targetToRun == null) && (runUnderTarget != null)) { + targetToRun = runUnderTarget; + } + if (targetToRun == null) { + warningOrException(reporter, NO_TARGET_MESSAGE, keepGoing); + } + } + + // If keepGoing, print a warning and return the given collection. + // Otherwise, throw InvalidTargetException. + private void warningOrException(Reporter reporter, String message, + boolean keepGoing) throws LoadingFailedException { + if (keepGoing) { + reporter.handle(Event.warn(message + ". Will continue anyway")); + } else { + throw new LoadingFailedException(message); + } + } + + private static String notExecutableError(Target target) { + return "Cannot run target " + target.getLabel() + ": Not executable"; + } + + /** Returns null if the target is a runnable rule, or an appropriate error message otherwise. */ + private static String validateTarget(Target target) { + return isExecutable(target) + ? null + : notExecutableError(target); + } + + /** + * Performs all available validation checks on an individual target. + * + * @param target ConfiguredTarget to validate + * @return ExitCode.SUCCESS if all checks succeeded, otherwise a different error code. + */ + private ExitCode fullyValidateTarget(BlazeRuntime runtime, ConfiguredTarget target) { + String targetError = validateTarget(target.getTarget()); + + if (targetError != null) { + runtime.getReporter().handle(Event.error(targetError)); + return ExitCode.COMMAND_LINE_ERROR; + } + + Artifact executable = target.getProvider(FilesToRunProvider.class).getExecutable(); + if (executable == null) { + runtime.getReporter().handle(Event.error(notExecutableError(target.getTarget()))); + return ExitCode.COMMAND_LINE_ERROR; + } + + // Shouldn't happen: We just validated the target. + Preconditions.checkState(executable != null, + "Could not find executable for target %s", target); + Path executablePath = executable.getPath(); + try { + if (!executablePath.exists() || !executablePath.isExecutable()) { + runtime.getReporter().handle(Event.error( + null, "Non-existent or non-executable " + executablePath)); + return ExitCode.BLAZE_INTERNAL_ERROR; + } + } catch (IOException e) { + runtime.getReporter().handle(Event.error( + "Error checking " + executablePath.getPathString() + ": " + e.getMessage())); + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } + + return ExitCode.SUCCESS; + } + + /** + * Return true iff {@code target} is a rule that has an executable file. This includes + * *_test rules, *_binary rules, and generated outputs. + */ + private static boolean isExecutable(Target target) { + return isOutputFile(target) || isExecutableNonTestRule(target) + || TargetUtils.isTestRule(target); + } + + /** + * Return true iff {@code target} is a rule that generates an executable file and is user-executed + * code. + */ + private static boolean isExecutableNonTestRule(Target target) { + if (!(target instanceof Rule)) { + return false; + } + Rule rule = ((Rule) target); + if (rule.getRuleClassObject().hasAttr("$is_executable", Type.BOOLEAN)) { + return NonconfigurableAttributeMapper.of(rule).get("$is_executable", Type.BOOLEAN); + } + return false; + } + + private static boolean isOutputFile(Target target) { + return (target instanceof OutputFile); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java new file mode 100644 index 0000000..fb9ba39 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java
@@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +/** + * The 'blaze shutdown' command. + */ +@Command(name = "shutdown", + options = { ShutdownCommand.Options.class }, + allowResidue = false, + mustRunInWorkspace = false, + shortDescription = "Stops the Blaze server.", + help = "This command shuts down the memory resident Blaze server process.\n%{options}") +public final class ShutdownCommand implements BlazeCommand { + + public static class Options extends OptionsBase { + + @Option(name="iff_heap_size_greater_than", + defaultValue = "0", + category = "misc", + help="Iff non-zero, then shutdown will only shut down the " + + "server if the total memory (in MB) consumed by the JVM " + + "exceeds this value.") + public int heapSizeLimit; + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) + throws ShutdownBlazeServerException { + + int limit = options.getOptions(Options.class).heapSizeLimit; + + // Iff limit is non-zero, shut down the server if total memory exceeds the + // limit. totalMemory is the actual heap size that the VM currently uses + // *from the OS perspective*. That is, it's not the size occupied by all + // objects (which is totalMemory() - freeMemory()), and not the -Xmx + // (which is maxMemory()). It's really how much memory this process + // currently consumes, in addition to the JVM code and C heap. + + if (limit == 0 || + Runtime.getRuntime().totalMemory() > limit * 1000L * 1000) { + throw new ShutdownBlazeServerException(0); + } + return ExitCode.SUCCESS; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java new file mode 100644 index 0000000..70082ef --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java
@@ -0,0 +1,82 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.docgen.SkylarkDocumentationProcessor; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.Map; + +/** + * The 'doc_ext' command, which prints the extension API doc. + */ +@Command(name = "doc_ext", +allowResidue = true, +mustRunInWorkspace = false, +shortDescription = "Prints help for commands, or the index.", +help = "resource:skylark.txt") +public final class SkylarkCommand implements BlazeCommand { + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) + throws ShutdownBlazeServerException { + OutErr outErr = runtime.getReporter().getOutErr(); + if (options.getResidue().isEmpty()) { + printTopLevelAPIDoc(outErr); + return ExitCode.SUCCESS; + } + if (options.getResidue().size() != 1) { + runtime.getReporter().handle(Event.error("Cannot specify more than one parameters")); + return ExitCode.COMMAND_LINE_ERROR; + } + return printAPIDoc(options.getResidue().get(0), outErr, runtime.getReporter()); + } + + private ExitCode printAPIDoc(String param, OutErr outErr, Reporter reporter) { + String params[] = param.split("\\."); + if (params.length > 2) { + reporter.handle(Event.error("Identifier not found: " + param)); + return ExitCode.COMMAND_LINE_ERROR; + } + SkylarkDocumentationProcessor processor = new SkylarkDocumentationProcessor(); + String doc = processor.getCommandLineAPIDoc(params); + if (doc == null) { + reporter.handle(Event.error("Identifier not found: " + param)); + return ExitCode.COMMAND_LINE_ERROR; + } + outErr.printOut(doc); + return ExitCode.SUCCESS; + } + + private void printTopLevelAPIDoc(OutErr outErr) { + SkylarkDocumentationProcessor processor = new SkylarkDocumentationProcessor(); + outErr.printOut("Top level language modules, methods and objects:\n\n"); + for (Map.Entry<String, String> entry : processor.collectTopLevelModules().entrySet()) { + outErr.printOut(entry.getKey() + ": " + entry.getValue()); + } + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java new file mode 100644 index 0000000..561c54a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java
@@ -0,0 +1,161 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.BuildResult; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.rules.test.TestStrategy; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat; +import com.google.devtools.build.lib.runtime.AggregatingTestListener; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandEventHandler; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier; +import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions; +import com.google.devtools.build.lib.runtime.TestResultAnalyzer; +import com.google.devtools.build.lib.runtime.TestResultNotifier; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter; +import com.google.devtools.common.options.OptionPriority; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.Collection; +import java.util.List; + +/** + * Handles the 'test' command on the Blaze command line. + */ +@Command(name = "test", + builds = true, + inherits = { BuildCommand.class }, + options = { TestSummaryOptions.class }, + shortDescription = "Builds and runs the specified test targets.", + help = "resource:test.txt", + allowResidue = true) +public class TestCommand implements BlazeCommand { + private AnsiTerminalPrinter printer; + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) + throws AbruptExitException { + ProjectFileSupport.handleProjectFiles(runtime, optionsParser, "test"); + + TestOutputFormat testOutput = optionsParser.getOptions(ExecutionOptions.class).testOutput; + + if (testOutput == TestStrategy.TestOutputFormat.STREAMED) { + runtime.getReporter().handle(Event.warn( + "Streamed test output requested so all tests will be run locally, without sharding, " + + "one at a time")); + try { + optionsParser.parse(OptionPriority.SOFTWARE_REQUIREMENT, + "streamed output requires locally run tests, without sharding", + ImmutableList.of("--test_sharding_strategy=disabled", "--test_strategy=exclusive")); + } catch (OptionsParsingException e) { + throw new IllegalStateException("Known options failed to parse", e); + } + } + } + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + TestResultAnalyzer resultAnalyzer = new TestResultAnalyzer( + runtime.getExecRoot(), + options.getOptions(TestSummaryOptions.class), + options.getOptions(ExecutionOptions.class), + runtime.getEventBus()); + + printer = new AnsiTerminalPrinter(runtime.getReporter().getOutErr().getOutputStream(), + options.getOptions(BlazeCommandEventHandler.Options.class).useColor()); + + // Initialize test handler. + AggregatingTestListener testListener = new AggregatingTestListener( + resultAnalyzer, runtime.getEventBus(), runtime.getReporter()); + + runtime.getEventBus().register(testListener); + return doTest(runtime, options, testListener); + } + + private ExitCode doTest(BlazeRuntime runtime, + OptionsProvider options, + AggregatingTestListener testListener) { + // Run simultaneous build and test. + List<String> targets = ProjectFileSupport.getTargets(runtime, options); + BuildRequest request = BuildRequest.create( + getClass().getAnnotation(Command.class).name(), options, + runtime.getStartupOptionsProvider(), targets, + runtime.getReporter().getOutErr(), runtime.getCommandId(), runtime.getCommandStartTime()); + if (request.getBuildOptions().compileOnly) { + String message = "The '" + getClass().getAnnotation(Command.class).name() + + "' command is incompatible with the --compile_only option"; + runtime.getReporter().handle(Event.error(message)); + return ExitCode.COMMAND_LINE_ERROR; + } + request.setRunTests(); + + BuildResult buildResult = runtime.getBuildTool().processRequest(request, null); + + Collection<ConfiguredTarget> testTargets = buildResult.getTestTargets(); + // TODO(bazel-team): don't handle isEmpty here or fix up a bunch of tests + if (buildResult.getSuccessfulTargets() == null) { + // This can happen if there were errors in the target parsing or loading phase + // (original exitcode=BUILD_FAILURE) or if there weren't but --noanalyze was given + // (original exitcode=SUCCESS). + runtime.getReporter().handle(Event.error("Couldn't start the build. Unable to run tests")); + return buildResult.getSuccess() ? ExitCode.PARSING_FAILURE : buildResult.getExitCondition(); + } + // TODO(bazel-team): the check above shadows NO_TESTS_FOUND, but switching the conditions breaks + // more tests + if (testTargets.isEmpty()) { + runtime.getReporter().handle(Event.error( + null, "No test targets were found, yet testing was requested")); + return buildResult.getSuccess() ? ExitCode.NO_TESTS_FOUND : buildResult.getExitCondition(); + } + + boolean buildSuccess = buildResult.getSuccess(); + boolean testSuccess = analyzeTestResults(testTargets, testListener, options); + + if (testSuccess && !buildSuccess) { + // If all tests run successfully, test summary should include warning if + // there were build errors not associated with the test targets. + printer.printLn(AnsiTerminalPrinter.Mode.ERROR + + "One or more non-test targets failed to build.\n" + + AnsiTerminalPrinter.Mode.DEFAULT); + } + + return buildSuccess ? + (testSuccess ? ExitCode.SUCCESS : ExitCode.TESTS_FAILED) + : buildResult.getExitCondition(); + } + + /** + * Analyzes test results and prints summary information. + * Returns true if and only if all tests were successful. + */ + private boolean analyzeTestResults(Collection<ConfiguredTarget> testTargets, + AggregatingTestListener listener, + OptionsProvider options) { + TestResultNotifier notifier = new TerminalTestResultNotifier(printer, options); + return listener.getAnalyzer().differentialAnalyzeAndReport( + testTargets, listener, notifier); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java new file mode 100644 index 0000000..0804cf6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +/** + * The 'blaze version' command, which informs users about the blaze version + * information. + */ +@Command(name = "version", + options = {}, + allowResidue = false, + mustRunInWorkspace = false, + help = "resource:version.txt", + shortDescription = "Prints version information for Blaze.") +public final class VersionCommand implements BlazeCommand { + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + BlazeVersionInfo info = BlazeVersionInfo.instance(); + if (info.getSummary() == null) { + runtime.getReporter().handle(Event.error("Version information not available")); + return ExitCode.COMMAND_LINE_ERROR; + } + runtime.getReporter().getOutErr().printOutLn(info.getSummary()); + return ExitCode.SUCCESS; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt new file mode 100644 index 0000000..0ef55a8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt
@@ -0,0 +1,14 @@ + +Usage: blaze %{command} <options> <profile-files> [<profile-file> ...] + +Analyzes build profile data for the given profile data files. + +Analyzes each specified profile data file and prints the results. The +input files must have been produced by the 'blaze build +--profile=file' command. + +By default, a summary of the analysis is printed. For post-processing +with scripts, the --dump=raw option is recommended, causing this +command to dump profile data in easily-parsed format. + +%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt new file mode 100644 index 0000000..5e8d88a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt
@@ -0,0 +1,10 @@ + +Usage: blaze %{command} <options> <targets> + +Builds the specified targets, using the options. + +See 'blaze help target-syntax' for details and examples on how to +specify targets to build. + +%{options} +
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt new file mode 100644 index 0000000..11541ff --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt
@@ -0,0 +1,8 @@ + +Usage: blaze canonicalize-flags <options> -- <options-to-canonicalize> + +Canonicalizes Blaze flags for the test and build commands. This command is +intended to be used for tools that wish to check if two lists of options have +the same effect at runtime. + +%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt new file mode 100644 index 0000000..7633888 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt
@@ -0,0 +1,10 @@ + +Usage: blaze %{command} [<option> ...] + +Removes Blaze-created output, including all object files, and Blaze +metadata. + +If '--expunge' is specified, the entire working tree will be removed +and the server stopped. + +%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt new file mode 100644 index 0000000..a2040c8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt
@@ -0,0 +1,7 @@ + +Usage: blaze help [<command>] + +Prints a help page for the given command, or, if no command is +specified, prints the index of available commands. + +%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt new file mode 100644 index 0000000..9c8b552 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt
@@ -0,0 +1,23 @@ + +Usage: blaze info <options> [key] + +Displays information about the state of the blaze process in the +form of several "key: value" pairs. This includes the locations of +several output directories. Because some of the +values are affected by the options passed to 'blaze build', the +info command accepts the same set of options. + +A single non-option argument may be specified (e.g. "blaze-bin"), in +which case only the value for that key will be printed. + +If --show_make_env is specified, the output includes the set of key/value +pairs in the "Make" environment, accessible within BUILD files. + +The full list of keys and the meaning of their values is documented in +the Blaze User Manual, and can be programmatically obtained with +'blaze help info-keys'. + +See also 'blaze version' for more detailed blaze version +information. + +%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt new file mode 100644 index 0000000..ce10211 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt
@@ -0,0 +1,19 @@ + +Usage: blaze %{command} <options> <query-expression> + +Executes a query language expression over a specified subgraph of the +build dependency graph. + +For example, to show all C++ test rules in the strings package, use: + + % blaze query 'kind("cc_.*test", strings:*)' + +or to find all dependencies of chubby lockserver, use: + + % blaze query 'deps(//path/to/package:target)' + +or to find a dependency path between //path/to/package:target and //dependency: + + % blaze query 'somepath(//path/to/package:target, //dependency)' + +%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt new file mode 100644 index 0000000..57283d5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt
@@ -0,0 +1,12 @@ + +Usage: blaze %{command} <options> -- <binary target> <flags to binary> + +Build the specified target and run it with the given arguments. + +'run' accepts any 'build' options, and will inherit any defaults +provided by .blazerc. + +If your script needs stdin or execution not constrained by the Blaze lock, +use 'blaze run --script_path' to write a script and then execute it. + +%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt new file mode 100644 index 0000000..5414707 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt
@@ -0,0 +1,14 @@ + +Startup options +=============== + +These options affect how Blaze starts up, or more specifically, how +the virtual machine hosting Blaze starts up, and how the Blaze server +starts up. These options must be specified to the left of the Blaze +command (e.g. 'build'), and they must not contain any space between +option name and value. + +Example: + % blaze --host_jvm_args=-Xmx1400m --output_base=/tmp/foo build //base + +%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt new file mode 100644 index 0000000..1fac498 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt
@@ -0,0 +1,64 @@ + +Target pattern syntax +===================== + +The BUILD file label syntax is used to specify a single target. Target +patterns generalize this syntax to sets of targets, and also support +working-directory-relative forms, recursion, subtraction and filtering. +Examples: + +Specifying a single target: + + //foo/bar:wiz The single target '//foo/bar:wiz'. + foo/bar/wiz Equivalent to the first existing one of these: + //foo/bar:wiz + //foo:bar/wiz + //foo/bar Equivalent to '//foo/bar:bar'. + +Specifying all rules in a package: + + //foo/bar:all Matches all rules in package 'foo/bar'. + +Specifying all rules recursively beneath a package: + + //foo/...:all Matches all rules in all packages beneath directory 'foo'. + //foo/... (ditto) + +Working-directory relative forms: (assume cwd = 'workspace/foo') + + Target patterns which do not begin with '//' are taken relative to + the working directory. Patterns which begin with '//' are always + absolute. + + ...:all Equivalent to '//foo/...:all'. + ... (ditto) + + bar/...:all Equivalent to '//foo/bar/...:all'. + bar/... (ditto) + + bar:wiz Equivalent to '//foo/bar:wiz'. + :foo Equivalent to '//foo:foo'. + + bar:all Equivalent to '//foo/bar:all'. + :all Equivalent to '//foo:all'. + +Summary of target wildcards: + + :all, Match all rules in the specified packages. + :*, :all-targets Match all targets (rules and files) in the specified + packages, including .par and _deploy.jar files. + +Subtractive patterns: + + Target patterns may be preceded by '-', meaning they should be + subtracted from the set of targets accumulated by preceding + patterns. For example: + + % blaze build -- foo/... -foo/contrib/... + + builds everything in 'foo', except 'contrib'. In case a target not + under 'contrib' depends on something under 'contrib' though, in order to + build the former blaze has to build the latter too. As usual, the '--' is + required to prevent '-b' from being interpreted as an option. + +%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt new file mode 100644 index 0000000..a1f0523 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt
@@ -0,0 +1,15 @@ + +Usage: blaze %{command} <options> <test-targets> + +Builds the specified targets and runs all test targets among them (test targets +might also need to satisfy provided tag, size or language filters) using +the specified options. + +This command accepts all valid options to 'build', and inherits +defaults for 'build' from your .blazerc. If you don't use .blazerc, +don't forget to pass all your 'build' options to '%{command}' too. + +See 'blaze help target-syntax' for details and examples on how to +specify targets. + +%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt new file mode 100644 index 0000000..10e1df7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt
@@ -0,0 +1,3 @@ +Prints the version information that was embedded when blaze was built. + +%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/server/IdleServerTasks.java b/src/main/java/com/google/devtools/build/lib/server/IdleServerTasks.java new file mode 100644 index 0000000..ad3e475 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/server/IdleServerTasks.java
@@ -0,0 +1,158 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.server; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.ProcMeminfoParser; +import com.google.devtools.build.lib.vfs.FileStatus; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Symlinks; + +import java.io.IOException; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +/** + * Run cleanup-related tasks during idle periods in the server. + * idle() and busy() must be called in that order, and only once. + */ +class IdleServerTasks { + + private final Path workspaceDir; + private final ScheduledThreadPoolExecutor executor; + private static final Logger LOG = Logger.getLogger(IdleServerTasks.class.getName()); + + private static final long FIVE_MIN_MILLIS = 1000 * 60 * 5; + + /** + * Must be called from the main thread. + */ + public IdleServerTasks(@Nullable Path workspaceDir) { + this.executor = new ScheduledThreadPoolExecutor(1); + this.workspaceDir = workspaceDir; + } + + /** + * Called when the server becomes idle. Should not block, but may invoke + * new threads. + */ + public void idle() { + Preconditions.checkState(!executor.isShutdown()); + + // Do a GC cycle while the server is idle. + executor.schedule(new Runnable() { + @Override public void run() { + long before = System.currentTimeMillis(); + System.gc(); + LOG.info("Idle GC: " + (System.currentTimeMillis() - before) + "ms"); + } + }, 10, TimeUnit.SECONDS); + } + + /** + * Called by the main thread when the server gets to work. + * Should return quickly. + */ + public void busy() { + Preconditions.checkState(!executor.isShutdown()); + + // Make sure tasks are finished after shutdown(), so they do not intefere + // with subsequent server invocations. + executor.shutdown(); + executor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); + executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + + boolean interrupted = false; + while (true) { + try { + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.HOURS); + break; + } catch (InterruptedException e) { + // It's unsafe to leak threads - just reset the interrupt bit later. + interrupted = true; + } + } + + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + /** + * Return true iff the server should continue processing requests. + * Called from the main thread, so it should return quickly. + */ + public boolean continueProcessing(long idleMillis) { + if (!memoryHeuristic(idleMillis)) { + return false; + } + if (workspaceDir == null) { + return false; + } + + FileStatus stat; + try { + stat = workspaceDir.statIfFound(Symlinks.FOLLOW); + } catch (IOException e) { + // Do not terminate the server if the workspace is temporarily inaccessible, for example, + // if it is on a network filesystem and the connection is down. + return true; + } + return stat != null && stat.isDirectory(); + } + + private boolean memoryHeuristic(long idleMillis) { + if (idleMillis < FIVE_MIN_MILLIS) { + // Don't check memory health until after five minutes. + return true; + } + + ProcMeminfoParser memInfo = null; + try { + memInfo = new ProcMeminfoParser(); + } catch (IOException e) { + LOG.info("Could not process /proc/meminfo: " + e); + return true; + } + + long totalPhysical, totalFree; + try { + totalPhysical = memInfo.getTotalKb(); + totalFree = memInfo.getFreeRamKb(); // See method javadoc. + } catch (IllegalArgumentException e) { + // Ugly capture of unchecked exception, similar to that in + // LocalHostCapacity. + LoggingUtil.logToRemote(Level.WARNING, + "Could not read memInfo during idle query", e); + return true; + } + double fractionFree = (double) totalFree / totalPhysical; + + // If the system as a whole is low on memory, let this server die. + if (fractionFree < .1) { + LOG.info("Terminating due to memory constraints"); + LOG.info(String.format("Total physical:%d\nTotal free: %d\n", + totalPhysical, totalFree)); + return false; + } + + return true; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/server/RPCServer.java b/src/main/java/com/google/devtools/build/lib/server/RPCServer.java new file mode 100644 index 0000000..a1e9982 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/server/RPCServer.java
@@ -0,0 +1,562 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.server; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.server.RPCService.UnknownCommandException; +import com.google.devtools.build.lib.server.signal.InterruptSignalHandler; +import com.google.devtools.build.lib.unix.FilesystemUtils; +import com.google.devtools.build.lib.unix.LocalClientSocket; +import com.google.devtools.build.lib.unix.LocalServerSocket; +import com.google.devtools.build.lib.unix.LocalSocketAddress; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.ThreadUtils; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.util.io.StreamMultiplexer; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Logger; + +/** + * An RPCServer server is a Java object that sits and waits for RPC requests + * (the sit-and-wait is implemented in {@link #serve()}). These requests + * arrive via UNIX file sockets. The RPCServer then calls the application + * (which implements ServerCommand) to handle the request. (Since the Blaze + * server may need to stat hundreds of directories during initialization, this + * is a significant speedup.) The server thread will terminate after idling + * for a user-specified time. + * + * Note: If you are contemplating to call into the RPCServer from + * within Java, consider using the {@link RPCService} class instead. + */ +// TODO(bazel-team): Signal handling. +// TODO(bazel-team): Gives clients status information when the server is busy. One +// way to do this is to put the server status in a file (pid, the current +// target, etc) in the server directory. Alternatively, we can have a separate +// thread taking care of the server socket and put the information into socket +// handshakes. +// TODO(bazel-team): Use Reporter for server-side messages. +public final class RPCServer { + + private final Clock clock; + private final RPCService rpcService; + private final LocalServerSocket serverSocket; + private final long maxIdleMillis; + private final long statusCheckMillis; + private final Path serverDirectory; + private final Path workspaceDir; + private static final Logger LOG = Logger.getLogger(RPCServer.class.getName()); + private volatile boolean lameDuck; + + private static final long STATUS_CHECK_PERIOD_MILLIS = 1000 * 60; // 1 minute. + private static final Splitter NULLTERMINATOR_SPLITTER = Splitter.on('\0'); + + /** + * Create a new server instance. After creating the server, you can start it + * by calling the {@link #serve()} method. + * + * @param clock The clock to take time measurements + * @param rpcService The underlying service object, which takes + * care of dispatching to the {@link ServerCommand} + * instances, as requests arrive. + * @param maxIdleMillis The maximum time the server will wait idly. + * @param statusCheckPeriodMillis How long to wait between system status checks. + * @param serverDirectory Directory to put file socket and pid files, etc. + * @param workspaceDir The workspace. Used solely to ensure it persists. + * @throws IOException + */ + public RPCServer(Clock clock, RPCService rpcService, + long maxIdleMillis, long statusCheckPeriodMillis, + Path serverDirectory, Path workspaceDir) + throws IOException { + this.clock = clock; + this.rpcService = rpcService; + this.maxIdleMillis = maxIdleMillis; + this.statusCheckMillis = statusCheckPeriodMillis; + this.serverDirectory = serverDirectory; + this.workspaceDir = workspaceDir; + + this.serverSocket = openServerSocket(); + serverSocket.setSoTimeout(Math.min(maxIdleMillis, statusCheckMillis)); + lameDuck = false; + } + + /** + * Create a new server instance. After creating the server, you can start it + * by calling the {@link #serve()} method. + * + * @param clock The clock to take time measurements + * @param rpcService The underlying service object, which takes + * care of dispatching to the {@link ServerCommand} + * instances, as requests arrive. + * @param maxIdleMillis The maximum time the server will wait idly. + * @param serverDirectory Directory to put file socket and pid files, etc. + * @param workspaceDir The workspace. Used solely to ensure it persists. + * @throws IOException + */ + public RPCServer(Clock clock, RPCService rpcService, + long maxIdleMillis, Path serverDirectory, Path workspaceDir) + throws IOException { + this(clock, rpcService, maxIdleMillis, STATUS_CHECK_PERIOD_MILLIS, + serverDirectory, workspaceDir); + } + + private static void printStack(IOException e) { + /* + * Hopefully this never happens. It's not very nice to just write this + * to the user's console, but I'm not sure what better choice we have. + */ + StringWriter err = new StringWriter(); + PrintWriter printErr = new PrintWriter(err); + printErr.println("=======[BLAZE SERVER: ENCOUNTERED IO EXCEPTION]======="); + e.printStackTrace(printErr); + printErr.println("====================================================="); + LOG.severe(err.toString()); + } + + /** + * Wait on a socket for business (answer requests). Note that this + * method won't return until the server shuts down. + */ + public void serve() { + // Register the signal handler. + final AtomicBoolean inAction = new AtomicBoolean(false); + final AtomicBoolean allowingInterrupt = new AtomicBoolean(true); + final AtomicLong cmdNum = new AtomicLong(); + final Thread mainThread = Thread.currentThread(); + final Object interruptLock = new Object(); + + InterruptSignalHandler sigintHandler = new InterruptSignalHandler() { + @Override + public void run() { + LOG.severe("User interrupt"); + + // Only interrupt during actions - otherwise we may end up setting the interrupt bit + // at the end of a build and responding to it at the beginning of the subsequent build. + synchronized (interruptLock) { + if (allowingInterrupt.get()) { + mainThread.interrupt(); + } + } + + Runnable interruptWatcher = new Runnable() { + @Override + public void run() { + try { + long originalCmd = cmdNum.get(); + Thread.sleep(10 * 1000); + if (inAction.get() && cmdNum.get() == originalCmd) { + // We're still operating on the same command. + // Interrupt took too long. + ThreadUtils.warnAboutSlowInterrupt(); + } + } catch (InterruptedException e) { + // Ignore. + } + } + }; + + if (inAction.get()) { + Thread interruptWatcherThread = + new Thread(interruptWatcher, "interrupt-watcher-" + cmdNum); + interruptWatcherThread.setDaemon(true); + interruptWatcherThread.start(); + } + } + }; + + try { + while (!lameDuck) { + try { + IdleServerTasks idleChecker = new IdleServerTasks(workspaceDir); + idleChecker.idle(); + RequestIo requestIo; + + long startTime = clock.currentTimeMillis(); + while (true) { + try { + allowingInterrupt.set(true); + Socket socket = serverSocket.accept(); + long firstContactTime = clock.currentTimeMillis(); + requestIo = new RequestIo(socket, firstContactTime); + break; + } catch (SocketTimeoutException e) { + long idleTime = clock.currentTimeMillis() - startTime; + if (lameDuck) { + closeServerSocket(); + return; + } else if (idleTime > maxIdleMillis || + (idleTime > statusCheckMillis && !idleChecker.continueProcessing(idleTime))) { + enterLameDuck(); + } + } + } + idleChecker.busy(); + + try { + cmdNum.incrementAndGet(); + inAction.set(true); + executeRequest(requestIo); + } finally { + inAction.set(false); + synchronized (interruptLock) { + allowingInterrupt.set(false); + Thread.interrupted(); // clears thread interrupted status + } + requestIo.shutdown(); + if (rpcService.isShutdown()) { + return; + } + } + } catch (IOException e) { + if (e.getMessage().equals("Broken pipe")) { + LOG.info("Connection to the client lost: " + + e.getMessage()); + } else { + // Other cases: print the stack for debugging. + printStack(e); + } + } + } + } finally { + rpcService.shutdown(); + LOG.info("Logging finished"); + sigintHandler.uninstall(); + } + } + + private void closeServerSocket() { + LOG.info("Closing serverSocket."); + try { + serverSocket.close(); + } catch (IOException e) { + printStack(e); + } + + if (!lameDuck) { + try { + getSocketPath().delete(); + } catch (IOException e) { + printStack(e); + } + } + } + + /** + * Allow one last request to be serviced. + */ + private void enterLameDuck() { + lameDuck = true; + try { + getSocketPath().delete(); + } catch (IOException e) { + e.printStackTrace(); + } + serverSocket.setSoTimeout(1); + } + + /** + * Returns the path of the socket file to be used. + */ + public Path getSocketPath() { + return serverDirectory.getRelative("server.socket"); + } + + /** + * Ensures no other server is running for the current socket file. This + * guarantees that no two servers are running against the same output + * directory. + * + * @throws IOException if another server holds the lock for the socket file. + */ + public static void ensureExclusiveAccess(Path socketFile) throws IOException { + LocalSocketAddress address = + new LocalSocketAddress(socketFile.getPathFile()); + if (socketFile.exists()) { + try { + new LocalClientSocket(address).close(); + } catch (IOException e) { + // The previous server process is dead--unlink the file: + socketFile.delete(); + return; + } + // TODO(bazel-team): (2009) Read the previous server's pid from the "hello" message + // and add it to the message. + throw new IOException("Socket file " + socketFile.getPathString() + + " is locked by another server"); + } + } + + /** + * Schedule the specified file for (attempted) deletion at JVM exit. + */ + private static void deleteAtExit(final Path socketFile, final boolean deleteParent) { + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + try { + socketFile.delete(); + if (deleteParent) { + socketFile.getParentDirectory().delete(); + } + } catch (IOException e) { + printStack(e); + } + } + }); + } + + /** + * Opens a UNIX local server socket. + * @throws IOException if the socket file is used by another server or can + * not be made exclusive. + */ + private LocalServerSocket openServerSocket() throws IOException { + // This is the "well known" socket path via which the server is found... + Path socketFile = getSocketPath(); + + // ...but it may have a name that's too long for AF_UNIX, in which case we + // make it a symlink to /tmp/something. This typically only happens in + // tests where the --output_base is beneath a very deep temp dir. + // (All this extra complexity is just used in tests... *sigh*). + if (socketFile.toString().length() >= 108) { // = UNIX_PATH_MAX + Path socketLink = socketFile; + String tmpDir = System.getProperty("blaze.rpcserver.tmpdir", "/tmp"); + socketFile = createTempSocketDirectory(socketFile.getRelative(tmpDir)). + getRelative("server.socket"); + LOG.info("Using symlinked socket at " + socketFile); + + socketLink.delete(); // Remove stale symlink, if any. + socketLink.createSymbolicLink(socketFile); + + deleteAtExit(socketLink, /*deleteParent=*/false); + deleteAtExit(socketFile, /*deleteParent=*/true); + } else { + deleteAtExit(socketFile, /*deleteParent=*/false); + } + + ensureExclusiveAccess(socketFile); + + LocalServerSocket serverSocket = new LocalServerSocket(); + serverSocket.bind(new LocalSocketAddress(socketFile.getPathFile())); + FilesystemUtils.chmod(socketFile.getPathFile(), 0600); // Lock it down. + serverSocket.listen(/*backlog=*/50); + return serverSocket; + } + + // Atomically create a new directory in the (assumed sticky) /tmp directory for use with a + // Unix domain socket. The directory will be mode 0700. Retries indefinitely until it + // succeeds. + private static Path createTempSocketDirectory(Path tempDir) { + Random random = new Random(); + while (true) { + Path socketDir = tempDir.getRelative(String.format("blaze-%d", random.nextInt())); + try { + if (socketDir.createDirectory()) { + // Make sure it's private; unfortunately, createDirectory() doesn't take a mode + // argument. + socketDir.chmod(0700); + return socketDir; // Created. + } + // Already existed; try again. + } catch (IOException e) { + // Failed; try again. + } + } + } + + /** + * Read a string in platform default encoding and split it into a list of + * NUL-separated words. + * + * <p>Blaze consistently uses the platform default encoding (defined in + * blaze.cc) to interface with Unix APIs. + */ + private static List<String> readRequest(InputStream input) throws IOException { + byte[] inputBytes = ByteStreams.toByteArray(input); + if (inputBytes.length == 0) { + return null; + } + String s = new String(inputBytes, Charset.defaultCharset()); + return ImmutableList.copyOf(NULLTERMINATOR_SPLITTER.split(s)); + } + + private void executeRequest(RequestIo requestIo) { + int exitStatus = 2; + try { + List<String> request = readRequest(requestIo.in); + if (request == null) { + LOG.info("Short-circuiting empty request"); + return; + } + exitStatus = rpcService.executeRequest(request, requestIo.requestOutErr, + requestIo.firstContactTime); + LOG.info("Finished executing request"); + } catch (UnknownCommandException e) { + requestIo.requestOutErr.printErrLn("SERVER ERROR: " + e.getMessage()); + LOG.severe("SERVER ERROR: " + e.getMessage()); + } catch (Exception e) { + // Stacktrace for unknown exception. + StringWriter trace = new StringWriter(); + e.printStackTrace(new PrintWriter(trace, true)); + requestIo.requestOutErr.printErr("SERVER ERROR: " + trace); + LOG.severe("SERVER ERROR: " + trace); + } + + if (rpcService.isShutdown()) { + // In case of shutdown, disable the listening socket *before* we write + // the last part of the response. Otherwise, a sufficiently fast client + // could read the response and exit, and a new client could make a + // connection to this server, which is still in the listening state, even + // though it is about to shut down imminently. + closeServerSocket(); + } + + requestIo.writeExitStatus(exitStatus); + } + + /** + * Because it's a little complicated, this class factors out all the IO Hook + * up we need per request, that is, in + * {@link RPCServer#executeRequest(RequestIo)}. + * It's unfortunately complicated, so it's explained here. + */ + private static class RequestIo { + + // Used by the client code + private final InputStream in; + private final OutErr requestOutErr; + private final OutputStream controlChannel; + + // just used by this class to keep the state around + private final Socket requestSocket; + private final OutputStream requestOut; + private final long firstContactTime; + + RequestIo(Socket requestSocket, long firstContactTime) throws IOException { + this.requestSocket = requestSocket; + this.firstContactTime = firstContactTime; + this.in = requestSocket.getInputStream(); + this.requestOut = requestSocket.getOutputStream(); + + // We encode the response sent to the client with a multiplexer so + // we can send three streams (out / err / control) over one wire stream + // (requestOut). + StreamMultiplexer multiplexer = new StreamMultiplexer(requestOut); + + // We'll be writing control messages (exit code + out of date message) + // to this control channel. + controlChannel = multiplexer.createControl(); + + // This is the outErr part of the multiplexed output. + requestOutErr = OutErr.create(multiplexer.createStdout(), + multiplexer.createStderr()); + // We hook up System.out / System.err to our IO object. Stuff written to + // System.out / System.err will show up on the user's screen, prefixed + // with "System.out "/"System.err ". + requestOutErr.addSystemOutErrAsSource(); + } + + public void writeExitStatus(int exitStatus) { + // Make sure to flush the output / error streams prior to writing the exit status. + // The client may stop reading that direction of the socket immediately upon reading the + // exit code. + flushOutErr(); + try { + controlChannel.write(("" + exitStatus + "\n").getBytes(UTF_8)); + controlChannel.flush(); + LOG.info("" + exitStatus); + } catch (IOException ignored) { + // This exception is historically ignored. + } + } + + private void flushOutErr() { + try { + requestOutErr.getOutputStream().flush(); + } catch (IOException e) { + printStack(e); + } + try { + requestOutErr.getErrorStream().flush(); + } catch (IOException e) { + printStack(e); + } + } + + public void shutdown() { + try { + requestOut.close(); + } catch (IOException e) { + printStack(e); + } + try { + in.close(); + } catch (IOException e) { + printStack(e); + } + try { + requestSocket.close(); + } catch (IOException e) { + printStack(e); + } + } + } + + /** + * Creates and returns a new RPC server. + * Use {@link RPCServer#serve()} to start the server. + * + * @param appCommand The application's ServerCommand implementation. + * @param serverDirectory The directory for server-related files. The caller + * must ensure the directory has been created. + * @param workspaceDir The workspace, used solely to ensure it persists. + * @param maxIdleSeconds The idle time in seconds after which the rpc + * server will die unless it receives a request. + */ + public static RPCServer newServerWith(Clock clock, + ServerCommand appCommand, + Path serverDirectory, + Path workspaceDir, + int maxIdleSeconds) + throws IOException { + if (!serverDirectory.exists()) { + serverDirectory.createDirectory(); + } + + // Creates and starts the RPC server. + RPCService service = new RPCService(appCommand); + + return new RPCServer(clock, service, maxIdleSeconds * 1000L, + serverDirectory, workspaceDir); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/server/RPCService.java b/src/main/java/com/google/devtools/build/lib/server/RPCService.java new file mode 100644 index 0000000..379e83c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/server/RPCService.java
@@ -0,0 +1,95 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.server; + +import com.google.devtools.build.lib.util.io.OutErr; + +import java.util.List; +import java.util.logging.Logger; + +/** + * An RPCService is a Java object that can process RPC requests. Requests may + * be of the form: + * <pre> + * blaze <blaze-arguments> + * </pre> + * Requests are delegated to the ServerCommand instance provided + * to the constructor. + */ +public final class RPCService { + + private boolean isShutdown; + private static final Logger LOG = Logger.getLogger(RPCService.class.getName()); + private final ServerCommand appCommand; + + public RPCService(ServerCommand appCommand) { + this.appCommand = appCommand; + } + + /** + * The {@link #executeRequest(List, OutErr, long)} method may + * throw this exception if a command is unknown to the RPC service. + */ + public static class UnknownCommandException extends Exception { + private static final long serialVersionUID = 1L; + UnknownCommandException(String command) { + super("Unknown command: " + command); + } + } + + /** + * Executes the request; returns Unix like return codes (0 means success). May + * also throw arbitrary exceptions. + */ + public int executeRequest(List<String> request, + OutErr outErr, + long firstContactTime) throws Exception { + if (isShutdown) { + throw new IllegalStateException("Received request after shutdown."); + } + String command = request.isEmpty() ? "" : request.get(0); + if (appCommand != null && command.equals("blaze")) { // an application request + int result = appCommand.exec(request.subList(1, request.size()), outErr, firstContactTime); + if (appCommand.shutdown()) { // an application shutdown request + shutdown(); + } + return result; + } else { + throw new UnknownCommandException(command); + } + } + + /** + * After executing this function, further requests will fail, and + * {@link #isShutdown()} will return true. + */ + public void shutdown() { + if (isShutdown) { + return; + } + LOG.info("RPC Service: shutting down ..."); + isShutdown = true; + } + + /** + * Has this service been shutdown. If so, any call to + * {@link #executeRequest(List, OutErr, long)} will result in an + * {@link IllegalStateException} + */ + public boolean isShutdown() { + return isShutdown; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/server/ServerCommand.java b/src/main/java/com/google/devtools/build/lib/server/ServerCommand.java new file mode 100644 index 0000000..972753c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/server/ServerCommand.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.server; + +import com.google.devtools.build.lib.util.io.OutErr; + +import java.util.List; + +/** + * The {@link RPCServer} calls an arbitrary command implementing this + * interface. + */ +public interface ServerCommand { + + /** + * Executes the request, writing any output or error messages into err. + * Returns 0 on success; any other value or exception indicates an error. + */ + int exec(List<String> args, OutErr outErr, long firstContactTime) throws Exception; + + /** + * The implementation returns true from this method to initiate a shutdown. + * No further requests will be handled. + */ + boolean shutdown(); + +}
diff --git a/src/main/java/com/google/devtools/build/lib/server/ServerResponse.java b/src/main/java/com/google/devtools/build/lib/server/ServerResponse.java new file mode 100644 index 0000000..e5ab930 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/server/ServerResponse.java
@@ -0,0 +1,114 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.server; + +import com.google.common.base.Preconditions; + +import java.io.ByteArrayOutputStream; +import java.io.UnsupportedEncodingException; + +/** + * This class models a response from the {@link RPCServer}. This is a + * tuple of an error message and the exit status. The encoding of the response + * is extremely simple {@link #toString()}: + * + * <ul><li>Iff a message is present, the wire format is + * <pre>message + '\n' + exit code as string + '\n'</pre> + * </li> + * <li>Otherwise it's just the exit code as string + '\n'</li> + * </ul> + */ +final class ServerResponse { + + /** + * Parses an input string into a {@link ServerResponse} object. + */ + public static ServerResponse parseFrom(String input) { + if (input.charAt(input.length() - 1) != '\n') { + String msg = "Response must end with newline (" + input + ")"; + throw new IllegalArgumentException(msg); + } + int newlineAt = input.lastIndexOf('\n', input.length() - 2); + + final String exitStatusString; + final String errorMessage; + if (newlineAt == -1) { + errorMessage = ""; + exitStatusString = input.substring(0, input.length() - 1); + } else { + errorMessage = input.substring(0, newlineAt); + exitStatusString = input.substring(newlineAt + 1, input.length() - 1); + } + + return new ServerResponse(errorMessage, Integer.parseInt(exitStatusString)); + } + + /** + * Parses {@code bytes} into a {@link ServerResponse} instance, assuming + * Latin 1 encoding. + */ + public static ServerResponse parseFrom(byte[] bytes) { + try { + return parseFrom(new String(bytes, "ISO-8859-1")); + } catch (UnsupportedEncodingException e) { + throw new AssertionError(e); // Latin 1 is everywhere. + } + } + + /** + * Parses {@code bytes} into a {@link ServerResponse} instance, assuming + * Latin 1 encoding. + */ + public static ServerResponse parseFrom(ByteArrayOutputStream bytes) { + return parseFrom(bytes.toByteArray()); + } + + private final String errorMessage; + private final int exitStatus; + + /** + * Construct a new instance given an error message and an exit status. + */ + public ServerResponse(String errorMessage, int exitStatus) { + Preconditions.checkNotNull(errorMessage); + this.errorMessage = errorMessage; + this.exitStatus = exitStatus; + } + + /** + * The wire representation of this response object. + */ + @Override + public String toString() { + if (errorMessage.length() == 0) { + return Integer.toString(exitStatus) + '\n'; + } + return errorMessage + '\n' + Integer.toString(exitStatus) + '\n'; + } + + @Override + public boolean equals(Object other) { + if (other == null || !(other instanceof ServerResponse)) return false; + ServerResponse otherResponse = (ServerResponse) other; + return exitStatus == otherResponse.exitStatus + && errorMessage.equals(otherResponse.errorMessage); + } + + @Override + public int hashCode() { + return exitStatus * 31 ^ errorMessage.hashCode(); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/server/signal/InterruptSignalHandler.java b/src/main/java/com/google/devtools/build/lib/server/signal/InterruptSignalHandler.java new file mode 100644 index 0000000..521dcef --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/server/signal/InterruptSignalHandler.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.server.signal; + + +import com.google.common.base.Preconditions; + +import sun.misc.Signal; +import sun.misc.SignalHandler; + +/** + * A facade around sun.misc.Signal providing special-purpose SIGINT handling. + * + * We use this code in preference to using sun.misc directly since the latter + * is deprecated, and depending on it causes the jdk1.6 javac to emit an + * unsuppressable warning that sun.misc is "Sun proprietary API and may be + * removed in a future release". + */ +public abstract class InterruptSignalHandler implements Runnable { + + private static final Signal SIGINT = new Signal("INT"); + + private SignalHandler oldHandler; + + /** + * Constructs an InterruptSignalHandler instance. Until the uninstall() + * method is invoked, the delivery of a SIGINT signal to this process will + * cause the run() method to be invoked in another thread. + */ + protected InterruptSignalHandler() { + this.oldHandler = Signal.handle(SIGINT, new SignalHandler() { + @Override + public void handle(Signal signal) { + run(); + } + }); + } + + /** + * Disables SIGINT handling. + */ + public synchronized final void uninstall() { + Preconditions.checkNotNull(oldHandler, "uninstall() already called"); + Signal.handle(SIGINT, oldHandler); + oldHandler = null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/AbnormalTerminationException.java b/src/main/java/com/google/devtools/build/lib/shell/AbnormalTerminationException.java new file mode 100644 index 0000000..30562c6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/AbnormalTerminationException.java
@@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +/** + * Thrown when a command's execution terminates abnormally -- for example, + * if it is killed, or if it terminates with a non-zero exit status. + */ +public class AbnormalTerminationException extends CommandException { + + private final CommandResult result; + + public AbnormalTerminationException(final Command command, + final CommandResult result, + final String message) { + super(command, message); + this.result = result; + } + + public AbnormalTerminationException(final Command command, + final CommandResult result, + final Throwable cause) { + super(command, cause); + this.result = result; + } + + public AbnormalTerminationException(final Command command, + final CommandResult result, + final String message, + final Throwable cause) { + super(command, message, cause); + this.result = result; + } + + public CommandResult getResult() { + return result; + } + + private static final long serialVersionUID = 2L; +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/BadExitStatusException.java b/src/main/java/com/google/devtools/build/lib/shell/BadExitStatusException.java new file mode 100644 index 0000000..324007a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/BadExitStatusException.java
@@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +/** + * Thrown when a command's execution terminates with a non-zero exit status. + */ +public final class BadExitStatusException extends AbnormalTerminationException { + + public BadExitStatusException(final Command command, + final CommandResult result, + final String message) { + super(command, result, message); + } + + public BadExitStatusException(final Command command, + final CommandResult result, + final String message, + final Throwable cause) { + super(command, result, message, cause); + } + + private static final long serialVersionUID = 1L; +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/Command.java b/src/main/java/com/google/devtools/build/lib/shell/Command.java new file mode 100644 index 0000000..ab4a7fc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/Command.java
@@ -0,0 +1,960 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * <p>Represents an executable command, including its arguments and + * runtime environment (environment variables, working directory). This class + * lets a caller execute a command, get its results, and optionally try to kill + * the task during execution.</p> + * + * <p>The use of "shell" in the full name of this class is a misnomer. In + * terms of the way its arguments are interpreted, this class is closer to + * {@code execve(2)} than to {@code system(3)}. No Bourne shell is executed. + * + * <p>The most basic use-case for this class is as follows: + * <pre> + * String[] args = { "/bin/du", "-s", directory }; + * CommandResult result = new Command(args).execute(); + * String output = new String(result.getStdout()); + * </pre> + * which writes the output of the {@code du(1)} command into {@code output}. + * More complex cases might inspect the stderr stream, kill the subprocess + * asynchronously, feed input to its standard input, handle the exceptions + * thrown if the command fails, or print the termination status (exit code or + * signal name). + * + * <h4>Invoking the Bourne shell</h4> + * + * <p>Perhaps the most common command invoked programmatically is the UNIX + * shell, {@code /bin/sh}. Because the shell is a general-purpose programming + * language, care must be taken to ensure that variable parts of the shell + * command (e.g. strings entered by the user) do not contain shell + * metacharacters, as this poses a correctness and/or security risk. + * + * <p>To execute a shell command directly, use the following pattern: + * <pre> + * String[] args = { "/bin/sh", "-c", shellCommand }; + * CommandResult result = new Command(args).execute(); + * </pre> + * {@code shellCommand} is a complete Bourne shell program, possibly containing + * all kinds of unescaped metacharacters. For example, here's a shell command + * that enumerates the working directories of all processes named "foo": + * <pre>ps auxx | grep foo | awk '{print $1}' | + * while read pid; do readlink /proc/$pid/cwd; done</pre> + * It is the responsibility of the caller to ensure that this string means what + * they intend. + * + * <p>Consider the risk posed by allowing the "foo" part of the previous + * command to be some arbitrary (untrusted) string called {@code processName}: + * <pre> + * // WARNING: unsafe! + * String shellCommand = "ps auxx | grep " + processName + " | awk '{print $1}' | " + * + "while read pid; do readlink /proc/$pid/cwd; done";</pre> + * </pre> + * Passing this string to {@link Command} is unsafe because if the string + * {@processName} contains shell metacharacters, the meaning of the command can + * be arbitrarily changed; consider: + * <pre>String processName = ". ; rm -fr $HOME & ";</pre> + * + * <p>To defend against this possibility, it is essential to properly quote the + * variable portions of the shell command so that shell metacharacters are + * escaped. Use {@link ShellUtils#shellEscape} for this purpose: + * <pre> + * // Safe. + * String shellCommand = "ps auxx | grep " + ShellUtils.shellEscape(processName) + * + " | awk '{print $1}' | while read pid; do readlink /proc/$pid/cwd; done"; + * </pre> + * + * <p>Tip: if you are only invoking a single known command, and no shell + * features (e.g. $PATH lookup, output redirection, pipelines, etc) are needed, + * call it directly without using a shell, as in the {@code du(1)} example + * above. + * + * <h4>Other features</h4> + * + * <p>A caller can optionally specify bytes to be written to the process's + * "stdin". The returned {@link CommandResult} object gives the caller access to + * the exit status, as well as output from "stdout" and "stderr". To use + * this class with processes that generate very large amounts of input/output, + * consider + * {@link #execute(InputStream, KillableObserver, OutputStream, OutputStream)} + * and + * {@link #execute(byte[], KillableObserver, OutputStream, OutputStream)}. + * </p> + * + * <p>This class ensures that stdout and stderr streams are read promptly, + * avoiding potential deadlock if the output is large. See <a + * href="http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html"> When + * <code>Runtime.exec()</code> won't</a>.</p> + * + * <p>This class is immutable and therefore thread-safe.</p> + */ +public final class Command { + + private static final Logger log = + Logger.getLogger("com.google.devtools.build.lib.shell.Command"); + + /** + * Pass this value to {@link #execute(byte[])} to indicate that no input + * should be written to stdin. + */ + public static final byte[] NO_INPUT = new byte[0]; + + private static final String[] EMPTY_STRING_ARRAY = new String[0]; + + /** + * Pass this to {@link #execute(byte[], KillableObserver, boolean)} to + * indicate that you do not wish to observe / kill the underlying + * process. + */ + public static final KillableObserver NO_OBSERVER = new KillableObserver() { + @Override + public void startObserving(final Killable killable) { + // do nothing + } + @Override + public void stopObserving(final Killable killable) { + // do nothing + } + }; + + private final ProcessBuilder processBuilder; + + // Start of public API ----------------------------------------------------- + + /** + * Creates a new {@link Command} that will execute a command line that + * is described by a {@link ProcessBuilder}. Command line elements, + * environment, and working directory are taken from this object. The + * command line is executed exactly as given, without a shell. + * + * @param processBuilder {@link ProcessBuilder} describing command line + * to execute + */ + public Command(final ProcessBuilder processBuilder) { + this(processBuilder.command().toArray(EMPTY_STRING_ARRAY), + processBuilder.environment(), + processBuilder.directory()); + } + + /** + * Creates a new {@link Command} for the given command line elements. The + * command line is executed exactly as given, without a shell. + * Subsequent calls to {@link #execute()} will use the JVM's working + * directory and environment. + * + * @param commandLineElements elements of raw command line to execute + * @throws IllegalArgumentException if commandLine is null or empty + */ + /* TODO(bazel-team): Use varargs here + */ + public Command(final String[] commandLineElements) { + this(commandLineElements, null, null); + } + + /** + * <p>Creates a new {@link Command} for the given command line elements. + * Subsequent calls to {@link #execute()} will use the JVM's working + * directory and environment.</p> + * + * <p>Note: be careful when setting useShell to <code>true</code>; you + * may inadvertently expose a security hole. See + * {@link #Command(String, Map, File)}.</p> + * + * @param commandLineElements elements of raw command line to execute + * @param useShell if true, command is executed using a shell interpreter + * (e.g. <code>/bin/sh</code> on Linux); if false, command is executed + * exactly as given + * @throws IllegalArgumentException if commandLine is null or empty + */ + public Command(final String[] commandLineElements, final boolean useShell) { + this(commandLineElements, useShell, null, null); + } + + /** + * Creates a new {@link Command} for the given command line elements. The + * command line is executed exactly as given, without a shell. The given + * environment variables and working directory are used in subsequent + * calls to {@link #execute()}. + * + * @param commandLineElements elements of raw command line to execute + * @param environmentVariables environment variables to replace JVM's + * environment variables; may be null + * @param workingDirectory working directory for execution; if null, current + * working directory is used + * @throws IllegalArgumentException if commandLine is null or empty + */ + public Command(final String[] commandLineElements, + final Map<String, String> environmentVariables, + final File workingDirectory) { + this(commandLineElements, false, environmentVariables, workingDirectory); + } + + /** + * <p>Creates a new {@link Command} for the given command line elements. The + * given environment variables and working directory are used in subsequent + * calls to {@link #execute()}.</p> + * + * <p>Note: be careful when setting useShell to <code>true</code>; you + * may inadvertently expose a security hole. See + * {@link #Command(String, Map, File)}.</p> + * + * @param commandLineElements elements of raw command line to execute + * @param useShell if true, command is executed using a shell interpreter + * (e.g. <code>/bin/sh</code> on Linux); if false, command is executed + * exactly as given + * @param environmentVariables environment variables to replace JVM's + * environment variables; may be null + * @param workingDirectory working directory for execution; if null, current + * working directory is used + * @throws IllegalArgumentException if commandLine is null or empty + */ + public Command(final String[] commandLineElements, + final boolean useShell, + final Map<String, String> environmentVariables, + final File workingDirectory) { + if (commandLineElements == null || commandLineElements.length == 0) { + throw new IllegalArgumentException("command line is null or empty"); + } + this.processBuilder = + new ProcessBuilder(maybeAddShell(commandLineElements, useShell)); + if (environmentVariables != null) { + // TODO(bazel-team) remove next line eventually; it is here to mimic old + // Runtime.exec() behavior + this.processBuilder.environment().clear(); + this.processBuilder.environment().putAll(environmentVariables); + } + this.processBuilder.directory(workingDirectory); + } + + private static String[] maybeAddShell(final String[] commandLineElements, + final boolean useShell) { + if (useShell) { + final StringBuilder builder = new StringBuilder(); + for (final String element : commandLineElements) { + if (builder.length() > 0) { + builder.append(' '); + } + builder.append(element); + } + return Shell.getPlatformShell().shellify(builder.toString()); + } else { + return commandLineElements; + } + } + + /** + * @return raw command line elements to be executed + */ + public String[] getCommandLineElements() { + final List<String> elements = processBuilder.command(); + return elements.toArray(new String[elements.size()]); + } + + /** + * @return (unmodifiable) {@link Map} view of command's environment variables + */ + public Map<String, String> getEnvironmentVariables() { + return Collections.unmodifiableMap(processBuilder.environment()); + } + + /** + * @return working directory used for execution, or null if the current + * working directory is used + */ + public File getWorkingDirectory() { + return processBuilder.directory(); + } + + /** + * Execute this command with no input to stdin. This call will block until the + * process completes or an error occurs. + * + * @return {@link CommandResult} representing result of the execution + * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any + * reason + * @throws AbnormalTerminationException if an {@link IOException} is + * encountered while reading from the process, or the process was terminated + * due to a signal. + * @throws BadExitStatusException if the process exits with a + * non-zero status + */ + public CommandResult execute() throws CommandException { + return execute(NO_INPUT); + } + + /** + * Execute this command with given input to stdin. This call will block until + * the process completes or an error occurs. + * + * @param stdinInput bytes to be written to process's stdin + * @return {@link CommandResult} representing result of the execution + * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any + * reason + * @throws AbnormalTerminationException if an {@link IOException} is + * encountered while reading from the process, or the process was terminated + * due to a signal. + * @throws BadExitStatusException if the process exits with a + * non-zero status + * @throws NullPointerException if stdin is null + */ + public CommandResult execute(final byte[] stdinInput) + throws CommandException { + nullCheck(stdinInput, "stdinInput"); + return doExecute(new ByteArrayInputSource(stdinInput), + NO_OBSERVER, + Consumers.createAccumulatingConsumers(), + /*killSubprocess=*/false, /*closeOutput=*/false).get(); + } + + /** + * <p>Execute this command with given input to stdin. This call will block + * until the process completes or an error occurs. Caller may specify + * whether the method should ignore stdout/stderr output. If the + * given number of milliseconds elapses before the command has + * completed, this method will attempt to kill the command.</p> + * + * @param stdinInput bytes to be written to process's stdin, or + * {@link #NO_INPUT} if no bytes should be written + * @param timeout number of milliseconds to wait for command completion + * before attempting to kill the command + * @param ignoreOutput if true, method will ignore stdout/stderr output + * and return value will not contain this data + * @return {@link CommandResult} representing result of the execution + * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any + * reason + * @throws AbnormalTerminationException if an {@link IOException} is + * encountered while reading from the process, or the process was terminated + * due to a signal. + * @throws BadExitStatusException if the process exits with a + * non-zero status + * @throws NullPointerException if stdin is null + */ + public CommandResult execute(final byte[] stdinInput, + final long timeout, + final boolean ignoreOutput) + throws CommandException { + return execute(stdinInput, + new TimeoutKillableObserver(timeout), + ignoreOutput); + } + + /** + * <p>Execute this command with given input to stdin. This call will block + * until the process completes or an error occurs. Caller may specify + * whether the method should ignore stdout/stderr output. The given {@link + * KillableObserver} may also terminate the process early while running.</p> + * + * @param stdinInput bytes to be written to process's stdin, or + * {@link #NO_INPUT} if no bytes should be written + * @param observer {@link KillableObserver} that should observe the running + * process, or {@link #NO_OBSERVER} if caller does not wish to kill + * the process + * @param ignoreOutput if true, method will ignore stdout/stderr output + * and return value will not contain this data + * @return {@link CommandResult} representing result of the execution + * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any + * reason + * @throws AbnormalTerminationException if the process is interrupted (or + * killed) before completion, if an {@link IOException} is encountered while + * reading from the process, or the process was terminated due to a signal. + * @throws BadExitStatusException if the process exits with a + * non-zero status + * @throws NullPointerException if stdin is null + */ + public CommandResult execute(final byte[] stdinInput, + final KillableObserver observer, + final boolean ignoreOutput) + throws CommandException { + // supporting "null" here for backwards compatibility + final KillableObserver theObserver = + observer == null ? NO_OBSERVER : observer; + return doExecute(new ByteArrayInputSource(stdinInput), + theObserver, + ignoreOutput ? Consumers.createDiscardingConsumers() + : Consumers.createAccumulatingConsumers(), + /*killSubprocess=*/false, /*closeOutput=*/false).get(); + } + + /** + * <p>Execute this command with given input to stdin. This call blocks + * until the process completes or an error occurs. The caller provides + * {@link OutputStream} instances into which the process writes its + * stdout/stderr output; these streams are <em>not</em> closed when the + * process terminates. The given {@link KillableObserver} may also + * terminate the process early while running.</p> + * + * <p>Note that stdout and stderr are written concurrently. If these are + * aliased to each other, it is the caller's duty to ensure thread safety. + * </p> + * + * @param stdinInput bytes to be written to process's stdin, or + * {@link #NO_INPUT} if no bytes should be written + * @param observer {@link KillableObserver} that should observe the running + * process, or {@link #NO_OBSERVER} if caller does not wish to kill the + * process + * @param stdOut the process will write its standard output into this stream. + * E.g., you could pass {@link System#out} as <code>stdOut</code>. + * @param stdErr the process will write its standard error into this stream. + * E.g., you could pass {@link System#err} as <code>stdErr</code>. + * @return {@link CommandResult} representing result of the execution. Note + * that {@link CommandResult#getStdout()} and + * {@link CommandResult#getStderr()} will yield {@link IllegalStateException} + * in this case, as the output is written to <code>stdOut/stdErr</code> + * instead. + * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any + * reason + * @throws AbnormalTerminationException if the process is interrupted (or + * killed) before completion, if an {@link IOException} is encountered while + * reading from the process, or the process was terminated due to a signal. + * @throws BadExitStatusException if the process exits with a + * non-zero status + * @throws NullPointerException if any argument is null. + */ + public CommandResult execute(final byte[] stdinInput, + final KillableObserver observer, + final OutputStream stdOut, + final OutputStream stdErr) + throws CommandException { + return execute(stdinInput, observer, stdOut, stdErr, false); + } + + /** + * Like {@link #execute(byte[], KillableObserver, OutputStream, OutputStream)} + * but enables setting of the killSubprocessOnInterrupt attribute. + * + * @param killSubprocessOnInterrupt if set to true, the execution of + * this command is <i>interruptible</i>: in other words, if this thread is + * interrupted during a call to execute, the subprocess will be terminated + * and the call will return in a timely manner. If false, the subprocess + * will run to completion; this is the default value use by all other + * constructors. The thread's interrupted status is preserved in all cases, + * however. + */ + public CommandResult execute(final byte[] stdinInput, + final KillableObserver observer, + final OutputStream stdOut, + final OutputStream stdErr, + final boolean killSubprocessOnInterrupt) + throws CommandException { + nullCheck(stdinInput, "stdinInput"); + nullCheck(observer, "observer"); + nullCheck(stdOut, "stdOut"); + nullCheck(stdErr, "stdErr"); + return doExecute(new ByteArrayInputSource(stdinInput), + observer, + Consumers.createStreamingConsumers(stdOut, stdErr), + killSubprocessOnInterrupt, false).get(); + } + + /** + * <p>Execute this command with given input to stdin; this stream is closed + * when the process terminates, and exceptions raised when closing this + * stream are ignored. This call blocks + * until the process completes or an error occurs. The caller provides + * {@link OutputStream} instances into which the process writes its + * stdout/stderr output; these streams are <em>not</em> closed when the + * process terminates. The given {@link KillableObserver} may also + * terminate the process early while running.</p> + * + * @param stdinInput The input to this process's stdin + * @param observer {@link KillableObserver} that should observe the running + * process, or {@link #NO_OBSERVER} if caller does not wish to kill the + * process + * @param stdOut the process will write its standard output into this stream. + * E.g., you could pass {@link System#out} as <code>stdOut</code>. + * @param stdErr the process will write its standard error into this stream. + * E.g., you could pass {@link System#err} as <code>stdErr</code>. + * @return {@link CommandResult} representing result of the execution. Note + * that {@link CommandResult#getStdout()} and + * {@link CommandResult#getStderr()} will yield {@link IllegalStateException} + * in this case, as the output is written to <code>stdOut/stdErr</code> + * instead. + * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any + * reason + * @throws AbnormalTerminationException if the process is interrupted (or + * killed) before completion, if an {@link IOException} is encountered while + * reading from the process, or the process was terminated due to a signal. + * @throws BadExitStatusException if the process exits with a + * non-zero status + * @throws NullPointerException if any argument is null. + */ + public CommandResult execute(final InputStream stdinInput, + final KillableObserver observer, + final OutputStream stdOut, + final OutputStream stdErr) + throws CommandException { + nullCheck(stdinInput, "stdinInput"); + nullCheck(observer, "observer"); + nullCheck(stdOut, "stdOut"); + nullCheck(stdErr, "stdErr"); + return doExecute(new InputStreamInputSource(stdinInput), + observer, + Consumers.createStreamingConsumers(stdOut, stdErr), + /*killSubprocess=*/false, /*closeOutput=*/false).get(); + } + + /** + * <p>Execute this command with given input to stdin; this stream is closed + * when the process terminates, and exceptions raised when closing this + * stream are ignored. This call blocks + * until the process completes or an error occurs. The caller provides + * {@link OutputStream} instances into which the process writes its + * stdout/stderr output; these streams are closed when the process terminates + * if closeOut is set. The given {@link KillableObserver} may also + * terminate the process early while running.</p> + * + * @param stdinInput The input to this process's stdin + * @param observer {@link KillableObserver} that should observe the running + * process, or {@link #NO_OBSERVER} if caller does not wish to kill the + * process + * @param stdOut the process will write its standard output into this stream. + * E.g., you could pass {@link System#out} as <code>stdOut</code>. + * @param stdErr the process will write its standard error into this stream. + * E.g., you could pass {@link System#err} as <code>stdErr</code>. + * @param closeOut whether to close the output streams when the subprocess + * terminates. + * @return {@link CommandResult} representing result of the execution. Note + * that {@link CommandResult#getStdout()} and + * {@link CommandResult#getStderr()} will yield {@link IllegalStateException} + * in this case, as the output is written to <code>stdOut/stdErr</code> + * instead. + * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any + * reason + * @throws AbnormalTerminationException if the process is interrupted (or + * killed) before completion, if an {@link IOException} is encountered while + * reading from the process, or the process was terminated due to a signal. + * @throws BadExitStatusException if the process exits with a + * non-zero status + * @throws NullPointerException if any argument is null. + */ + public CommandResult execute(final InputStream stdinInput, + final KillableObserver observer, + final OutputStream stdOut, + final OutputStream stdErr, + boolean closeOut) + throws CommandException { + nullCheck(stdinInput, "stdinInput"); + nullCheck(observer, "observer"); + nullCheck(stdOut, "stdOut"); + nullCheck(stdErr, "stdErr"); + return doExecute(new InputStreamInputSource(stdinInput), + observer, + Consumers.createStreamingConsumers(stdOut, stdErr), + false, closeOut).get(); + } + + /** + * <p>Executes this command with the given stdinInput, but does not + * wait for it to complete. The caller may choose to observe the status + * of the launched process by calling methods on the returned object. + * + * @param stdinInput bytes to be written to process's stdin, or + * {@link #NO_INPUT} if no bytes should be written + * @return An object that can be used to check if the process terminated and + * obtain the process results. + * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any + * reason + * @throws NullPointerException if stdin is null + */ + public FutureCommandResult executeAsynchronously(final byte[] stdinInput) + throws CommandException { + return executeAsynchronously(stdinInput, NO_OBSERVER); + } + + /** + * <p>Executes this command with the given input to stdin, but does + * not wait for it to complete. The caller may choose to observe the + * status of the launched process by calling methods on the returned + * object. This method performs the minimum cleanup after the + * process terminates: It closes the input stream, and it ignores + * exceptions that result from closing it. The given {@link + * KillableObserver} may also terminate the process early while + * running.</p> + * + * <p>Note that in this case the {@link KillableObserver} will be assigned + * to start observing the process via + * {@link KillableObserver#startObserving(Killable)} but will only be + * unassigned via {@link KillableObserver#stopObserving(Killable)}, if + * {@link FutureCommandResult#get()} is called. If the + * {@link KillableObserver} implementation used with this method will + * not work correctly without calls to + * {@link KillableObserver#stopObserving(Killable)} then a new instance + * should be used for each call to this method.</p> + * + * @param stdinInput bytes to be written to process's stdin, or + * {@link #NO_INPUT} if no bytes should be written + * @param observer {@link KillableObserver} that should observe the running + * process, or {@link #NO_OBSERVER} if caller does not wish to kill + * the process + * @return An object that can be used to check if the process terminated and + * obtain the process results. + * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any + * reason + * @throws NullPointerException if stdin is null + */ + public FutureCommandResult executeAsynchronously(final byte[] stdinInput, + final KillableObserver observer) + throws CommandException { + // supporting "null" here for backwards compatibility + final KillableObserver theObserver = + observer == null ? NO_OBSERVER : observer; + nullCheck(stdinInput, "stdinInput"); + return doExecute(new ByteArrayInputSource(stdinInput), + theObserver, + Consumers.createDiscardingConsumers(), + /*killSubprocess=*/false, /*closeOutput=*/false); + } + + /** + * <p>Executes this command with the given input to stdin, but does + * not wait for it to complete. The caller may choose to observe the + * status of the launched process by calling methods on the returned + * object. This method performs the minimum cleanup after the + * process terminates: It closes the input stream, and it ignores + * exceptions that result from closing it. The caller provides + * {@link OutputStream} instances into which the process writes its + * stdout/stderr output; these streams are <em>not</em> closed when + * the process terminates. The given {@link KillableObserver} may + * also terminate the process early while running.</p> + * + * <p>Note that stdout and stderr are written concurrently. If these are + * aliased to each other, or if the caller continues to write to these + * streams, it is the caller's duty to ensure thread safety. + * </p> + * + * <p>Note that in this case the {@link KillableObserver} will be assigned + * to start observing the process via + * {@link KillableObserver#startObserving(Killable)} but will only be + * unassigned via {@link KillableObserver#stopObserving(Killable)}, if + * {@link FutureCommandResult#get()} is called. If the + * {@link KillableObserver} implementation used with this method will + * not work correctly without calls to + * {@link KillableObserver#stopObserving(Killable)} then a new instance + * should be used for each call to this method.</p> + * + * @param stdinInput The input to this process's stdin + * @param observer {@link KillableObserver} that should observe the running + * process, or {@link #NO_OBSERVER} if caller does not wish to kill + * the process + * @param stdOut the process will write its standard output into this stream. + * E.g., you could pass {@link System#out} as <code>stdOut</code>. + * @param stdErr the process will write its standard error into this stream. + * E.g., you could pass {@link System#err} as <code>stdErr</code>. + * @return An object that can be used to check if the process terminated and + * obtain the process results. + * @throws ExecFailedException if {@link Runtime#exec(String[])} fails for any + * reason + * @throws NullPointerException if stdin is null + */ + public FutureCommandResult executeAsynchronously(final InputStream stdinInput, + final KillableObserver observer, + final OutputStream stdOut, + final OutputStream stdErr) + throws CommandException { + // supporting "null" here for backwards compatibility + final KillableObserver theObserver = + observer == null ? NO_OBSERVER : observer; + nullCheck(stdinInput, "stdinInput"); + return doExecute(new InputStreamInputSource(stdinInput), + theObserver, + Consumers.createStreamingConsumers(stdOut, stdErr), + /*killSubprocess=*/false, /*closeOutput=*/false); + } + + // End of public API ------------------------------------------------------- + + private void nullCheck(Object argument, String argumentName) { + if (argument == null) { + String message = argumentName + " argument must not be null."; + throw new NullPointerException(message); + } + } + + private FutureCommandResult doExecute(final InputSource stdinInput, + final KillableObserver observer, + final Consumers.OutErrConsumers outErrConsumers, + final boolean killSubprocessOnInterrupt, + final boolean closeOutputStreams) + throws CommandException { + + logCommand(); + + final Process process = startProcess(); + + outErrConsumers.logConsumptionStrategy(); + + outErrConsumers.registerInputs(process.getInputStream(), + process.getErrorStream(), + closeOutputStreams); + + processInput(stdinInput, process); + + // TODO(bazel-team): if the input stream is unbounded, observers will not get start + // notification in a timely manner! + final Killable processKillable = observeProcess(process, observer); + + return new FutureCommandResult() { + @Override + public CommandResult get() throws AbnormalTerminationException { + return waitForProcessToComplete(process, + observer, + processKillable, + outErrConsumers, + killSubprocessOnInterrupt); + } + + @Override + public boolean isDone() { + try { + // exitValue seems to be the only non-blocking call for + // checking process liveness. + process.exitValue(); + return true; + } catch (IllegalThreadStateException e) { + return false; + } + } + }; + } + + private Process startProcess() + throws ExecFailedException { + try { + return processBuilder.start(); + } catch (IOException ioe) { + throw new ExecFailedException(this, ioe); + } + } + + private static interface InputSource { + void copyTo(OutputStream out) throws IOException; + boolean isEmpty(); + String toLogString(String sourceName); + } + + private static class ByteArrayInputSource implements InputSource { + private byte[] bytes; + ByteArrayInputSource(byte[] bytes){ + this.bytes = bytes; + } + @Override + public void copyTo(OutputStream out) throws IOException { + out.write(bytes); + out.flush(); + } + @Override + public boolean isEmpty() { + return bytes.length == 0; + } + @Override + public String toLogString(String sourceName) { + if (isEmpty()) { + return "No input to " + sourceName; + } else { + return "Input to " + sourceName + ": " + + LogUtil.toTruncatedString(bytes); + } + } + } + + private static class InputStreamInputSource implements InputSource { + private InputStream inputStream; + InputStreamInputSource(InputStream inputStream){ + this.inputStream = inputStream; + } + @Override + public void copyTo(OutputStream out) throws IOException { + byte[] buf = new byte[4096]; + int r; + while ((r = inputStream.read(buf)) != -1) { + out.write(buf, 0, r); + out.flush(); + } + } + @Override + public boolean isEmpty() { + return false; + } + @Override + public String toLogString(String sourceName) { + return "Input to " + sourceName + " is a stream."; + } + } + + private static void processInput(final InputSource stdinInput, + final Process process) { + if (log.isLoggable(Level.FINER)) { + log.finer(stdinInput.toLogString("stdin")); + } + try { + if (stdinInput.isEmpty()) { + return; + } + stdinInput.copyTo(process.getOutputStream()); + } catch (IOException ioe) { + // Note: this is not an error! Perhaps the command just isn't hungry for + // our input and exited with success. Process.waitFor (later) will tell + // us. + // + // (Unlike out/err streams, which are read asynchronously, the input stream is written + // synchronously, in its entirety, before processInput returns. If the input is + // infinite, and is passed through e.g. "cat" subprocess and back into the + // ByteArrayOutputStream, that will eventually run out of memory, causing the output stream + // to be closed, "cat" to terminate with SIGPIPE, and processInput to receive an IOException. + } finally { + // if this statement is ever deleted, the process's outputStream + // must be closed elsewhere -- it is not closed automatically + Command.silentClose(process.getOutputStream()); + } + } + + private static Killable observeProcess(final Process process, + final KillableObserver observer) { + final Killable processKillable = new ProcessKillable(process); + observer.startObserving(processKillable); + return processKillable; + } + + private CommandResult waitForProcessToComplete( + final Process process, + final KillableObserver observer, + final Killable processKillable, + final Consumers.OutErrConsumers outErr, + final boolean killSubprocessOnInterrupt) + throws AbnormalTerminationException { + + log.finer("Waiting for process..."); + + TerminationStatus status = + waitForProcess(process, killSubprocessOnInterrupt); + + observer.stopObserving(processKillable); + + log.finer(status.toString()); + + try { + outErr.waitForCompletion(); + } catch (IOException ioe) { + CommandResult noOutputResult = + new CommandResult(CommandResult.EMPTY_OUTPUT, + CommandResult.EMPTY_OUTPUT, + status); + if (status.success()) { + // If command was otherwise successful, throw an exception about this + throw new AbnormalTerminationException(this, noOutputResult, ioe); + } else { + // Otherwise, throw the more important exception -- command + // was not successful + String message = status + + "; also encountered an error while attempting to retrieve output"; + throw status.exited() + ? new BadExitStatusException(this, noOutputResult, message, ioe) + : new AbnormalTerminationException(this, + noOutputResult, message, ioe); + } + } + + CommandResult result = new CommandResult(outErr.getAccumulatedOut(), + outErr.getAccumulatedErr(), + status); + result.logThis(); + if (status.success()) { + return result; + } else if (status.exited()) { + throw new BadExitStatusException(this, result, status.toString()); + } else { + throw new AbnormalTerminationException(this, result, status.toString()); + } + } + + private static TerminationStatus waitForProcess(Process process, + boolean killSubprocessOnInterrupt) { + boolean wasInterrupted = false; + try { + while (true) { + try { + return new TerminationStatus(process.waitFor()); + } catch (InterruptedException ie) { + wasInterrupted = true; + if (killSubprocessOnInterrupt) { + process.destroy(); + } + } + } + } finally { + // Read this for detailed explanation: + // http://www-128.ibm.com/developerworks/java/library/j-jtp05236.html + if (wasInterrupted) { + Thread.currentThread().interrupt(); // preserve interrupted status + } + } + } + + private void logCommand() { + if (!log.isLoggable(Level.FINE)) { + return; + } + log.fine(toDebugString()); + } + + /** + * A string representation of this command object which includes + * the arguments, the environment, and the working directory. Avoid + * relying on the specifics of this format. Note that the size + * of the result string will reflect the size of the command. + */ + public String toDebugString() { + StringBuilder message = new StringBuilder(128); + message.append("Executing (without brackets):"); + for (final String arg : processBuilder.command()) { + message.append(" ["); + message.append(arg); + message.append(']'); + } + message.append("; environment: "); + message.append(processBuilder.environment().toString()); + final File workingDirectory = processBuilder.directory(); + message.append("; working dir: "); + message.append(workingDirectory == null ? + "(current)" : + workingDirectory.toString()); + return message.toString(); + } + + /** + * Close the <code>out</code> stream and log a warning if anything happens. + */ + private static void silentClose(final OutputStream out) { + try { + out.close(); + } catch (IOException ioe) { + String message = "Unexpected exception while closing output stream"; + log.log(Level.WARNING, message, ioe); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/CommandException.java b/src/main/java/com/google/devtools/build/lib/shell/CommandException.java new file mode 100644 index 0000000..a11be97 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/CommandException.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +/** + * Superclass of all exceptions that may be thrown during command execution. + * It exists to unify them. It also provides access to the command name + * and arguments for the failing command. + */ +public class CommandException extends Exception { + + private final Command command; + + /** Returns the command that failed. */ + public Command getCommand() { + return command; + } + + public CommandException(Command command, final String message) { + super(message); + this.command = command; + } + + public CommandException(Command command, final Throwable cause) { + super(cause); + this.command = command; + } + + public CommandException(Command command, final String message, + final Throwable cause) { + super(message, cause); + this.command = command; + } + + private static final long serialVersionUID = 2L; +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/CommandResult.java b/src/main/java/com/google/devtools/build/lib/shell/CommandResult.java new file mode 100644 index 0000000..185f91d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/CommandResult.java
@@ -0,0 +1,116 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.io.ByteArrayOutputStream; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Encapsulates the results of a command execution, including exit status + * and output to stdout and stderr. + */ +public final class CommandResult { + + private static final Logger log = + Logger.getLogger("com.google.devtools.build.lib.shell.Command"); + + private static final byte[] NO_BYTES = new byte[0]; + + static final ByteArrayOutputStream EMPTY_OUTPUT = + new ByteArrayOutputStream() { + + @Override + public byte[] toByteArray() { + return NO_BYTES; + } + }; + + static final ByteArrayOutputStream NO_OUTPUT_COLLECTED = + new ByteArrayOutputStream(){ + + @Override + public byte[] toByteArray() { + throw new IllegalStateException("Output was not collected"); + } + }; + + private final ByteArrayOutputStream stdout; + private final ByteArrayOutputStream stderr; + private final TerminationStatus terminationStatus; + + CommandResult(final ByteArrayOutputStream stdout, + final ByteArrayOutputStream stderr, + final TerminationStatus terminationStatus) { + checkNotNull(stdout); + checkNotNull(stderr); + checkNotNull(terminationStatus); + this.stdout = stdout; + this.stderr = stderr; + this.terminationStatus = terminationStatus; + } + + /** + * @return raw bytes that were written to stdout by the command, or + * null if caller did chose to ignore output + * @throws IllegalStateException if output was not collected + */ + public byte[] getStdout() { + return stdout.toByteArray(); + } + + /** + * @return raw bytes that were written to stderr by the command, or + * null if caller did chose to ignore output + * @throws IllegalStateException if output was not collected + */ + public byte[] getStderr() { + return stderr.toByteArray(); + } + + /** + * @return the result of Process.waitFor for the subprocess. + * @deprecated this returns the result of Process.waitFor, which is not + * precisely defined, and is not to be confused with the value passed to + * exit(2) by the subprocess. Use getTerminationStatus() instead. + */ + @Deprecated + public int getExitStatus() { + return terminationStatus.getRawResult(); + } + + /** + * @return the termination status of the subprocess. + */ + public TerminationStatus getTerminationStatus() { + return terminationStatus; + } + + void logThis() { + if (!log.isLoggable(Level.FINER)) { + return; + } + log.finer(terminationStatus.toString()); + + if (stdout == NO_OUTPUT_COLLECTED) { + return; + } + log.finer("Stdout: " + LogUtil.toTruncatedString(stdout.toByteArray())); + log.finer("Stderr: " + LogUtil.toTruncatedString(stderr.toByteArray())); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/Consumers.java b/src/main/java/com/google/devtools/build/lib/shell/Consumers.java new file mode 100644 index 0000000..3ed5b7e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/Consumers.java
@@ -0,0 +1,359 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.shell; + +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * This class provides convenience methods for consuming (actively reading) + * output and error streams with different consumption policies: + * discarding ({@link #createDiscardingConsumers()}, + * accumulating ({@link #createAccumulatingConsumers()}, + * and streaming ({@link #createStreamingConsumers(OutputStream, OutputStream)}). + */ +class Consumers { + + private static final Logger log = + Logger.getLogger("com.google.devtools.build.lib.shell.Command"); + + private Consumers() {} + + private static final ExecutorService pool = + Executors.newCachedThreadPool(new AccumulatorThreadFactory()); + + static OutErrConsumers createDiscardingConsumers() { + return new OutErrConsumers(new DiscardingConsumer(), + new DiscardingConsumer()); + } + + static OutErrConsumers createAccumulatingConsumers() { + return new OutErrConsumers(new AccumulatingConsumer(), + new AccumulatingConsumer()); + } + + static OutErrConsumers createStreamingConsumers(OutputStream out, + OutputStream err) { + return new OutErrConsumers(new StreamingConsumer(out), + new StreamingConsumer(err)); + } + + static class OutErrConsumers { + + private final OutputConsumer out; + private final OutputConsumer err; + + private OutErrConsumers(final OutputConsumer out, final OutputConsumer err){ + this.out = out; + this.err = err; + } + + void registerInputs(InputStream outInput, InputStream errInput, boolean closeStreams){ + out.registerInput(outInput, closeStreams); + err.registerInput(errInput, closeStreams); + } + + void cancel() { + out.cancel(); + err.cancel(); + } + + void waitForCompletion() throws IOException { + out.waitForCompletion(); + err.waitForCompletion(); + } + + ByteArrayOutputStream getAccumulatedOut(){ + return out.getAccumulatedOut(); + } + + ByteArrayOutputStream getAccumulatedErr() { + return err.getAccumulatedOut(); + } + + void logConsumptionStrategy() { + // The creation methods guarantee that the consumption strategy is + // the same for out and err - doesn't matter whether we call out or err, + // let's pick out. + out.logConsumptionStrategy(); + } + + } + + /** + * This interface describes just one consumer, which consumes the + * InputStream provided by {@link #registerInput(InputStream, boolean)}. + * Implementations implement different consumption strategies. + */ + private static interface OutputConsumer { + /** + * Returns whatever the consumer accumulated internally, or + * {@link CommandResult#NO_OUTPUT_COLLECTED} if it doesn't accumulate + * any output. + * + * @see AccumulatingConsumer + */ + ByteArrayOutputStream getAccumulatedOut(); + + void logConsumptionStrategy(); + + void registerInput(InputStream in, boolean closeConsumer); + + void cancel(); + + void waitForCompletion() throws IOException; + } + + /** + * This consumer sends the input to a stream while consuming it. + */ + private static class StreamingConsumer extends FutureConsumption + implements OutputConsumer { + private OutputStream out; + + StreamingConsumer(OutputStream out) { + this.out = out; + } + + @Override + public ByteArrayOutputStream getAccumulatedOut() { + return CommandResult.NO_OUTPUT_COLLECTED; + } + + @Override + public void logConsumptionStrategy() { + log.finer("Output will be sent to streams provided by client"); + } + + @Override protected Runnable createConsumingAndClosingSink(InputStream in, + boolean closeConsumer) { + return new ClosingSink(in, out, closeConsumer); + } + } + + /** + * This consumer sends the input to a {@link ByteArrayOutputStream} + * while consuming it. This accumulated stream can be obtained by + * calling {@link #getAccumulatedOut()}. + */ + private static class AccumulatingConsumer extends FutureConsumption + implements OutputConsumer { + private ByteArrayOutputStream out = new ByteArrayOutputStream(); + + @Override + public ByteArrayOutputStream getAccumulatedOut() { + return out; + } + + @Override + public void logConsumptionStrategy() { + log.finer("Output will be accumulated (promptly read off) and returned"); + } + + @Override public Runnable createConsumingAndClosingSink(InputStream in, boolean closeConsumer) { + return new ClosingSink(in, out); + } + } + + /** + * This consumer just discards whatever it reads. + */ + private static class DiscardingConsumer extends FutureConsumption + implements OutputConsumer { + private DiscardingConsumer() { + } + + @Override + public ByteArrayOutputStream getAccumulatedOut() { + return CommandResult.NO_OUTPUT_COLLECTED; + } + + @Override + public void logConsumptionStrategy() { + log.finer("Output will be ignored"); + } + + @Override public Runnable createConsumingAndClosingSink(InputStream in, boolean closeConsumer) { + return new ClosingSink(in); + } + } + + /** + * A mixin that makes consumers active - this is where we kick of + * multithreading ({@link #registerInput(InputStream, boolean)}), cancel actions + * and wait for the consumers to complete. + */ + private abstract static class FutureConsumption implements OutputConsumer { + + private Future<?> future; + + @Override + public void registerInput(InputStream in, boolean closeConsumer){ + Runnable sink = createConsumingAndClosingSink(in, closeConsumer); + future = pool.submit(sink); + } + + protected abstract Runnable createConsumingAndClosingSink(InputStream in, boolean close); + + @Override + public void cancel() { + future.cancel(true); + } + + @Override + public void waitForCompletion() throws IOException { + boolean wasInterrupted = false; + try { + while (true) { + try { + future.get(); + break; + } catch (InterruptedException ie) { + wasInterrupted = true; + // continue waiting + } catch (ExecutionException ee) { + // Runnable threw a RuntimeException + Throwable nested = ee.getCause(); + if (nested instanceof RuntimeException) { + final RuntimeException re = (RuntimeException) nested; + // The stream sink classes, unfortunately, tunnel IOExceptions + // out of run() in a RuntimeException. If that's the case, + // unpack and re-throw the IOException. Otherwise, re-throw + // this unexpected RuntimeException + final Throwable cause = re.getCause(); + if (cause instanceof IOException) { + throw (IOException) cause; + } else { + throw re; + } + } else if (nested instanceof OutOfMemoryError) { + // OutOfMemoryError does not support exception chaining. + throw (OutOfMemoryError) nested; + } else if (nested instanceof Error) { + throw new Error("unhandled Error in worker thread", ee); + } else { + throw new RuntimeException("unknown execution problem", ee); + } + } + } + } finally { + // Read this for detailed explanation: + // http://www-128.ibm.com/developerworks/java/library/j-jtp05236.html + if (wasInterrupted) { + Thread.currentThread().interrupt(); // preserve interrupted status + } + } + } + } + + /** + * Factory which produces threads with a 32K stack size. + */ + private static class AccumulatorThreadFactory implements ThreadFactory { + + private static final int THREAD_STACK_SIZE = 32 * 1024; + + private static int threadInitNumber; + + private static synchronized int nextThreadNum() { + return threadInitNumber++; + } + + @Override + public Thread newThread(final Runnable runnable) { + final Thread t = + new Thread(null, + runnable, + "Command-Accumulator-Thread-" + nextThreadNum(), + THREAD_STACK_SIZE); + // Don't let this thread hold up JVM exit + t.setDaemon(true); + return t; + } + + } + + /** + * A sink that closes its input stream once its done. + */ + private static class ClosingSink implements Runnable { + + private final InputStream in; + private final OutputStream out; + private final Runnable sink; + private final boolean close; + + /** + * Creates a sink that will pump InputStream <code>in</code> + * into OutputStream <code>out</code>. + */ + ClosingSink(final InputStream in, OutputStream out) { + this(in, out, false); + } + + /** + * Creates a sink that will read <code>in</code> and discard it. + */ + ClosingSink(final InputStream in) { + this.sink = InputStreamSink.newRunnableSink(in); + this.in = in; + this.close = false; + this.out = null; + } + + ClosingSink(final InputStream in, OutputStream out, boolean close){ + this.sink = InputStreamSink.newRunnableSink(in, out); + this.in = in; + this.out = out; + this.close = close; + } + + + @Override + public void run() { + try { + sink.run(); + } finally { + silentClose(in); + if (close && out != null) { + silentClose(out); + } + } + } + + } + + /** + * Close the <code>in</code> stream and log a warning if anything happens. + */ + private static void silentClose(final Closeable closeable) { + try { + closeable.close(); + } catch (IOException ioe) { + String message = "Unexpected exception while closing input stream"; + log.log(Level.WARNING, message, ioe); + } + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/ExecFailedException.java b/src/main/java/com/google/devtools/build/lib/shell/ExecFailedException.java new file mode 100644 index 0000000..24f42a6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/ExecFailedException.java
@@ -0,0 +1,28 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +/** + * Thrown when a command could not even be executed by the JVM -- + * in particular, when {@link Runtime#exec(String[])} fails. + */ +public final class ExecFailedException extends CommandException { + + public ExecFailedException(Command command, final Throwable cause) { + super(command, cause); + } + + private static final long serialVersionUID = 2L; +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/FutureCommandResult.java b/src/main/java/com/google/devtools/build/lib/shell/FutureCommandResult.java new file mode 100644 index 0000000..3e1f5c9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/FutureCommandResult.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.shell; + +/** + * Supplier of the command result which additionally allows to check if + * the command already terminated. Implementing full fledged Future would + * be a much harder undertaking, so a bare minimum that makes this class still + * useful for asynchronous command execution is implemented. + */ +public interface FutureCommandResult { + /** + * Returns the result of command execution. If the process is not finished + * yet (as reported by {@link #isDone()}, the call will block until that + * process terminates. + * + * @return non-null result of command execution + * @throws AbnormalTerminationException if command execution failed + */ + CommandResult get() throws AbnormalTerminationException; + + /** + * Returns true if the process terminated, the command result is available + * and the call to {@link #get()} will not block. + * + * @return true if the process terminated + */ + boolean isDone(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/InputStreamSink.java b/src/main/java/com/google/devtools/build/lib/shell/InputStreamSink.java new file mode 100644 index 0000000..c35552b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/InputStreamSink.java
@@ -0,0 +1,133 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Provides sinks for input streams. Continuously read an input stream + * until the end-of-file is encountered. The stream may be redirected to + * an {@link OutputStream}, or discarded. + * <p> + * This class is useful for handing the {@code stdout} and {@code stderr} + * streams from a {@link Process} started with {@link Runtime#exec(String)}. + * If these streams are not consumed, the Process may block resulting in a + * deadlock. + * + * @see <a href="http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html"> + * JavaWorld: When Runtime.exec() won't</a> + */ +public final class InputStreamSink { + + /** + * Black hole into which bytes are sometimes discarded by {@link NullSink}. + * It is shared by all threads since the actual contents of the buffer + * are irrelevant. + */ + private static final byte[] DISCARD = new byte[4096]; + + // Supresses default constructor; ensures non-instantiability + private InputStreamSink() { + } + + /** + * A {@link Thread} which reads and discards data from an + * {@link InputStream}. + */ + private static class NullSink implements Runnable { + private final InputStream in; + + public NullSink(InputStream in) { + this.in = in; + } + + @Override + public void run() { + try { + try { + // Attempt to just skip all input + do { + in.skip(Integer.MAX_VALUE); + } while (in.read() != -1); // Need to test for EOF + } catch (IOException ioe) { + // Some streams throw IOException when skip() is called; + // resort to reading off all input with read(): + while (in.read(DISCARD) != -1) { + // no loop body + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + /** + * A {@link Thread} which reads data from an {@link InputStream}, + * and translates it into an {@link OutputStream}. + */ + private static class CopySink implements Runnable { + + private final InputStream in; + private final OutputStream out; + + public CopySink(InputStream in, OutputStream out) { + this.in = in; + this.out = out; + } + + @Override + public void run() { + try { + byte[] buffer = new byte[2048]; + int bytesRead; + while ((bytesRead = in.read(buffer)) >= 0) { + out.write(buffer, 0, bytesRead); + out.flush(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + /** + * Creates a {@link Runnable} which consumes the provided + * {@link InputStream} 'in', discarding its contents. + */ + public static Runnable newRunnableSink(InputStream in) { + if (in == null) { + throw new NullPointerException("in"); + } + return new NullSink(in); + } + + /** + * Creates a {@link Runnable} which copies everything from 'in' + * to 'out'. 'out' will be written to and flushed after each + * read from 'in'. However, 'out' will not be closed. + */ + public static Runnable newRunnableSink(InputStream in, OutputStream out) { + if (in == null) { + throw new NullPointerException("in"); + } + if (out == null) { + throw new NullPointerException("out"); + } + return new CopySink(in, out); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/Killable.java b/src/main/java/com/google/devtools/build/lib/shell/Killable.java new file mode 100644 index 0000000..66d1146 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/Killable.java
@@ -0,0 +1,31 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +/** + * Implementations encapsulate a running process that can be killed. + * In particular, here, it is used to wrap up a {@link Process} object + * and expose it to a {@link KillableObserver}. It is wrapped in this way + * so that the actual {@link Process} object can't be altered by + * a {@link KillableObserver}. + */ +public interface Killable { + + /** + * Kill this killable instance. + */ + void kill(); + +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/KillableObserver.java b/src/main/java/com/google/devtools/build/lib/shell/KillableObserver.java new file mode 100644 index 0000000..62d9aa0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/KillableObserver.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +/** + * Implementations of this interface observe, and potentially kill, + * a {@link Killable} object. This is the mechanism by which "kill" + * functionality is exposed to callers in the + * {@link Command#execute(byte[], KillableObserver, boolean)} method. + * + */ +public interface KillableObserver { + + /** + * <p>Begin observing the given {@link Killable}. This method must return + * promptly; until it returns, {@link Command#execute()} cannot complete. + * Implementations may wish to start a new {@link Thread} here to handle + * kill logic, and to interrupt or otherwise ask the thread to stop in the + * {@link #stopObserving(Killable)} method. See + * <a href="http://builder.com.com/5100-6370-5144546.html"> + * Interrupting Java threads</a> for notes on how to implement this + * correctly.</p> + * + * <p>Implementations may or may not be able to observe more than + * one {@link Killable} at a time; see javadoc for details.</p> + * + * @param killable killable to observer + */ + void startObserving(Killable killable); + + /** + * Stop observing the given {@link Killable}, since it is + * no longer active. + */ + void stopObserving(Killable killable); + +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/LogUtil.java b/src/main/java/com/google/devtools/build/lib/shell/LogUtil.java new file mode 100644 index 0000000..ab646f6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/LogUtil.java
@@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.shell; + +/** + * Utilities for logging. + */ +class LogUtil { + + private LogUtil() {} + + private final static int TRUNCATE_STRINGS_AT = 150; + + /** + * Make a string out of a byte array, and truncate it to a reasonable length. + * Useful for preventing logs from becoming excessively large. + */ + static String toTruncatedString(final byte[] bytes) { + if(bytes == null || bytes.length == 0) { + return ""; + } + /* + * Yes, we'll use the platform encoding here, and this is one of the rare + * cases where it makes sense. You want the logs to be encoded so that + * your platform tools (vi, emacs, cat) can render them, don't you? + * In practice, this means ISO-8859-1 or UTF-8, I guess. + */ + try { + if (bytes.length > TRUNCATE_STRINGS_AT) { + return new String(bytes, 0, TRUNCATE_STRINGS_AT) + + "[... truncated. original size was " + bytes.length + " bytes.]"; + } + return new String(bytes); + } catch (Exception e) { + /* + * In case encoding a binary string doesn't work for some reason, we + * don't want to bring a logging server down - do we? So we're paranoid. + */ + return "IOUtil.toTruncatedString: " + e.getMessage(); + } + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/ProcessKillable.java b/src/main/java/com/google/devtools/build/lib/shell/ProcessKillable.java new file mode 100644 index 0000000..5d0cb8f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/ProcessKillable.java
@@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +/** + * {@link Killable} implementation which simply wraps a + * {@link Process} instance. + */ +final class ProcessKillable implements Killable { + + private final Process process; + + ProcessKillable(final Process process) { + this.process = process; + } + + /** + * Calls {@link Process#destroy()}. + */ + @Override + public void kill() { + process.destroy(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/Shell.java b/src/main/java/com/google/devtools/build/lib/shell/Shell.java new file mode 100644 index 0000000..2cae24e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/Shell.java
@@ -0,0 +1,132 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +import java.util.logging.Logger; + +/** + * <p>Represents an OS shell, such as "cmd" on Windows or "sh" on Unix-like + * platforms. Currently, Linux and Windows XP are supported.</p> + * + * <p>This class encapsulates shell-specific logic, like how to + * create a command line that uses the shell to invoke another command. + */ +public abstract class Shell { + + private static final Logger log = + Logger.getLogger("com.google.devtools.build.lib.shell.Shell"); + + private static final Shell platformShell; + + static { + final String osName = System.getProperty("os.name"); + if ("Linux".equals(osName)) { + platformShell = new SHShell(); + } else if ("Windows XP".equals(osName)) { + platformShell = new WindowsCMDShell(); + } else { + log.severe("OS not supported; will not be able to execute commands"); + platformShell = null; + } + log.config("Loaded shell support '" + platformShell + + "' for OS '" + osName + "'"); + } + + private Shell() { + // do nothing + } + + /** + * @return {@link Shell} subclass appropriate for the current platform + * @throws UnsupportedOperationException if no such subclass exists + */ + public static Shell getPlatformShell() { + if (platformShell == null) { + throw new UnsupportedOperationException("OS is not supported"); + } + return platformShell; + } + + /** + * Creates a command line suitable for execution by + * {@link Runtime#exec(String[])} from the given command string, + * a command line which uses a shell appropriate for a particular + * platform to execute the command (e.g. "/bin/sh" on Linux). + * + * @param command command for which to create a command line + * @return String[] suitable for execution by + * {@link Runtime#exec(String[])} + */ + public abstract String[] shellify(final String command); + + + /** + * Represents the <code>sh</code> shell commonly found on Unix-like + * operating systems, including Linux. + */ + private static final class SHShell extends Shell { + + /** + * <p>Returns a command line which uses <code>cmd</code> to execute + * the {@link Command}. Given the command <code>foo bar baz</code>, + * for example, this will return a String array corresponding + * to the command line:</p> + * + * <p><code>/bin/sh -c "foo bar baz"</code></p> + * + * <p>That is, it always returns a 3-element array.</p> + * + * @param command command for which to create a command line + * @return String[] suitable for execution by + * {@link Runtime#exec(String[])} + */ + @Override public String[] shellify(final String command) { + if (command == null || command.length() == 0) { + throw new IllegalArgumentException("command is null or empty"); + } + return new String[] { "/bin/sh", "-c", command }; + } + + } + + /** + * Represents the Windows command shell <code>cmd</code>. + */ + private static final class WindowsCMDShell extends Shell { + + /** + * <p>Returns a command line which uses <code>cmd</code> to execute + * the {@link Command}. Given the command <code>foo bar baz</code>, + * for example, this will return a String array corresponding + * to the command line:</p> + * + * <p><code>cmd /S /C "foo bar baz"</code></p> + * + * <p>That is, it always returns a 4-element array.</p> + * + * @param command command for which to create a command line + * @return String[] suitable for execution by + * {@link Runtime#exec(String[])} + */ + @Override public String[] shellify(final String command) { + if (command == null || command.length() == 0) { + throw new IllegalArgumentException("command is null or empty"); + } + return new String[] { "cmd", "/S", "/C", command }; + } + + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/ShellUtils.java b/src/main/java/com/google/devtools/build/lib/shell/ShellUtils.java new file mode 100644 index 0000000..5157f34 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/ShellUtils.java
@@ -0,0 +1,145 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +import java.util.List; + +/** + * Utility functions for Bourne shell commands, including escaping and + * tokenizing. + */ +public abstract class ShellUtils { + + private ShellUtils() {} + + /** + * Characters that have no special meaning to the shell. + */ + private static final String SAFE_PUNCTUATION = "@%-_+:,./"; + + /** + * Quotes a word so that it can be used, without further quoting, + * as an argument (or part of an argument) in a shell command. + */ + public static String shellEscape(String word) { + int len = word.length(); + if (len == 0) { + // Empty string is a special case: needs to be quoted to ensure that it gets + // treated as a separate argument. + return "''"; + } + for (int ii = 0; ii < len; ii++) { + char c = word.charAt(ii); + // We do this positively so as to be sure we don't inadvertently forget + // any unsafe characters. + if (!Character.isLetterOrDigit(c) && SAFE_PUNCTUATION.indexOf(c) == -1) { + // replace() actually means "replace all". + return "'" + word.replace("'", "'\\''") + "'"; + } + } + return word; + } + + /** + * Given an argv array such as might be passed to execve(2), returns a string + * that can be copied and pasted into a Bourne shell for a similar effect. + */ + public static String prettyPrintArgv(List<String> argv) { + StringBuilder buf = new StringBuilder(); + for (String arg: argv) { + if (buf.length() > 0) { + buf.append(' '); + } + buf.append(shellEscape(arg)); + } + return buf.toString(); + } + + + /** + * Thrown by tokenize method if there is an error + */ + public static class TokenizationException extends Exception { + TokenizationException(String message) { + super(message); + } + } + + /** + * Populates the passed list of command-line options extracted from {@code + * optionString}, which is a string containing multiple options, delimited in + * a Bourne shell-like manner. + * + * @param options the list to be populated with tokens. + * @param optionString the string to be tokenized. + * @throws TokenizationException if there was an error (such as an + * unterminated quotation). + */ + public static void tokenize(List<String> options, String optionString) + throws TokenizationException { + // See test suite for examples. + // + // Note: backslash escapes the following character, except within a + // single-quoted region where it is literal. + + StringBuilder token = new StringBuilder(); + boolean forceToken = false; + char quotation = '\0'; // NUL, '\'' or '"' + for (int ii = 0, len = optionString.length(); ii < len; ii++) { + char c = optionString.charAt(ii); + if (quotation != '\0') { // in quotation + if (c == quotation) { // end of quotation + quotation = '\0'; + } else if (c == '\\' && quotation == '"') { // backslash in "-quotation + if (++ii == len) { + throw new TokenizationException("backslash at end of string"); + } + c = optionString.charAt(ii); + if (c != '\\' && c != '"') { + token.append('\\'); + } + token.append(c); + } else { // regular char, in quotation + token.append(c); + } + } else { // not in quotation + if (c == '\'' || c == '"') { // begin single/double quotation + quotation = c; + forceToken = true; + } else if (c == ' ' || c == '\t') { // space, not quoted + if (forceToken || token.length() > 0) { + options.add(token.toString()); + token = new StringBuilder(); + forceToken = false; + } + } else if (c == '\\') { // backslash, not quoted + if (++ii == len) { + throw new TokenizationException("backslash at end of string"); + } + token.append(optionString.charAt(ii)); + } else { // regular char, not quoted + token.append(c); + } + } + } + if (quotation != '\0') { + throw new TokenizationException("unterminated quotation"); + } + if (forceToken || token.length() > 0) { + options.add(token.toString()); + } + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/SimpleKillableObserver.java b/src/main/java/com/google/devtools/build/lib/shell/SimpleKillableObserver.java new file mode 100644 index 0000000..85794b8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/SimpleKillableObserver.java
@@ -0,0 +1,60 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +/** + * <p>A simple implementation of {@link KillableObserver} which can be told + * explicitly to kill its {@link Killable} by calling {@link #kill()}. This + * is the sort of functionality that callers might expect to find available + * on the {@link Command} class.</p> + * + * <p>Note that this class can only observe one {@link Killable} at a time; + * multiple instances should be used for concurrent calls to + * {@link Command#execute(byte[], KillableObserver, boolean)}.</p> + */ +public final class SimpleKillableObserver implements KillableObserver { + + private Killable killable; + + /** + * Does nothing except store a reference to the given {@link Killable}. + * + * @param killable {@link Killable} to kill + */ + public synchronized void startObserving(final Killable killable) { + this.killable = killable; + } + + /** + * Forgets reference to {@link Killable} provided to + * {@link #startObserving(Killable)} + */ + public synchronized void stopObserving(final Killable killable) { + if (!this.killable.equals(killable)) { + throw new IllegalStateException("start/stopObservering called with " + + "different Killables"); + } + this.killable = null; + } + + /** + * Calls {@link Killable#kill()} on the saved {@link Killable}. + */ + public synchronized void kill() { + if (killable != null) { + killable.kill(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/TerminationStatus.java b/src/main/java/com/google/devtools/build/lib/shell/TerminationStatus.java new file mode 100644 index 0000000..73616c4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/TerminationStatus.java
@@ -0,0 +1,162 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +/** + * Represents the termination status of a command. {@link Process#waitFor} is + * not very precisely specified, so this class encapsulates the interpretation + * of values returned by it. + * + * Caveat: due to the lossy encoding, it's not always possible to accurately + * distinguish signal and exit cases. In particular, processes that exit with + * a value within the interval [129, 191] will be mistaken for having been + * terminated by a signal. + * + * Instances are immutable. + */ +public final class TerminationStatus { + + private final int waitResult; + + /** + * Values taken from the glibc strsignal(3) function. + */ + private static final String[] SIGNAL_STRINGS = { + null, + "Hangup", + "Interrupt", + "Quit", + "Illegal instruction", + "Trace/breakpoint trap", + "Aborted", + "Bus error", + "Floating point exception", + "Killed", + "User defined signal 1", + "Segmentation fault", + "User defined signal 2", + "Broken pipe", + "Alarm clock", + "Terminated", + "Stack fault", + "Child exited", + "Continued", + "Stopped (signal)", + "Stopped", + "Stopped (tty input)", + "Stopped (tty output)", + "Urgent I/O condition", + "CPU time limit exceeded", + "File size limit exceeded", + "Virtual timer expired", + "Profiling timer expired", + "Window changed", + "I/O possible", + "Power failure", + "Bad system call", + }; + + private static String getSignalString(int signum) { + return signum > 0 && signum < SIGNAL_STRINGS.length + ? SIGNAL_STRINGS[signum] + : "Signal " + signum; + } + + /** + * Construct a TerminationStatus instance from a Process waitFor code. + * + * @param waitResult the value returned by {@link java.lang.Process#waitFor}. + */ + public TerminationStatus(int waitResult) { + this.waitResult = waitResult; + } + + /** + * Returns the "raw" result returned by Process.waitFor. + */ + int getRawResult() { + return waitResult; + } + + /** + * Returns true iff the process exited with code 0. + */ + public boolean success() { + return exited() && getExitCode() == 0; + } + + // We're relying on undocumented behaviour of Process.waitFor, specifically + // that waitResult is the exit status when the process returns normally, or + // 128+signalnumber when the process is terminated by a signal. We further + // assume that value signal numbers fall in the interval [1, 63]. + private static final int SIGNAL_1 = 128 + 1; + private static final int SIGNAL_63 = 128 + 63; + + /** + * Returns true iff the process exited normally. + */ + public boolean exited() { + return waitResult < SIGNAL_1 || waitResult > SIGNAL_63; + } + + /** + * Returns the exit code of the subprocess. Undefined if exited() is false. + */ + public int getExitCode() { + if (!exited()) { + throw new IllegalStateException("getExitCode() not defined"); + } + return waitResult; + } + + /** + * Returns the number of the signal that terminated the process. Undefined + * if exited() returns true. + */ + public int getTerminatingSignal() { + if (exited()) { + throw new IllegalStateException("getTerminatingSignal() not defined"); + } + return waitResult - SIGNAL_1 + 1; + } + + /** + * Returns a short string describing the termination status. + * e.g. "Exit 1" or "Hangup". + */ + public String toShortString() { + return exited() + ? ("Exit " + getExitCode()) + : (getSignalString(getTerminatingSignal())); + } + + @Override + public String toString() { + return exited() + ? ("Process exited with status " + getExitCode()) + : ("Process terminated by signal " + getTerminatingSignal()); + } + + @Override + public int hashCode() { + return waitResult; + } + + @Override + public boolean equals(Object other) { + return other instanceof TerminationStatus && + ((TerminationStatus) other).waitResult == this.waitResult; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/shell/TimeoutKillableObserver.java b/src/main/java/com/google/devtools/build/lib/shell/TimeoutKillableObserver.java new file mode 100644 index 0000000..c2ed033 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/TimeoutKillableObserver.java
@@ -0,0 +1,102 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * <p>{@link KillableObserver} implementation which will kill its observed + * {@link Killable} if it is still being observed after a given amount + * of time has elapsed.</p> + * + * <p>Note that this class can only observe one {@link Killable} at a time; + * multiple instances should be used for concurrent calls to + * {@link Command#execute(byte[], KillableObserver, boolean)}.</p> + */ +public final class TimeoutKillableObserver implements KillableObserver { + + private static final Logger log = + Logger.getLogger(TimeoutKillableObserver.class.getCanonicalName()); + + private final long timeoutMS; + private Killable killable; + private SleeperThread sleeperThread; + private boolean timedOut; + + // TODO(bazel-team): I'd like to use ThreadPool2, but it doesn't currently + // provide a way to interrupt a thread + + public TimeoutKillableObserver(final long timeoutMS) { + this.timeoutMS = timeoutMS; + } + + /** + * Starts a new {@link Thread} to wait for the timeout period. This is + * interrupted by the {@link #stopObserving(Killable)} method. + * + * @param killable killable to kill when the timeout period expires + */ + @Override + public synchronized void startObserving(final Killable killable) { + this.timedOut = false; + this.killable = killable; + this.sleeperThread = new SleeperThread(); + this.sleeperThread.start(); + } + + @Override + public synchronized void stopObserving(final Killable killable) { + if (!this.killable.equals(killable)) { + throw new IllegalStateException("start/stopObservering called with " + + "different Killables"); + } + if (sleeperThread.isAlive()) { + sleeperThread.interrupt(); + } + this.killable = null; + sleeperThread = null; + } + + private final class SleeperThread extends Thread { + @Override public void run() { + try { + if (log.isLoggable(Level.FINE)) { + log.fine("Waiting for " + timeoutMS + "ms to kill process"); + } + Thread.sleep(timeoutMS); + // timeout expired; kill it + synchronized (TimeoutKillableObserver.this) { + if (killable != null) { + log.fine("Killing process"); + killable.kill(); + timedOut = true; + } + } + } catch (InterruptedException ie) { + // continue -- process finished before timeout + log.fine("Wait interrupted since process finished; continuing..."); + } + } + } + + /** + * Returns true if the observed process was killed by this observer. + */ + public synchronized boolean hasTimedOut() { + // synchronized needed for memory model visibility. + return timedOut; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java new file mode 100644 index 0000000..6dcc224 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java
@@ -0,0 +1,177 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.packages.CachingPackageLocator; +import com.google.devtools.build.lib.packages.RuleClassProvider; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.syntax.BuildFileAST; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.Nullable; + +/** + * A SkyFunction for {@link ASTFileLookupValue}s. Tries to locate a file and load it as a + * syntax tree and cache the resulting {@link BuildFileAST}. If the file doesn't exist + * the function doesn't fail but returns a specific NO_FILE ASTLookupValue. + */ +public class ASTFileLookupFunction implements SkyFunction { + + private abstract static class FileLookupResult { + /** Returns whether the file lookup was successful. */ + public abstract boolean lookupSuccessful(); + + /** If {@code lookupSuccessful()}, returns the {@link RootedPath} to the file. */ + public abstract RootedPath rootedPath(); + + static FileLookupResult noFile() { + return UnsuccessfulFileResult.INSTANCE; + } + + static FileLookupResult file(RootedPath rootedPath) { + return new SuccessfulFileResult(rootedPath); + } + + private static class SuccessfulFileResult extends FileLookupResult { + private final RootedPath rootedPath; + + private SuccessfulFileResult(RootedPath rootedPath) { + this.rootedPath = rootedPath; + } + + @Override + public boolean lookupSuccessful() { + return true; + } + + @Override + public RootedPath rootedPath() { + return rootedPath; + } + } + + private static class UnsuccessfulFileResult extends FileLookupResult { + private static final UnsuccessfulFileResult INSTANCE = new UnsuccessfulFileResult(); + private UnsuccessfulFileResult() { + } + + @Override + public boolean lookupSuccessful() { + return false; + } + + @Override + public RootedPath rootedPath() { + throw new IllegalStateException("unsucessful lookup"); + } + } + } + + private final AtomicReference<PathPackageLocator> pkgLocator; + private final RuleClassProvider ruleClassProvider; + private final CachingPackageLocator packageManager; + + public ASTFileLookupFunction(AtomicReference<PathPackageLocator> pkgLocator, + CachingPackageLocator packageManager, + RuleClassProvider ruleClassProvider) { + this.pkgLocator = pkgLocator; + this.packageManager = packageManager; + this.ruleClassProvider = ruleClassProvider; + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, + InterruptedException { + PathFragment astFilePathFragment = (PathFragment) skyKey.argument(); + FileLookupResult lookupResult = getASTFile(env, astFilePathFragment); + if (lookupResult == null) { + return null; + } + + BuildFileAST ast = null; + if (!lookupResult.lookupSuccessful()) { + // Return the specific NO_FILE ASTLookupValue instance if no file was found. + return ASTFileLookupValue.NO_FILE; + } else { + Path path = lookupResult.rootedPath().asPath(); + // Skylark files end with bzl. + boolean parseAsSkylark = astFilePathFragment.getPathString().endsWith(".bzl"); + try { + ast = parseAsSkylark + ? BuildFileAST.parseSkylarkFile(path, env.getListener(), + packageManager, ruleClassProvider.getSkylarkValidationEnvironment().clone()) + : BuildFileAST.parseBuildFile(path, env.getListener(), + packageManager, false); + } catch (IOException e) { + throw new ASTLookupFunctionException(new ErrorReadingSkylarkExtensionException( + e.getMessage()), Transience.TRANSIENT); + } + } + + return new ASTFileLookupValue(ast); + } + + private FileLookupResult getASTFile(Environment env, PathFragment astFilePathFragment) + throws ASTLookupFunctionException { + for (Path packagePathEntry : pkgLocator.get().getPathEntries()) { + RootedPath rootedPath = RootedPath.toRootedPath(packagePathEntry, astFilePathFragment); + SkyKey fileSkyKey = FileValue.key(rootedPath); + FileValue fileValue = null; + try { + fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class, + FileSymlinkCycleException.class, InconsistentFilesystemException.class); + } catch (IOException | FileSymlinkCycleException e) { + throw new ASTLookupFunctionException(new ErrorReadingSkylarkExtensionException( + e.getMessage()), Transience.PERSISTENT); + } catch (InconsistentFilesystemException e) { + throw new ASTLookupFunctionException(e, Transience.PERSISTENT); + } + if (fileValue == null) { + return null; + } + if (fileValue.isFile()) { + return FileLookupResult.file(rootedPath); + } + } + return FileLookupResult.noFile(); + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + private static final class ASTLookupFunctionException extends SkyFunctionException { + private ASTLookupFunctionException(ErrorReadingSkylarkExtensionException e, + Transience transience) { + super(e, transience); + } + + private ASTLookupFunctionException(InconsistentFilesystemException e, Transience transience) { + super(e, transience); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupValue.java new file mode 100644 index 0000000..1061c86 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupValue.java
@@ -0,0 +1,61 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.syntax.BuildFileAST; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import javax.annotation.Nullable; + +/** + * A value that represents an AST file lookup result. + */ +public class ASTFileLookupValue implements SkyValue { + + static final ASTFileLookupValue NO_FILE = new ASTFileLookupValue(null); + + @Nullable private final BuildFileAST ast; + + public ASTFileLookupValue(@Nullable BuildFileAST ast) { + this.ast = ast; + } + + /** + * Returns the original AST file. + */ + @Nullable public BuildFileAST getAST() { + return ast; + } + + static void checkInputArgument(PathFragment astFilePathFragment) throws ASTLookupInputException { + if (astFilePathFragment.isAbsolute()) { + throw new ASTLookupInputException(String.format( + "Input file '%s' cannot be an absolute path.", astFilePathFragment)); + } + } + + static SkyKey key(PathFragment astFilePathFragment) throws ASTLookupInputException { + checkInputArgument(astFilePathFragment); + return new SkyKey(SkyFunctions.AST_FILE_LOOKUP, astFilePathFragment); + } + + static final class ASTLookupInputException extends Exception { + private ASTLookupInputException(String msg) { + super(msg); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/AbstractLabelCycleReporter.java b/src/main/java/com/google/devtools/build/lib/skyframe/AbstractLabelCycleReporter.java new file mode 100644 index 0000000..797f158 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/AbstractLabelCycleReporter.java
@@ -0,0 +1,130 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.CycleInfo; +import com.google.devtools.build.skyframe.CyclesReporter; +import com.google.devtools.build.skyframe.SkyKey; + +/** Reports cycles between skyframe values whose keys contains {@link Label}s. */ +abstract class AbstractLabelCycleReporter implements CyclesReporter.SingleCycleReporter { + + private final LoadedPackageProvider loadedPackageProvider; + + AbstractLabelCycleReporter(LoadedPackageProvider loadedPackageProvider) { + this.loadedPackageProvider = loadedPackageProvider; + } + + /** Returns the String representation of the {@code SkyKey}. */ + protected abstract String prettyPrint(SkyKey key); + + /** Returns the associated Label of the SkyKey. */ + protected abstract Label getLabel(SkyKey key); + + protected abstract boolean canReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo); + + protected String getAdditionalMessageAboutCycle(SkyKey topLevelKey, CycleInfo cycleInfo) { + return ""; + } + + @Override + public boolean maybeReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo, + boolean alreadyReported, EventHandler eventHandler) { + Preconditions.checkNotNull(eventHandler); + if (!canReportCycle(topLevelKey, cycleInfo)) { + return false; + } + + if (alreadyReported) { + Label label = getLabel(topLevelKey); + Target target = getTargetForLabel(label); + eventHandler.handle(Event.error(target.getLocation(), + "in " + target.getTargetKind() + " " + label + + ": cycle in dependency graph: target depends on an already-reported cycle")); + } else { + StringBuilder cycleMessage = new StringBuilder("cycle in dependency graph:"); + ImmutableList<SkyKey> pathToCycle = cycleInfo.getPathToCycle(); + ImmutableList<SkyKey> cycle = cycleInfo.getCycle(); + for (SkyKey value : pathToCycle) { + cycleMessage.append("\n "); + cycleMessage.append(prettyPrint(value)); + } + + SkyKey cycleValue = printCycle(cycle, cycleMessage, new Function<SkyKey, String>() { + @Override + public String apply(SkyKey input) { + return prettyPrint(input); + } + }); + + cycleMessage.append(getAdditionalMessageAboutCycle(topLevelKey, cycleInfo)); + + Label label = getLabel(cycleValue); + Target target = getTargetForLabel(label); + eventHandler.handle( + Event.error(target.getLocation(), "in " + target.getTargetKind() + " " + label + + ": " + cycleMessage.toString())); + } + + return true; + } + + /** + * Prints the SkyKey-s in cycle into cycleMessage using the print function. + */ + static SkyKey printCycle(ImmutableList<SkyKey> cycle, StringBuilder cycleMessage, + Function<SkyKey, String> printFunction) { + Iterable<SkyKey> valuesToPrint = cycle.size() > 1 + ? Iterables.concat(cycle, ImmutableList.of(cycle.get(0))) : cycle; + SkyKey cycleValue = null; + for (SkyKey value : valuesToPrint) { + if (cycleValue == null) { + cycleValue = value; + } + if (value == cycleValue) { + cycleMessage.append("\n * "); + } else { + cycleMessage.append("\n "); + } + cycleMessage.append(printFunction.apply(value)); + } + + if (cycle.size() == 1) { + cycleMessage.append(" [self-edge]"); + } + + return cycleValue; + } + + protected final Target getTargetForLabel(Label label) { + try { + return loadedPackageProvider.getLoadedTarget(label); + } catch (NoSuchThingException e) { + // This method is used for getting the target from a label in a circular dependency. + // If we have a cycle that means that we need to have accessed the target (to get its + // dependencies). So all the labels in a dependency cycle need to exist. + throw new IllegalStateException(e); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionArtifactCycleReporter.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionArtifactCycleReporter.java new file mode 100644 index 0000000..3105539 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionArtifactCycleReporter.java
@@ -0,0 +1,77 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.analysis.LabelAndConfiguration; +import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider; +import com.google.devtools.build.lib.skyframe.ArtifactValue.OwnedArtifact; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.CycleInfo; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; + +/** + * Reports cycles between Actions and Artifacts. These indicates cycles within a rule. + */ +public class ActionArtifactCycleReporter extends AbstractLabelCycleReporter { + + private static final Predicate<SkyKey> IS_ARTIFACT_OR_ACTION_SKY_KEY = Predicates.or( + SkyFunctions.isSkyFunction(SkyFunctions.ARTIFACT), + SkyFunctions.isSkyFunction(SkyFunctions.ACTION_EXECUTION), + SkyFunctions.isSkyFunction(SkyFunctions.TARGET_COMPLETION)); + + ActionArtifactCycleReporter(LoadedPackageProvider loadedPackageProvider) { + super(loadedPackageProvider); + } + + @Override + protected String prettyPrint(SkyKey key) { + return prettyPrint(key.functionName(), key.argument()); + } + + private String prettyPrint(SkyFunctionName skyFunctionName, Object arg) { + if (arg instanceof OwnedArtifact) { + return "file: " + ((OwnedArtifact) arg).getArtifact().getRootRelativePathString(); + } else if (arg instanceof Action) { + return "action: " + ((Action) arg).getMnemonic(); + } else if (arg instanceof LabelAndConfiguration + && skyFunctionName == SkyFunctions.TARGET_COMPLETION) { + return "configured target: " + ((LabelAndConfiguration) arg).getLabel().toString(); + } + throw new IllegalStateException( + "Argument is not Action, TargetCompletion, or OwnedArtifact: " + arg); + } + + @Override + protected Label getLabel(SkyKey key) { + Object arg = key.argument(); + if (arg instanceof OwnedArtifact) { + return ((OwnedArtifact) arg).getArtifact().getOwner(); + } else if (arg instanceof Action) { + return ((Action) arg).getOwner().getLabel(); + } + throw new IllegalStateException("Argument is not Action or OwnedArtifact: " + arg); + } + + @Override + protected boolean canReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo) { + return IS_ARTIFACT_OR_ACTION_SKY_KEY.apply(topLevelKey) + && Iterables.all(cycleInfo.getCycle(), IS_ARTIFACT_OR_ACTION_SKY_KEY); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java new file mode 100644 index 0000000..1420860 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java
@@ -0,0 +1,338 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.Collections2; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionCacheChecker.Token; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.AlreadyReportedActionExecutionException; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.MissingInputFileException; +import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit; +import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.skyframe.ValueOrException2; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * A builder for {@link ActionExecutionValue}s. + */ +public class ActionExecutionFunction implements SkyFunction { + + private static final Predicate<Artifact> IS_SOURCE_ARTIFACT = new Predicate<Artifact>() { + @Override + public boolean apply(Artifact input) { + return input.isSourceArtifact(); + } + }; + + private final SkyframeActionExecutor skyframeActionExecutor; + private final TimestampGranularityMonitor tsgm; + + public ActionExecutionFunction(SkyframeActionExecutor skyframeActionExecutor, + TimestampGranularityMonitor tsgm) { + this.skyframeActionExecutor = skyframeActionExecutor; + this.tsgm = tsgm; + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws ActionExecutionFunctionException, + InterruptedException { + Action action = (Action) skyKey.argument(); + Map<Artifact, FileArtifactValue> inputArtifactData = null; + Map<Artifact, Collection<Artifact>> expandedMiddlemen = null; + boolean alreadyRan = skyframeActionExecutor.probeActionExecution(action); + try { + Pair<Map<Artifact, FileArtifactValue>, Map<Artifact, Collection<Artifact>>> checkedInputs = + checkInputs(env, action, alreadyRan); // Declare deps on known inputs to action. + + if (checkedInputs != null) { + inputArtifactData = checkedInputs.first; + expandedMiddlemen = checkedInputs.second; + } + } catch (ActionExecutionException e) { + throw new ActionExecutionFunctionException(e); + } + // TODO(bazel-team): Non-volatile NotifyOnActionCacheHit actions perform worse in Skyframe than + // legacy when they are not at the top of the action graph. In legacy, they are stored + // separately, so notifying non-dirty actions is cheap. In Skyframe, they depend on the + // BUILD_ID, forcing invalidation of upward transitive closure on each build. + if (action.isVolatile() || action instanceof NotifyOnActionCacheHit) { + // Volatile build actions may need to execute even if none of their known inputs have changed. + // Depending on the buildID ensure that these actions have a chance to execute. + PrecomputedValue.BUILD_ID.get(env); + } + if (env.valuesMissing()) { + return null; + } + + ActionExecutionValue result; + try { + result = checkCacheAndExecuteIfNeeded(action, inputArtifactData, expandedMiddlemen, env); + } catch (ActionExecutionException e) { + // In this case we do not report the error to the action reporter because we have already + // done it in SkyframeExecutor.reportErrorIfNotAbortingMode() method. That method + // prints the error in the top-level reporter and also dumps the recorded StdErr for the + // action. Label can be null in the case of, e.g., the SystemActionOwner (for build-info.txt). + throw new ActionExecutionFunctionException(new AlreadyReportedActionExecutionException(e)); + } finally { + declareAdditionalDependencies(env, action); + } + if (env.valuesMissing()) { + return null; + } + + return result; + } + + private ActionExecutionValue checkCacheAndExecuteIfNeeded( + Action action, + Map<Artifact, FileArtifactValue> inputArtifactData, + Map<Artifact, Collection<Artifact>> expandedMiddlemen, + Environment env) throws ActionExecutionException, InterruptedException { + // Don't initialize the cache if the result has already been computed and this is just a + // rerun. + FileAndMetadataCache fileAndMetadataCache = null; + MetadataHandler metadataHandler = null; + Token token = null; + long actionStartTime = System.nanoTime(); + // inputArtifactData is null exactly when we know that the execution result was already + // computed on a prior run of this SkyFunction. If it is null we don't need to initialize + // anything -- we will get the result directly from SkyframeActionExecutor's cache. + if (inputArtifactData != null) { + // Check action cache to see if we need to execute anything. Checking the action cache only + // needs to happen on the first run, since a cache hit means we'll return immediately, and + // there'll be no second run. + fileAndMetadataCache = new FileAndMetadataCache( + inputArtifactData, + expandedMiddlemen, + skyframeActionExecutor.getExecRoot(), + action.getOutputs(), + // Only give the metadata cache the ability to look up Skyframe values if the action + // might have undeclared inputs. If those undeclared inputs are generated, they are + // present in Skyframe, so we can save a stat by looking them up directly. + action.discoversInputs() ? env : null, + tsgm); + metadataHandler = + skyframeActionExecutor.constructMetadataHandler(fileAndMetadataCache); + token = skyframeActionExecutor.checkActionCache(action, metadataHandler, actionStartTime); + } + if (token == null && inputArtifactData != null) { + // We got a hit from the action cache -- no need to execute. + return new ActionExecutionValue( + fileAndMetadataCache.getOutputData(), + fileAndMetadataCache.getAdditionalOutputData()); + } else { + ActionExecutionContext actionExecutionContext = null; + if (inputArtifactData != null) { + actionExecutionContext = skyframeActionExecutor.constructActionExecutionContext( + fileAndMetadataCache, + metadataHandler); + if (action.discoversInputs()) { + skyframeActionExecutor.discoverInputs(action, actionExecutionContext); + } + } + // If this is the second time we are here (because the action discovers inputs, and we had + // to restart the value builder after declaring our dependence on newly discovered inputs), + // the result returned here is the already-computed result from the first run. + // Similarly, if this is a shared action and the other action is the one that executed, we + // must use that other action's value, provided here, since it is populated with metadata + // for the outputs. + // If this action was not shared and this is the first run of the action, this returned + // result was computed during the call. + return skyframeActionExecutor.executeAction(action, fileAndMetadataCache, token, + actionStartTime, actionExecutionContext); + } + } + + private static Iterable<SkyKey> toKeys(Iterable<Artifact> inputs, + Iterable<Artifact> mandatoryInputs) { + if (mandatoryInputs == null) { + // This is a non inputs-discovering action, so no need to distinguish mandatory from regular + // inputs. + return Iterables.transform(inputs, new Function<Artifact, SkyKey>() { + @Override + public SkyKey apply(Artifact artifact) { + return ArtifactValue.key(artifact, true); + } + }); + } else { + Collection<SkyKey> discoveredArtifacts = new HashSet<>(); + Set<Artifact> mandatory = Sets.newHashSet(mandatoryInputs); + for (Artifact artifact : inputs) { + discoveredArtifacts.add(ArtifactValue.key(artifact, mandatory.contains(artifact))); + } + + // In case the action violates the invariant that getInputs() is a superset of + // getMandatoryInputs(), explicitly add the mandatory inputs. See bug about an + // "action not in canonical form" error message. Also note that we may add Skyframe edges on + // these potentially stale deps due to the way loading inputs from the action cache functions. + // In practice, this is safe since C++ actions (the only ones which discover inputs) only add + // possibly stale inputs on source artifacts, which we treat as non-mandatory. + for (Artifact artifact : mandatory) { + discoveredArtifacts.add(ArtifactValue.key(artifact, true)); + } + return discoveredArtifacts; + } + } + + /** + * Declare dependency on all known inputs of action. Throws exception if any are known to be + * missing. Some inputs may not yet be in the graph, in which case the builder should abort. + */ + private Pair<Map<Artifact, FileArtifactValue>, Map<Artifact, Collection<Artifact>>> checkInputs( + Environment env, Action action, boolean alreadyRan) throws ActionExecutionException { + Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> inputDeps = + env.getValuesOrThrow(toKeys(action.getInputs(), action.discoversInputs() + ? action.getMandatoryInputs() : null), MissingInputFileException.class, + ActionExecutionException.class); + + // If the action was already run, then break out early. This avoids the cost of constructing the + // input map and expanded middlemen if they're not going to be used. + if (alreadyRan) { + return null; + } + + int missingCount = 0; + int actionFailures = 0; + boolean catastrophe = false; + // Only populate input data if we have the input values, otherwise they'll just go unused. + // We still want to loop through the inputs to collect missing deps errors. During the + // evaluator "error bubbling", we may get one last chance at reporting errors even though + // some deps are stilling missing. + boolean populateInputData = !env.valuesMissing(); + NestedSetBuilder<Label> rootCauses = NestedSetBuilder.stableOrder(); + Map<Artifact, FileArtifactValue> inputArtifactData = + new HashMap<>(populateInputData ? inputDeps.size() : 0); + Map<Artifact, Collection<Artifact>> expandedMiddlemen = + new HashMap<>(populateInputData ? 128 : 0); + + ActionExecutionException firstActionExecutionException = null; + for (Map.Entry<SkyKey, ValueOrException2<MissingInputFileException, + ActionExecutionException>> depsEntry : inputDeps.entrySet()) { + Artifact input = ArtifactValue.artifact(depsEntry.getKey()); + try { + ArtifactValue value = (ArtifactValue) depsEntry.getValue().get(); + if (populateInputData && value instanceof AggregatingArtifactValue) { + AggregatingArtifactValue aggregatingValue = (AggregatingArtifactValue) value; + for (Pair<Artifact, FileArtifactValue> entry : aggregatingValue.getInputs()) { + inputArtifactData.put(entry.first, entry.second); + } + // We have to cache the "digest" of the aggregating value itself, because the action cache + // checker may want it. + inputArtifactData.put(input, aggregatingValue.getSelfData()); + expandedMiddlemen.put(input, + Collections2.transform(aggregatingValue.getInputs(), + Pair.<Artifact, FileArtifactValue>firstFunction())); + } else if (populateInputData && value instanceof FileArtifactValue) { + // TODO(bazel-team): Make sure middleman "virtual" artifact data is properly processed. + inputArtifactData.put(input, (FileArtifactValue) value); + } + } catch (MissingInputFileException e) { + missingCount++; + if (input.getOwner() != null) { + rootCauses.add(input.getOwner()); + } + } catch (ActionExecutionException e) { + actionFailures++; + if (firstActionExecutionException == null) { + firstActionExecutionException = e; + } + catastrophe = catastrophe || e.isCatastrophe(); + rootCauses.addTransitive(e.getRootCauses()); + } + } + // We need to rethrow first exception because it can contain useful error message + if (firstActionExecutionException != null) { + if (missingCount == 0 && actionFailures == 1) { + // In the case a single action failed, just propagate the exception upward. This avoids + // having to copy the root causes to the upwards transitive closure. + throw firstActionExecutionException; + } + throw new ActionExecutionException(firstActionExecutionException.getMessage(), + firstActionExecutionException.getCause(), action, rootCauses.build(), catastrophe); + } + + if (missingCount > 0) { + for (Label missingInput : rootCauses.build()) { + env.getListener().handle(Event.error(action.getOwner().getLocation(), String.format( + "%s: missing input file '%s'", action.getOwner().getLabel(), missingInput))); + } + throw new ActionExecutionException(missingCount + " input file(s) do not exist", action, + rootCauses.build(), /*catastrophe=*/false); + } + return Pair.of( + Collections.unmodifiableMap(inputArtifactData), + Collections.unmodifiableMap(expandedMiddlemen)); + } + + private static void declareAdditionalDependencies(Environment env, Action action) { + if (action.discoversInputs()) { + // TODO(bazel-team): Should this be all inputs, or just source files? + env.getValues(toKeys(Iterables.filter(action.getInputs(), IS_SOURCE_ARTIFACT), + action.getMandatoryInputs())); + } + } + + /** + * All info/warning messages associated with actions should be always displayed. + */ + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link ActionExecutionFunction#compute}. + */ + private static final class ActionExecutionFunctionException extends SkyFunctionException { + + private final ActionExecutionException actionException; + + public ActionExecutionFunctionException(ActionExecutionException e) { + // We conservatively assume that the error is transient. We don't have enough information to + // distinguish non-transient errors (e.g. compilation error from a deterministic compiler) + // from transient ones (e.g. IO error). + // TODO(bazel-team): Have ActionExecutionExceptions declare their transience. + super(e, Transience.TRANSIENT); + this.actionException = e; + } + + @Override + public boolean isCatastrophic() { + return actionException.isCatastrophe(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionInactivityWatchdog.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionInactivityWatchdog.java new file mode 100644 index 0000000..87e3e0d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionInactivityWatchdog.java
@@ -0,0 +1,180 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * An object that can monitor whether actions are getting completed in a timely manner. + * + * <p>If there's nothing happening for a while, a background thread will print (and update) the + * "Still waiting for N actions to complete..." message. + */ +public final class ActionExecutionInactivityWatchdog { + + /** An object used in monitoring action execution inactivity. */ + public interface InactivityMonitor { + + /** Returns whether action execution has started. */ + boolean hasStarted(); + + /** Returns the number of enqueued but not yet completed actions. */ + int getPending(); + + /** + * Waits for any action to complete, or the timeout to elapse. + * + * <p>The thread must wait at least for the specified timeout, unless some action completes in + * the meantime. It's not allowed to return 0 too early. + * + * <p>Note that it's acceptable to return (any value) later than specified by the timeout. + * + * @return the number of actions completed during the wait + */ + int waitForNextCompletion(int timeoutMilliseconds) throws InterruptedException; + } + + /** An object that the watchdog can report inactivity to. */ + public interface InactivityReporter { + + /** + * Report that actions are not getting completed in a timely manner. + * + * <p>Inactivity is typically not reported if tests with streaming output are being run. + */ + void maybeReportInactivity(); + } + + @VisibleForTesting + interface Sleep { + void sleep(int durationMilliseconds) throws InterruptedException; + } + + private static final class WaitTime { + private final int progressIntervalFlagValue; + private int prev; + + public WaitTime(int progressIntervalFlagValue) { + this.progressIntervalFlagValue = progressIntervalFlagValue; + } + + public void reset() { + prev = 0; + } + + public int next() { + prev = ActionExecutionStatusReporter.getWaitTime(progressIntervalFlagValue, prev); + return prev; + } + } + + private final AtomicBoolean isRunning = new AtomicBoolean(false); + private final InactivityMonitor monitor; + private final InactivityReporter reporter; + private final Sleep sleeper; + private final Thread thread; + private final WaitTime waitTime; + + public ActionExecutionInactivityWatchdog(InactivityMonitor monitor, InactivityReporter reporter, + int progressIntervalFlagValue) { + this(monitor, reporter, progressIntervalFlagValue, new Sleep() { + @Override + public void sleep(int durationMilliseconds) throws InterruptedException { + Thread.sleep(durationMilliseconds); + } + }); + } + + @VisibleForTesting + public ActionExecutionInactivityWatchdog(InactivityMonitor monitor, InactivityReporter reporter, + int progressIntervalFlagValue, Sleep sleeper) { + this.monitor = Preconditions.checkNotNull(monitor); + this.reporter = Preconditions.checkNotNull(reporter); + this.sleeper = Preconditions.checkNotNull(sleeper); + this.waitTime = new WaitTime(progressIntervalFlagValue); + this.thread = new Thread(new Runnable() { + @Override + public void run() { + enterWatchdogLoop(); + } + }); + this.thread.setDaemon(true); + this.thread.setName("action-execution-watchdog"); + } + + /** Starts the watchdog thread. This method should only be called once. */ + public void start() { + Preconditions.checkState(!isRunning.getAndSet(true)); + thread.start(); + } + + /** + * Stops the watchdog thread. This method should only be called once. + * + * <p>The method waits for the thread to terminate. If the caller thread is interrupted + * in the meantime, the interrupted status will be set. + */ + public void stop() { + Preconditions.checkState(isRunning.getAndSet(false)); + thread.interrupt(); + try { + thread.join(); + } catch (InterruptedException e) { + // When Thread.join throws, the interrupted status is cleared. We need to set it again. + Thread.currentThread().interrupt(); + } + } + + private void enterWatchdogLoop() { + while (isRunning.get()) { + try { + // Wait a while for any SkyFunction to finish. The returned number indicates how many + // actions completed during the wait. It's possible that this is more than 1, since + // this thread may not immediately regain control. + int completedActions = monitor.waitForNextCompletion(waitTime.next() * 1000); + if (!isRunning.get()) { + break; + } + + int pending = monitor.getPending(); + if (!monitor.hasStarted() || completedActions > 0 || pending == 0) { + // If no keys have been enqueued yet (execution hasn't started), or some actions + // were completed since this thread was notified (we are making visible progress), + // or there are currently no enqueued actions waiting to be processed (perhaps all + // have completed and we are about to stop monitoring), then there's no need to + // display any messages. + waitTime.reset(); + + // Sleep a while before checking again. Actions might be executing at a nice rate, no + // need to worry about inactivity. This extra sleep isn't required but it's nice to + // have: without it we would, at times of high action completion rate, unnecessarily + // put the monitor into a fast sleep-wake cycle --- not a big problem but wasteful. + sleeper.sleep(1000); + } else { + // If actions are executing but we haven't made any progress in a while (no new + // action completion), then reassure the user that we're still running. Next time + // wait a little longer. + reporter.maybeReportInactivity(); + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return; + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionValue.java new file mode 100644 index 0000000..de63c3b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionValue.java
@@ -0,0 +1,117 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Action.MiddlemanType; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * A value representing an executed action. + */ +@Immutable +@ThreadSafe +public class ActionExecutionValue implements SkyValue { + private final ImmutableMap<Artifact, FileValue> artifactData; + private final ImmutableMap<Artifact, FileArtifactValue> additionalOutputData; + + /** + * @param artifactData Map from Artifacts to corresponding FileValues. + * @param additionalOutputData Map from Artifacts to values if the FileArtifactValue for this + * artifact cannot be derived from the corresponding FileValue (see {@link + * FileAndMetadataCache#getAdditionalOutputData} for when this is necessary). + */ + ActionExecutionValue(Map<Artifact, FileValue> artifactData, + Map<Artifact, FileArtifactValue> additionalOutputData) { + this.artifactData = ImmutableMap.copyOf(artifactData); + this.additionalOutputData = ImmutableMap.copyOf(additionalOutputData); + } + + /** + * Returns metadata for a given artifact, if that metadata cannot be inferred from the + * corresponding {@link #getData} call for that Artifact. See {@link + * FileAndMetadataCache#getAdditionalOutputData} for when that can happen. + */ + @Nullable + FileArtifactValue getArtifactValue(Artifact artifact) { + return additionalOutputData.get(artifact); + } + + /** + * @return The data for each non-middleman output of this action, in the form of the {@link + * FileValue} that would be created for the file if it were to be read from disk. + */ + FileValue getData(Artifact artifact) { + Preconditions.checkState(!additionalOutputData.containsKey(artifact), + "Should not be requesting data for already-constructed FileArtifactValue: %s", artifact); + return artifactData.get(artifact); + } + + /** + * @return The map from {@link Artifact} to the corresponding {@link FileValue} that would be + * returned by {@link #getData}. Should only be needed by {@link FilesystemValueChecker}. + */ + ImmutableMap<Artifact, FileValue> getAllOutputArtifactData() { + return artifactData; + } + + @ThreadSafe + @VisibleForTesting + public static SkyKey key(Action action) { + return new SkyKey(SkyFunctions.ACTION_EXECUTION, action); + } + + /** + * Returns whether the key corresponds to a ActionExecutionValue worth reporting status about. + * + * <p>If an action can do real work, it's probably worth counting and reporting status about. + * Actions that don't really do any work (typically middleman actions) should not be counted + * towards enqueued and completed actions. + */ + public static boolean isReportWorthyAction(SkyKey key) { + return key.functionName() == SkyFunctions.ACTION_EXECUTION + && isReportWorthyAction((Action) key.argument()); + } + + /** + * Returns whether the action is worth reporting status about. + * + * <p>If an action can do real work, it's probably worth counting and reporting status about. + * Actions that don't really do any work (typically middleman actions) should not be counted + * towards enqueued and completed actions. + */ + public static boolean isReportWorthyAction(Action action) { + return action.getActionType() == MiddlemanType.NORMAL; + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("artifactData", artifactData) + .add("additionalOutputData", additionalOutputData) + .toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionLookupValue.java new file mode 100644 index 0000000..1dfa722 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionLookupValue.java
@@ -0,0 +1,106 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.HashMap; +import java.util.Map; + +/** + * Base class for all values which can provide the generating action of an artifact. The primary + * instance of such lookup values is {@link ConfiguredTargetValue}. Values that hold the generating + * actions of target completion values and build info artifacts also fall into this category. + */ +public class ActionLookupValue implements SkyValue { + protected final ImmutableMap<Artifact, Action> generatingActionMap; + + ActionLookupValue(Iterable<Action> actions) { + // Duplicate/shared actions get passed in all the time. Blaze is weird. We can't double-register + // the generated artifacts in an immutable map builder, so we double-register them in a more + // forgiving map, and then use that map to create the immutable one. + Map<Artifact, Action> generatingActions = new HashMap<>(); + for (Action action : actions) { + for (Artifact artifact : action.getOutputs()) { + generatingActions.put(artifact, action); + } + } + generatingActionMap = ImmutableMap.copyOf(generatingActions); + } + + ActionLookupValue(Action action) { + this(ImmutableList.of(action)); + } + + Action getGeneratingAction(Artifact artifact) { + return generatingActionMap.get(artifact); + } + + /** To be used only when checking consistency of the action graph -- not by other values. */ + ImmutableMap<Artifact, Action> getMapForConsistencyCheck() { + return generatingActionMap; + } + + /** + * To be used only when setting the owners of deserialized artifacts whose owners were unknown at + * creation time -- not by other callers or values. + */ + Iterable<Action> getActionsForFindingArtifactOwners() { + return generatingActionMap.values(); + } + + @VisibleForTesting + public static SkyKey key(ActionLookupKey ownerKey) { + return ownerKey.getSkyKey(); + } + + /** + * ArtifactOwner is not a SkyKey, but we wish to convert any ArtifactOwner into a SkyKey as + * simply as possible. To that end, all subclasses of ActionLookupValue "own" artifacts with + * ArtifactOwners that are subclasses of ActionLookupKey. This allows callers to easily find the + * value key, while remaining agnostic to what ActionLookupValues actually exist. + * + * <p>The methods of this class should only be called by {@link ActionLookupValue#key}. + */ + protected abstract static class ActionLookupKey implements ArtifactOwner { + @Override + public Label getLabel() { + return null; + } + + /** + * Subclasses must override this to specify their specific value type, unless they override + * {@link #getSkyKey}, in which case they are free not to implement this method. + */ + abstract SkyFunctionName getType(); + + /** + * Prefer {@link ActionLookupValue#key} to calling this method directly. + * + * <p>Subclasses may override if the value key contents should not be the key itself. + */ + SkyKey getSkyKey() { + return new SkyKey(getType(), this); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/AggregatingArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/AggregatingArtifactValue.java new file mode 100644 index 0000000..8374efe --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/AggregatingArtifactValue.java
@@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.util.Pair; + +import java.util.Collection; + +/** Value for aggregating artifacts, which must be expanded to a set of other artifacts. */ +class AggregatingArtifactValue extends ArtifactValue { + private final FileArtifactValue selfData; + private final ImmutableList<Pair<Artifact, FileArtifactValue>> inputs; + + AggregatingArtifactValue(ImmutableList<Pair<Artifact, FileArtifactValue>> inputs, + FileArtifactValue selfData) { + this.inputs = inputs; + this.selfData = selfData; + } + + /** Returns the artifacts that this artifact expands to, together with their data. */ + Collection<Pair<Artifact, FileArtifactValue>> getInputs() { + return inputs; + } + + /** Returns the data of the artifact for this value, as computed by the action cache checker. */ + FileArtifactValue getSelfData() { + return selfData; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java new file mode 100644 index 0000000..e277476 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java
@@ -0,0 +1,230 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Action.MiddlemanType; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.actions.MissingInputFileException; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.skyframe.ActionLookupValue.ActionLookupKey; +import com.google.devtools.build.lib.skyframe.ArtifactValue.OwnedArtifact; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.util.Map; + +/** + * A builder for {@link ArtifactValue}s. + */ +class ArtifactFunction implements SkyFunction { + + private final Predicate<PathFragment> allowedMissingInputs; + + ArtifactFunction(Predicate<PathFragment> allowedMissingInputs) { + this.allowedMissingInputs = allowedMissingInputs; + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws ArtifactFunctionException { + OwnedArtifact ownedArtifact = (OwnedArtifact) skyKey.argument(); + Artifact artifact = ownedArtifact.getArtifact(); + if (artifact.isSourceArtifact()) { + try { + return createSourceValue(artifact, ownedArtifact.isMandatory(), env); + } catch (MissingInputFileException e) { + // The error is not necessarily truly transient, but we mark it as such because we have + // the above side effect of posting an event to the EventBus. Importantly, that event + // is potentially used to report root causes. + throw new ArtifactFunctionException(e, Transience.TRANSIENT); + } + } + + Action action = extractActionFromArtifact(artifact, env); + if (action == null) { + return null; + } + + ActionExecutionValue actionValue = + (ActionExecutionValue) env.getValue(ActionExecutionValue.key(action)); + if (actionValue == null) { + return null; + } + + if (!isAggregatingValue(action)) { + try { + return createSimpleValue(artifact, actionValue); + } catch (IOException e) { + ActionExecutionException ex = new ActionExecutionException(e, action, + /*catastrophe=*/false); + env.getListener().handle(Event.error(ex.getLocation(), ex.getMessage())); + // This is a transient error since we did the work that led to the IOException. + throw new ArtifactFunctionException(ex, Transience.TRANSIENT); + } + } else { + return createAggregatingValue(artifact, action, actionValue.getArtifactValue(artifact), env); + } + } + + private ArtifactValue createSourceValue(Artifact artifact, boolean mandatory, Environment env) + throws MissingInputFileException { + SkyKey fileSkyKey = FileValue.key(RootedPath.toRootedPath(artifact.getRoot().getPath(), + artifact.getPath())); + FileValue fileValue; + try { + fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class, + InconsistentFilesystemException.class, FileSymlinkCycleException.class); + } catch (IOException | InconsistentFilesystemException | FileSymlinkCycleException e) { + throw makeMissingInputFileExn(artifact, mandatory, e, env.getListener()); + } + if (fileValue == null) { + return null; + } + if (!fileValue.exists()) { + if (allowedMissingInputs.apply(((RootedPath) fileSkyKey.argument()).getRelativePath())) { + return FileArtifactValue.MISSING_FILE_MARKER; + } else { + return missingInputFile(artifact, mandatory, null, env.getListener()); + } + } + try { + return FileArtifactValue.create(artifact, fileValue); + } catch (IOException e) { + throw makeMissingInputFileExn(artifact, mandatory, e, env.getListener()); + } + } + + private static ArtifactValue missingInputFile(Artifact artifact, boolean mandatory, + Exception failure, EventHandler reporter) throws MissingInputFileException { + if (!mandatory) { + return FileArtifactValue.MISSING_FILE_MARKER; + } + throw makeMissingInputFileExn(artifact, mandatory, failure, reporter); + } + + private static MissingInputFileException makeMissingInputFileExn(Artifact artifact, + boolean mandatory, Exception failure, EventHandler reporter) { + String extraMsg = (failure == null) ? "" : (":" + failure.getMessage()); + MissingInputFileException ex = new MissingInputFileException( + constructErrorMessage(artifact) + extraMsg, null); + if (mandatory) { + reporter.handle(Event.error(ex.getLocation(), ex.getMessage())); + } + return ex; + } + + // Non-aggregating artifact -- should contain at most one piece of artifact data. + // data may be null if and only if artifact is a middleman artifact. + private ArtifactValue createSimpleValue(Artifact artifact, ActionExecutionValue actionValue) + throws IOException { + ArtifactValue value = actionValue.getArtifactValue(artifact); + if (value != null) { + return value; + } + // Middleman artifacts have no corresponding files, so their ArtifactValues should have already + // been constructed during execution of the action. + Preconditions.checkState(!artifact.isMiddlemanArtifact(), artifact); + FileValue data = Preconditions.checkNotNull(actionValue.getData(artifact), + "%s %s", artifact, actionValue); + Preconditions.checkNotNull(data.getDigest(), + "Digest should already have been calculated for %s (%s)", artifact, data); + return FileArtifactValue.create(artifact, data); + } + + private AggregatingArtifactValue createAggregatingValue(Artifact artifact, Action action, + FileArtifactValue value, SkyFunction.Environment env) { + // This artifact aggregates other artifacts. Keep track of them so callers can find them. + ImmutableList.Builder<Pair<Artifact, FileArtifactValue>> inputs = ImmutableList.builder(); + for (Map.Entry<SkyKey, SkyValue> entry : + env.getValues(ArtifactValue.mandatoryKeys(action.getInputs())).entrySet()) { + Artifact input = ArtifactValue.artifact(entry.getKey()); + ArtifactValue inputValue = (ArtifactValue) entry.getValue(); + Preconditions.checkNotNull(inputValue, "%s has null dep %s", artifact, input); + if (!(inputValue instanceof FileArtifactValue)) { + // We do not recurse in aggregating middleman artifacts. + Preconditions.checkState(!(inputValue instanceof AggregatingArtifactValue), + "%s %s %s", artifact, action, inputValue); + continue; + } + inputs.add(Pair.of(input, (FileArtifactValue) inputValue)); + } + return new AggregatingArtifactValue(inputs.build(), value); + } + + /** + * Returns whether this value needs to contain the data of all its inputs. Currently only tests to + * see if the action is an aggregating middleman action. However, may include runfiles middleman + * actions and Fileset artifacts in the future. + */ + private static boolean isAggregatingValue(Action action) { + return action.getActionType() == MiddlemanType.AGGREGATING_MIDDLEMAN; + } + + @Override + public String extractTag(SkyKey skyKey) { + return Label.print(((OwnedArtifact) skyKey.argument()).getArtifact().getOwner()); + } + + private Action extractActionFromArtifact(Artifact artifact, SkyFunction.Environment env) { + ArtifactOwner artifactOwner = artifact.getArtifactOwner(); + + Preconditions.checkState(artifactOwner instanceof ActionLookupKey, "", artifact, artifactOwner); + SkyKey actionLookupKey = ActionLookupValue.key((ActionLookupKey) artifactOwner); + ActionLookupValue value = (ActionLookupValue) env.getValue(actionLookupKey); + if (value == null) { + Preconditions.checkState(artifactOwner == CoverageReportValue.ARTIFACT_OWNER, + "Not-yet-present artifact owner: %s", artifactOwner); + return null; + } + // The value should already exist (except for the coverage report action output artifacts): + // ConfiguredTargetValues were created during the analysis phase, and BuildInfo*Values + // were created during the first analysis of a configured target. + Preconditions.checkNotNull(value, + "Owner %s of %s not in graph %s", artifactOwner, artifact, actionLookupKey); + return Preconditions.checkNotNull(value.getGeneratingAction(artifact), + "Value %s does not contain generating action of %s", value, artifact); + } + + private static final class ArtifactFunctionException extends SkyFunctionException { + ArtifactFunctionException(MissingInputFileException e, Transience transience) { + super(e, transience); + } + + ArtifactFunctionException(ActionExecutionException e, Transience transience) { + super(e, transience); + } + } + + private static String constructErrorMessage(Artifact artifact) { + if (artifact.getOwner() == null) { + return String.format("missing input file '%s'", artifact.getPath().getPathString()); + } else { + return String.format("missing input file '%s'", artifact.getOwner()); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactValue.java new file mode 100644 index 0000000..6139d2e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactValue.java
@@ -0,0 +1,160 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.Collections2; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.Collection; + +/** + * A value representing an artifact. Source artifacts are checked for existence, while output + * artifacts imply creation of the output file. + * + * <p>There are effectively two kinds of output artifact values. The first corresponds to an + * ordinary artifact {@link FileArtifactValue}. It stores the relevant data for the artifact -- + * digest/mtime and size. The second corresponds to an "aggregating" artifact -- the output of an + * aggregating middleman action. It stores the relevant data of all its inputs. + */ +@Immutable +@ThreadSafe +public abstract class ArtifactValue implements SkyValue { + + @ThreadSafe + static SkyKey key(Artifact artifact, boolean isMandatory) { + return new SkyKey(SkyFunctions.ARTIFACT, artifact.isSourceArtifact() + ? new OwnedArtifact(artifact, isMandatory) + : new OwnedArtifact(artifact)); + } + + private static final Function<Artifact, SkyKey> TO_MANDATORY_KEY = + new Function<Artifact, SkyKey>() { + @Override + public SkyKey apply(Artifact artifact) { + return key(artifact, true); + } + }; + + @ThreadSafe + public static Iterable<SkyKey> mandatoryKeys(Iterable<Artifact> artifacts) { + return Iterables.transform(artifacts, TO_MANDATORY_KEY); + } + + private static final Function<OwnedArtifact, Artifact> TO_ARTIFACT = + new Function<OwnedArtifact, Artifact>() { + @Override + public Artifact apply(OwnedArtifact key) { + return key.getArtifact(); + } + }; + + public static Collection<Artifact> artifacts(Collection<? extends OwnedArtifact> keys) { + return Collections2.transform(keys, TO_ARTIFACT); + } + + public static Artifact artifact(SkyKey key) { + return TO_ARTIFACT.apply((OwnedArtifact) key.argument()); + } + + /** + * Artifacts are compared using just their paths, but in Skyframe, the configured target that owns + * an artifact must also be part of the comparison. For example, suppose we build //foo:foo in + * configurationA, yielding artifact foo.out. If we change the configuration to configurationB in + * such a way that the path to the artifact does not change, requesting foo.out from the graph + * will result in the value entry for foo.out under configurationA being returned. This would + * prevent caching the graph in different configurations, and also causes big problems with change + * pruning, which assumes the invariant that a value's first dependency will always be the same. + * In this case, the value entry's old dependency on //foo:foo in configurationA would cause it to + * request (//foo:foo, configurationA) from the graph, causing an undesired re-analysis of + * (//foo:foo, configurationA). + * + * <p>In order to prevent that, instead of using Artifacts as keys in the graph, we use + * OwnedArtifacts, which compare for equality using both the Artifact, and the owner. The effect + * is functionally that of making Artifact.equals() check the owner, but only within Skyframe, + * since outside of Skyframe it is quite crucial that Artifacts with different owners be able to + * compare equal. + */ + public static class OwnedArtifact { + private final Artifact artifact; + // Always true for derived artifacts. + private final boolean isMandatory; + + /** Constructs an OwnedArtifact wrapper for a source artifact. */ + private OwnedArtifact(Artifact sourceArtifact, boolean mandatory) { + Preconditions.checkArgument(sourceArtifact.isSourceArtifact()); + this.artifact = Preconditions.checkNotNull(sourceArtifact); + this.isMandatory = mandatory; + } + + /** + * Constructs an OwnedArtifact wrapper for a derived artifact. The mandatory attribute is + * not needed because a derived artifact must be a mandatory input for some action in order to + * ensure that it is built in the first place. If it fails to build, then that fact is cached + * in the node, so any action that has it as a non-mandatory input can retrieve that + * information from the node. + */ + private OwnedArtifact(Artifact derivedArtifact) { + this.artifact = Preconditions.checkNotNull(derivedArtifact); + Preconditions.checkArgument(!derivedArtifact.isSourceArtifact(), derivedArtifact); + this.isMandatory = true; // Unused. + } + + @Override + public int hashCode() { + int initialHash = artifact.hashCode() + artifact.getArtifactOwner().hashCode(); + return isMandatory ? initialHash : 47 * initialHash + 1; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + if (!(that instanceof OwnedArtifact)) { + return false; + } + OwnedArtifact thatOwnedArtifact = ((OwnedArtifact) that); + Artifact thatArtifact = thatOwnedArtifact.artifact; + return artifact.equals(thatArtifact) + && artifact.getArtifactOwner().equals(thatArtifact.getArtifactOwner()) + && isMandatory == thatOwnedArtifact.isMandatory; + } + + Artifact getArtifact() { + return artifact; + } + + /** + * Returns whether the artifact is a mandatory input of its requesting action. May only be + * called for source artifacts, since a derived artifact must be a mandatory input of some + * action in order to have been built in the first place. + */ + public boolean isMandatory() { + Preconditions.checkState(artifact.isSourceArtifact(), artifact); + return isMandatory; + } + + @Override + public String toString() { + return artifact.prettyPrint() + " " + artifact.getArtifactOwner(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/AspectFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/AspectFunction.java new file mode 100644 index 0000000..f1aa2f6e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/AspectFunction.java
@@ -0,0 +1,187 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ListMultimap; +import com.google.devtools.build.lib.analysis.Aspect; +import com.google.devtools.build.lib.analysis.CachingAnalysisEnvironment; +import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.TargetAndConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.packages.AspectFactory; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.skyframe.AspectValue.AspectKey; +import com.google.devtools.build.lib.skyframe.ConfiguredTargetFunction.DependencyEvaluationException; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor.BuildViewProvider; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * The Skyframe function that generates aspects. + */ +public final class AspectFunction implements SkyFunction { + private final BuildViewProvider buildViewProvider; + + public AspectFunction(BuildViewProvider buildViewProvider) { + this.buildViewProvider = buildViewProvider; + } + + @Nullable + @Override + public SkyValue compute(SkyKey skyKey, Environment env) + throws AspectFunctionException { + SkyframeBuildView view = buildViewProvider.getSkyframeBuildView(); + AspectKey key = (AspectKey) skyKey.argument(); + ConfiguredAspectFactory aspectFactory = + (ConfiguredAspectFactory) AspectFactory.Util.create(key.getAspect()); + + PackageValue packageValue = + (PackageValue) env.getValue(PackageValue.key(key.getLabel().getPackageIdentifier())); + if (packageValue == null) { + return null; + } + + Target target; + try { + target = packageValue.getPackage().getTarget(key.getLabel().getName()); + } catch (NoSuchTargetException e) { + throw new AspectFunctionException(skyKey, e); + } + + if (!(target instanceof Rule)) { + throw new AspectFunctionException(new AspectCreationException( + "aspects must be attached to rules")); + } + + RuleConfiguredTarget associatedTarget = (RuleConfiguredTarget) + ((ConfiguredTargetValue) env.getValue(ConfiguredTargetValue.key( + key.getLabel(), key.getConfiguration()))).getConfiguredTarget(); + + if (associatedTarget == null) { + return null; + } + + SkyframeDependencyResolver resolver = view.createDependencyResolver(env); + if (resolver == null) { + return null; + } + + TargetAndConfiguration ctgValue = + new TargetAndConfiguration(target, key.getConfiguration()); + + try { + // Get the configuration targets that trigger this rule's configurable attributes. + Set<ConfigMatchingProvider> configConditions = + ConfiguredTargetFunction.getConfigConditions(target, env, resolver, ctgValue); + if (configConditions == null) { + // Those targets haven't yet been resolved. + return null; + } + + ListMultimap<Attribute, ConfiguredTarget> depValueMap = + ConfiguredTargetFunction.computeDependencies(env, resolver, ctgValue, + aspectFactory.getDefinition(), configConditions); + + return createAspect(env, key, associatedTarget, configConditions, depValueMap); + } catch (DependencyEvaluationException e) { + throw new AspectFunctionException(e.getRootCauseSkyKey(), e.getCause()); + } + } + + @Nullable + private AspectValue createAspect(Environment env, AspectKey key, + RuleConfiguredTarget associatedTarget, Set<ConfigMatchingProvider> configConditions, + ListMultimap<Attribute, ConfiguredTarget> directDeps) + throws AspectFunctionException { + SkyframeBuildView view = buildViewProvider.getSkyframeBuildView(); + BuildConfiguration configuration = associatedTarget.getConfiguration(); + boolean extendedSanityChecks = configuration != null && configuration.extendedSanityChecks(); + + StoredEventHandler events = new StoredEventHandler(); + CachingAnalysisEnvironment analysisEnvironment = view.createAnalysisEnvironment( + key, false, extendedSanityChecks, events, env, true); + if (env.valuesMissing()) { + return null; + } + + ConfiguredAspectFactory aspectFactory = + (ConfiguredAspectFactory) AspectFactory.Util.create(key.getAspect()); + Aspect aspect = view.createAspect( + analysisEnvironment, associatedTarget, aspectFactory, directDeps, configConditions); + + events.replayOn(env.getListener()); + if (events.hasErrors()) { + analysisEnvironment.disable(associatedTarget.getTarget()); + throw new AspectFunctionException(new AspectCreationException( + "Analysis of target '" + associatedTarget.getLabel() + "' failed; build aborted")); + } + Preconditions.checkState(!analysisEnvironment.hasErrors(), + "Analysis environment hasError() but no errors reported"); + + if (env.valuesMissing()) { + return null; + } + + analysisEnvironment.disable(associatedTarget.getTarget()); + Preconditions.checkNotNull(aspect); + + return new AspectValue( + aspect, ImmutableList.copyOf(analysisEnvironment.getRegisteredActions())); + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + /** + * An exception indicating that there was a problem creating an aspect. + */ + public static final class AspectCreationException extends Exception { + public AspectCreationException(String message) { + super(message); + } + } + + /** + * Used to indicate errors during the computation of an {@link AspectValue}. + */ + private static final class AspectFunctionException extends SkyFunctionException { + public AspectFunctionException(Exception e) { + super(e, Transience.PERSISTENT); + } + + /** Used to rethrow a child error that we cannot handle. */ + public AspectFunctionException(SkyKey childKey, Exception transitiveError) { + super(transitiveError, childKey); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/AspectValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/AspectValue.java new file mode 100644 index 0000000..9b863bd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/AspectValue.java
@@ -0,0 +1,109 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Objects; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.analysis.Aspect; +import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; + +/** + * An aspect in the context of the Skyframe graph. + */ +public final class AspectValue extends ActionLookupValue { + /** + * The key of an action that is generated by an aspect. + */ + public static final class AspectKey extends ActionLookupKey { + private final Label label; + private final BuildConfiguration configuration; + // TODO(bazel-team): class objects are not really hashable or comparable for equality other than + // by reference. We should identify the aspect here in a way that does not rely on comparison + // by reference so that keys can be serialized and deserialized properly. + private final Class<? extends ConfiguredAspectFactory> aspectFactory; + + private AspectKey(Label label, BuildConfiguration configuration, + Class<? extends ConfiguredAspectFactory> aspectFactory) { + this.label = label; + this.configuration = configuration; + this.aspectFactory = aspectFactory; + } + + @Override + public Label getLabel() { + return label; + } + + public BuildConfiguration getConfiguration() { + return configuration; + } + + public Class<? extends ConfiguredAspectFactory> getAspect() { + return aspectFactory; + } + + @Override + SkyFunctionName getType() { + return SkyFunctions.ASPECT; + } + + @Override + public int hashCode() { + return Objects.hashCode(label, configuration, aspectFactory); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (!(other instanceof AspectKey)) { + return false; + } + + AspectKey that = (AspectKey) other; + return Objects.equal(label, that.label) + && Objects.equal(configuration, that.configuration) + && Objects.equal(aspectFactory, that.aspectFactory); + } + + @Override + public String toString() { + return label + "#" + aspectFactory.getSimpleName() + " " + + (configuration == null ? "null" : configuration.shortCacheKey()); + } + } + + private final Aspect aspect; + + public AspectValue(Aspect aspect, Iterable<Action> actions) { + super(actions); + this.aspect = aspect; + } + + public Aspect get() { + return aspect; + } + + public static SkyKey key(Label label, BuildConfiguration configuration, + Class<? extends ConfiguredAspectFactory> aspectFactory) { + return new SkyKey(SkyFunctions.ASPECT, new AspectKey(label, configuration, aspectFactory)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BrokenDiffAwarenessException.java b/src/main/java/com/google/devtools/build/lib/skyframe/BrokenDiffAwarenessException.java new file mode 100644 index 0000000..a5b0272 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BrokenDiffAwarenessException.java
@@ -0,0 +1,27 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; + +/** + * Thrown on {@link DiffAwareness#getDiff} to indicate that something is wrong with the + * {@link DiffAwareness} instance and it should not be used again. + */ +public class BrokenDiffAwarenessException extends Exception { + + public BrokenDiffAwarenessException(String msg) { + super(Preconditions.checkNotNull(msg)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionFunction.java new file mode 100644 index 0000000..e717e51 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionFunction.java
@@ -0,0 +1,86 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Supplier; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoContext; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoType; +import com.google.devtools.build.lib.skyframe.BuildInfoCollectionValue.BuildInfoKeyAndConfig; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.Map; + +/** + * Creates a {@link BuildInfoCollectionValue}. Only depends on the unique + * {@link WorkspaceStatusValue} and the constant {@link PrecomputedValue#BUILD_INFO_FACTORIES} + * injected value. + */ +public class BuildInfoCollectionFunction implements SkyFunction { + // Supplier only because the artifact factory has not yet been created at constructor time. + private final Supplier<ArtifactFactory> artifactFactory; + private final Root buildDataDirectory; + + BuildInfoCollectionFunction(Supplier<ArtifactFactory> artifactFactory, + Root buildDataDirectory) { + this.artifactFactory = artifactFactory; + this.buildDataDirectory = buildDataDirectory; + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + final BuildInfoKeyAndConfig keyAndConfig = (BuildInfoKeyAndConfig) skyKey.argument(); + WorkspaceStatusValue infoArtifactValue = + (WorkspaceStatusValue) env.getValue(WorkspaceStatusValue.SKY_KEY); + if (infoArtifactValue == null) { + return null; + } + Map<BuildInfoKey, BuildInfoFactory> buildInfoFactories = + PrecomputedValue.BUILD_INFO_FACTORIES.get(env); + if (buildInfoFactories == null) { + return null; + } + final ArtifactFactory factory = artifactFactory.get(); + BuildInfoContext context = new BuildInfoContext() { + @Override + public Artifact getBuildInfoArtifact(PathFragment rootRelativePath, Root root, + BuildInfoType type) { + return type == BuildInfoType.NO_REBUILD + ? factory.getConstantMetadataArtifact(rootRelativePath, root, keyAndConfig) + : factory.getDerivedArtifact(rootRelativePath, root, keyAndConfig); + } + + @Override + public Root getBuildDataDirectory() { + return buildDataDirectory; + } + }; + + return new BuildInfoCollectionValue(buildInfoFactories.get( + keyAndConfig.getInfoKey()).create(context, keyAndConfig.getConfig(), + infoArtifactValue.getStableArtifact(), infoArtifactValue.getVolatileArtifact())); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionValue.java new file mode 100644 index 0000000..8958e1b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BuildInfoCollectionValue.java
@@ -0,0 +1,97 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoCollection; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyFunctionName; + +import java.util.Objects; + +/** + * Value that stores {@link BuildInfoCollection}s generated by {@link BuildInfoFactory} instances. + * These collections are used during analysis (see {@code CachingAnalysisEnvironment}). + */ +public class BuildInfoCollectionValue extends ActionLookupValue { + private final BuildInfoCollection collection; + + BuildInfoCollectionValue(BuildInfoCollection collection) { + super(collection.getActions()); + this.collection = collection; + } + + public BuildInfoCollection getCollection() { + return collection; + } + + @SuppressWarnings("deprecation") + @Override + public String toString() { + return com.google.common.base.Objects.toStringHelper(getClass()) + .add("collection", collection) + .add("generatingActionMap", generatingActionMap).toString(); + } + + /** Key for BuildInfoCollectionValues. */ + public static class BuildInfoKeyAndConfig extends ActionLookupKey { + private final BuildInfoFactory.BuildInfoKey infoKey; + private final BuildConfiguration config; + + public BuildInfoKeyAndConfig(BuildInfoFactory.BuildInfoKey key, BuildConfiguration config) { + this.infoKey = Preconditions.checkNotNull(key, config); + this.config = Preconditions.checkNotNull(config, key); + } + + @Override + SkyFunctionName getType() { + return SkyFunctions.BUILD_INFO_COLLECTION; + } + + BuildInfoFactory.BuildInfoKey getInfoKey() { + return infoKey; + } + + BuildConfiguration getConfig() { + return config; + } + + @Override + public Label getLabel() { + return null; + } + + @Override + public int hashCode() { + return Objects.hash(infoKey, config); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null) { + return false; + } + if (this.getClass() != other.getClass()) { + return false; + } + BuildInfoKeyAndConfig that = (BuildInfoKeyAndConfig) other; + return Objects.equals(this.infoKey, that.infoKey) && Objects.equals(this.config, that.config); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java b/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java new file mode 100644 index 0000000..7fdb55c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java
@@ -0,0 +1,75 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BuildFailedException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.TestExecException; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.util.AbruptExitException; + +import java.util.Collection; +import java.util.Set; + +/** + * A Builder consumes top-level artifacts, targets, and tests,, and executes them in some + * topological order, possibly concurrently, using some dependency-checking policy. + * + * <p> The methods of the Builder interface are typically long-running, but honor the + * {@link java.lang.Thread#interrupt} contract: if an interrupt is delivered to the thread in which + * a call to buildTargets or buildArtifacts is active, the Builder attempts to terminate the call + * prematurely, throwing InterruptedException. No guarantee is made about the timeliness of such + * termination, as it depends on the ability of the Actions being executed to be interrupted, but + * typically any running subprocesses will be quickly killed. + */ +public interface Builder { + + /** + * Transitively build all given artifacts, targets, and tests, and all necessary prerequisites + * thereof. For sequential implementations of this interface, the top-level requests will be + * built in the iteration order of the Set provided; for concurrent implementations, the order + * is undefined. + * + * <p>This method should not be invoked more than once concurrently on the same Builder instance. + * + * @param artifacts the set of Artifacts to build + * @param parallelTests tests to execute in parallel with the other top-level targetsToBuild and + * artifacts. + * @param exclusiveTests are executed one at a time, only after all other tasks have completed + * @param targetsToBuild Set of targets which will be built + * @param executor an opaque application-specific value that will be + * passed down to the execute() method of any Action executed during + * this call + * @param builtTargets (out) set of successfully built subset of targetsToBuild. This set is + * populated immediately upon confirmation that artifact is built so it will be + * valid even if a future action throws ActionExecutionException + * @throws BuildFailedException if there were problems establishing the action execution + * environment, if the the metadata of any file during the build could not be obtained, + * if any input files are missing, or if an action fails during execution + * @throws InterruptedException if there was an asynchronous stop request + * @throws TestExecException if any test fails + */ + @ThreadCompatible + void buildArtifacts(Set<Artifact> artifacts, + Set<ConfiguredTarget> parallelTests, + Set<ConfiguredTarget> exclusiveTests, + Collection<ConfiguredTarget> targetsToBuild, + Executor executor, + Set<ConfiguredTarget> builtTargets, + boolean explain) + throws BuildFailedException, AbruptExitException, InterruptedException, TestExecException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionFunction.java new file mode 100644 index 0000000..89828c3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionFunction.java
@@ -0,0 +1,164 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Supplier; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.analysis.config.PackageProviderForConfigurations; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.skyframe.ConfigurationCollectionValue.ConfigurationCollectionKey; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A builder for {@link ConfigurationCollectionValue} instances. + */ +public class ConfigurationCollectionFunction implements SkyFunction { + + private final Supplier<ConfigurationFactory> configurationFactory; + private final Supplier<Map<String, String>> clientEnv; + private final Supplier<Set<Package>> configurationPackages; + + public ConfigurationCollectionFunction( + Supplier<ConfigurationFactory> configurationFactory, + Supplier<Map<String, String>> clientEnv, + Supplier<Set<Package>> configurationPackages) { + this.configurationFactory = configurationFactory; + this.clientEnv = clientEnv; + this.configurationPackages = configurationPackages; + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException, + ConfigurationCollectionFunctionException { + ConfigurationCollectionKey collectionKey = (ConfigurationCollectionKey) skyKey.argument(); + try { + // We are not using this value, because test_environment can be created from clientEnv. But + // we want ConfigurationCollection to be recomputed each time when test_environment changes. + PrecomputedValue.TEST_ENVIRONMENT_VARIABLES.get(env); + BlazeDirectories directories = PrecomputedValue.BLAZE_DIRECTORIES.get(env); + if (env.valuesMissing()) { + return null; + } + + BuildConfigurationCollection result = + getConfigurations(env.getListener(), + new SkyframePackageLoaderWithValueEnvironment(env, configurationPackages.get()), + new BuildConfigurationKey(collectionKey.getBuildOptions(), directories, clientEnv.get(), + collectionKey.getMultiCpu())); + + // BuildConfigurationCollection can be created, but dependencies to some files might be + // missing. In that case we need to build configurationCollection again. + if (env.valuesMissing()) { + return null; + } + + for (BuildConfiguration config : result.getTargetConfigurations()) { + config.declareSkyframeDependencies(env); + } + if (env.valuesMissing()) { + return null; + } + return new ConfigurationCollectionValue(result, configurationPackages.get()); + } catch (InvalidConfigurationException e) { + throw new ConfigurationCollectionFunctionException(e); + } + } + + /** Create the build configurations with the given options. */ + private BuildConfigurationCollection getConfigurations(EventHandler eventHandler, + PackageProviderForConfigurations loadedPackageProvider, BuildConfigurationKey key) + throws InvalidConfigurationException { + List<BuildConfiguration> targetConfigurations = new ArrayList<>(); + if (!key.getMultiCpu().isEmpty()) { + for (String cpu : key.getMultiCpu()) { + BuildConfiguration targetConfiguration = createConfiguration( + eventHandler, loadedPackageProvider, key, cpu); + if (targetConfiguration == null || targetConfigurations.contains(targetConfiguration)) { + continue; + } + targetConfigurations.add(targetConfiguration); + } + if (loadedPackageProvider.valuesMissing()) { + return null; + } + } else { + BuildConfiguration targetConfiguration = createConfiguration( + eventHandler, loadedPackageProvider, key, null); + if (targetConfiguration == null) { + return null; + } + targetConfigurations.add(targetConfiguration); + } + return new BuildConfigurationCollection(targetConfigurations); + } + + @Nullable + public BuildConfiguration createConfiguration( + EventHandler originalEventListener, + PackageProviderForConfigurations loadedPackageProvider, + BuildConfigurationKey key, String cpuOverride) throws InvalidConfigurationException { + StoredEventHandler errorEventListener = new StoredEventHandler(); + BuildOptions buildOptions = key.getBuildOptions(); + if (cpuOverride != null) { + // TODO(bazel-team): Options classes should be immutable. This is a bit of a hack. + buildOptions = buildOptions.clone(); + buildOptions.get(BuildConfiguration.Options.class).cpu = cpuOverride; + } + + BuildConfiguration targetConfig = configurationFactory.get().createConfiguration( + loadedPackageProvider, buildOptions, key, errorEventListener); + if (targetConfig == null) { + return null; + } + errorEventListener.replayOn(originalEventListener); + if (errorEventListener.hasErrors()) { + throw new InvalidConfigurationException("Build options are invalid"); + } + return targetConfig; + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link ConfigurationCollectionFunction#compute}. + */ + private static final class ConfigurationCollectionFunctionException extends + SkyFunctionException { + public ConfigurationCollectionFunctionException(InvalidConfigurationException e) { + super(e, Transience.PERSISTENT); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionValue.java new file mode 100644 index 0000000..30e4fd7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationCollectionValue.java
@@ -0,0 +1,100 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.Serializable; +import java.util.Objects; +import java.util.Set; + +/** + * A Skyframe value representing a build configuration collection. + */ +@Immutable +@ThreadSafe +public class ConfigurationCollectionValue implements SkyValue { + + private final BuildConfigurationCollection configurationCollection; + private final ImmutableSet<Package> configurationPackages; + + ConfigurationCollectionValue(BuildConfigurationCollection configurationCollection, + Set<Package> configurationPackages) { + this.configurationCollection = Preconditions.checkNotNull(configurationCollection); + this.configurationPackages = ImmutableSet.copyOf(configurationPackages); + } + + public BuildConfigurationCollection getConfigurationCollection() { + return configurationCollection; + } + + /** + * Returns set of packages required for configuration. + */ + public Set<Package> getConfigurationPackages() { + return configurationPackages; + } + + @ThreadSafe + public static SkyKey key(BuildOptions buildOptions, ImmutableSet<String> multiCpu) { + return new SkyKey(SkyFunctions.CONFIGURATION_COLLECTION, + new ConfigurationCollectionKey(buildOptions, multiCpu)); + } + + static final class ConfigurationCollectionKey implements Serializable { + private final BuildOptions buildOptions; + private final ImmutableSet<String> multiCpu; + private final int hashCode; + + public ConfigurationCollectionKey(BuildOptions buildOptions, ImmutableSet<String> multiCpu) { + this.buildOptions = Preconditions.checkNotNull(buildOptions); + this.multiCpu = Preconditions.checkNotNull(multiCpu); + this.hashCode = Objects.hash(buildOptions, multiCpu); + } + + public BuildOptions getBuildOptions() { + return buildOptions; + } + + public ImmutableSet<String> getMultiCpu() { + return multiCpu; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ConfigurationCollectionKey)) { + return false; + } + ConfigurationCollectionKey confObject = (ConfigurationCollectionKey) o; + return Objects.equals(multiCpu, confObject.multiCpu) + && Objects.equals(buildOptions, confObject.buildOptions); + } + + @Override + public int hashCode() { + return hashCode; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentFunction.java new file mode 100644 index 0000000..0393b16 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentFunction.java
@@ -0,0 +1,146 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.analysis.config.PackageProviderForConfigurations; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.skyframe.ConfigurationFragmentValue.ConfigurationFragmentKey; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.util.Set; + +/** + * A builder for {@link ConfigurationFragmentValue}s. + */ +public class ConfigurationFragmentFunction implements SkyFunction { + + private final Supplier<ImmutableList<ConfigurationFragmentFactory>> configurationFragments; + private final Supplier<Set<Package>> configurationPackages; + + public ConfigurationFragmentFunction( + Supplier<ImmutableList<ConfigurationFragmentFactory>> configurationFragments, + Supplier<Set<Package>> configurationPackages) { + this.configurationFragments = configurationFragments; + this.configurationPackages = configurationPackages; + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException, + ConfigurationFragmentFunctionException { + ConfigurationFragmentKey configurationFragmentKey = + (ConfigurationFragmentKey) skyKey.argument(); + BuildOptions buildOptions = configurationFragmentKey.getBuildOptions(); + ConfigurationFragmentFactory factory = getFactory(configurationFragmentKey.getFragmentType()); + try { + PackageProviderForConfigurations loadedPackageProvider = + new SkyframePackageLoaderWithValueEnvironment(env, configurationPackages.get()); + ConfigurationEnvironment confEnv = new ConfigurationBuilderEnvironment(loadedPackageProvider); + Fragment fragment = factory.create(confEnv, buildOptions); + + if (env.valuesMissing()) { + return null; + } + return new ConfigurationFragmentValue(fragment); + } catch (InvalidConfigurationException e) { + // TODO(bazel-team): Rework the control-flow here so that we're not actually throwing this + // exception with missing Skyframe dependencies. + if (env.valuesMissing()) { + return null; + } + throw new ConfigurationFragmentFunctionException(e); + } + } + + private ConfigurationFragmentFactory getFactory(Class<? extends Fragment> fragmentType) { + for (ConfigurationFragmentFactory factory : configurationFragments.get()) { + if (factory.creates().equals(fragmentType)) { + return factory; + } + } + throw new IllegalStateException( + "There is no factory for fragment: " + fragmentType.getSimpleName()); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + /** + * A {@link ConfigurationEnvironment} implementation that can create dependencies on files. + */ + private final class ConfigurationBuilderEnvironment implements ConfigurationEnvironment { + private final PackageProviderForConfigurations loadedPackageProvider; + + ConfigurationBuilderEnvironment( + PackageProviderForConfigurations loadedPackageProvider) { + this.loadedPackageProvider = loadedPackageProvider; + } + + @Override + public Target getTarget(Label label) throws NoSuchPackageException, NoSuchTargetException { + return loadedPackageProvider.getLoadedTarget(label); + } + + @Override + public Path getPath(Package pkg, String fileName) { + Path result = pkg.getPackageDirectory().getRelative(fileName); + try { + loadedPackageProvider.addDependency(pkg, fileName); + } catch (IOException | SyntaxException e) { + return null; + } + return result; + } + + @Override + public <T extends Fragment> T getFragment(BuildOptions buildOptions, Class<T> fragmentType) + throws InvalidConfigurationException { + return loadedPackageProvider.getFragment(buildOptions, fragmentType); + } + + @Override + public BlazeDirectories getBlazeDirectories() { + return loadedPackageProvider.getDirectories(); + } + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link ConfigurationFragmentFunction#compute}. + */ + private static final class ConfigurationFragmentFunctionException extends SkyFunctionException { + public ConfigurationFragmentFunctionException(InvalidConfigurationException e) { + super(e, Transience.PERSISTENT); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentValue.java new file mode 100644 index 0000000..cc07216 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfigurationFragmentValue.java
@@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.Serializable; +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * A Skyframe node representing a build configuration fragment. + */ +@Immutable +@ThreadSafe +public class ConfigurationFragmentValue implements SkyValue { + + @Nullable + private final BuildConfiguration.Fragment fragment; + + ConfigurationFragmentValue(BuildConfiguration.Fragment fragment) { + this.fragment = fragment; + } + + public BuildConfiguration.Fragment getFragment() { + return fragment; + } + + @ThreadSafe + public static SkyKey key(BuildOptions buildOptions, Class<? extends Fragment> fragmentType) { + return new SkyKey(SkyFunctions.CONFIGURATION_FRAGMENT, + new ConfigurationFragmentKey(buildOptions, fragmentType)); + } + + static final class ConfigurationFragmentKey implements Serializable { + private final BuildOptions buildOptions; + private final Class<? extends Fragment> fragmentType; + + public ConfigurationFragmentKey(BuildOptions buildOptions, + Class<? extends Fragment> fragmentType) { + this.buildOptions = Preconditions.checkNotNull(buildOptions); + this.fragmentType = Preconditions.checkNotNull(fragmentType); + } + + public BuildOptions getBuildOptions() { + return buildOptions; + } + + public Class<? extends Fragment> getFragmentType() { + return fragmentType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ConfigurationFragmentKey)) { + return false; + } + ConfigurationFragmentKey confObject = (ConfigurationFragmentKey) o; + return Objects.equals(fragmentType, confObject.fragmentType) + && Objects.equals(buildOptions, confObject.buildOptions); + } + + @Override + public int hashCode() { + return Objects.hash(buildOptions, fragmentType); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java new file mode 100644 index 0000000..9fc3df4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java
@@ -0,0 +1,578 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; +import com.google.devtools.build.lib.analysis.Aspect; +import com.google.devtools.build.lib.analysis.CachingAnalysisEnvironment; +import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency; +import com.google.devtools.build.lib.analysis.LabelAndConfiguration; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.TargetAndConfiguration; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.packages.AspectDefinition; +import com.google.devtools.build.lib.packages.AspectFactory; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.PackageGroup; +import com.google.devtools.build.lib.packages.RawAttributeMapper; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.skyframe.AspectFunction.AspectCreationException; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor.BuildViewProvider; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.skyframe.ValueOrException2; +import com.google.devtools.build.skyframe.ValueOrException3; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * SkyFunction for {@link ConfiguredTargetValue}s. + */ +final class ConfiguredTargetFunction implements SkyFunction { + + /** + * Exception class that signals an error during the evaluation of a dependency. + */ + public static class DependencyEvaluationException extends Exception { + private final SkyKey rootCauseSkyKey; + + public DependencyEvaluationException(Exception cause) { + super(cause); + this.rootCauseSkyKey = null; + } + + public DependencyEvaluationException(SkyKey rootCauseSkyKey, Exception cause) { + super(cause); + this.rootCauseSkyKey = rootCauseSkyKey; + } + + /** + * Returns the key of the root cause or null if the problem was with this target. + */ + public SkyKey getRootCauseSkyKey() { + return rootCauseSkyKey; + } + + @Override + public Exception getCause() { + return (Exception) super.getCause(); + } + } + + private static final Function<Dependency, SkyKey> TO_KEYS = + new Function<Dependency, SkyKey>() { + @Override + public SkyKey apply(Dependency input) { + return ConfiguredTargetValue.key(input.getLabel(), input.getConfiguration()); + } + }; + + private final BuildViewProvider buildViewProvider; + + ConfiguredTargetFunction(BuildViewProvider buildViewProvider) { + this.buildViewProvider = buildViewProvider; + } + + @Override + public SkyValue compute(SkyKey key, Environment env) throws ConfiguredTargetFunctionException, + InterruptedException { + SkyframeBuildView view = buildViewProvider.getSkyframeBuildView(); + + ConfiguredTargetKey configuredTargetKey = (ConfiguredTargetKey) key.argument(); + LabelAndConfiguration lc = LabelAndConfiguration.of( + configuredTargetKey.getLabel(), configuredTargetKey.getConfiguration()); + + BuildConfiguration configuration = lc.getConfiguration(); + + PackageValue packageValue = + (PackageValue) env.getValue(PackageValue.key(lc.getLabel().getPackageIdentifier())); + if (packageValue == null) { + return null; + } + + Target target; + try { + target = packageValue.getPackage().getTarget(lc.getLabel().getName()); + } catch (NoSuchTargetException e1) { + throw new ConfiguredTargetFunctionException(new NoSuchTargetException(lc.getLabel(), + "No such target")); + } + // TODO(bazel-team): This is problematic - we create the right key, but then end up with a value + // that doesn't match; we can even have the same value multiple times. However, I think it's + // only triggered in tests (i.e., in normal operation, the configuration passed in is already + // null). + if (target instanceof InputFile) { + // InputFileConfiguredTarget expects its configuration to be null since it's not used. + configuration = null; + } else if (target instanceof PackageGroup) { + // Same for PackageGroupConfiguredTarget. + configuration = null; + } + TargetAndConfiguration ctgValue = + new TargetAndConfiguration(target, configuration); + + SkyframeDependencyResolver resolver = view.createDependencyResolver(env); + if (resolver == null) { + return null; + } + + try { + // Get the configuration targets that trigger this rule's configurable attributes. + Set<ConfigMatchingProvider> configConditions = + getConfigConditions(ctgValue.getTarget(), env, resolver, ctgValue); + if (configConditions == null) { + // Those targets haven't yet been resolved. + return null; + } + + ListMultimap<Attribute, ConfiguredTarget> depValueMap = + computeDependencies(env, resolver, ctgValue, null, configConditions); + return createConfiguredTarget( + view, env, target, configuration, depValueMap, configConditions); + } catch (DependencyEvaluationException e) { + throw new ConfiguredTargetFunctionException(e.getRootCauseSkyKey(), e.getCause()); + } + } + + /** + * Computes the direct dependencies of a node in the configured target graph (a configured + * target or an aspect). + * + * <p>Returns null if Skyframe hasn't evaluated the required dependencies yet. In this case, the + * caller should also return null to Skyframe. + * + * @param env the Skyframe environment + * @param resolver The dependency resolver + * @param ctgValue The label and the configuration of the node + * @param aspectDefinition the aspect of the node (if null, the node is a configured target, + * otherwise it's an asect) + * @param configConditions the configuration conditions for evaluating the attributes of the node + * @return an attribute -> direct dependency multimap + * @throws ConfiguredTargetFunctionException + */ + @Nullable + static ListMultimap<Attribute, ConfiguredTarget> computeDependencies( + Environment env, SkyframeDependencyResolver resolver, TargetAndConfiguration ctgValue, + AspectDefinition aspectDefinition, Set<ConfigMatchingProvider> configConditions) + throws DependencyEvaluationException { + + // 1. Create the map from attributes to list of (target, configuration) pairs. + ListMultimap<Attribute, Dependency> depValueNames; + try { + depValueNames = resolver.dependentNodeMap(ctgValue, aspectDefinition, configConditions); + } catch (EvalException e) { + env.getListener().handle(Event.error(e.getLocation(), e.getMessage())); + throw new DependencyEvaluationException(new ConfiguredValueCreationException(e.print())); + } + + // 2. Resolve configured target dependencies and handle errors. + Map<SkyKey, ConfiguredTarget> depValues = + resolveConfiguredTargetDependencies(env, depValueNames.values(), ctgValue.getTarget()); + if (depValues == null) { + return null; + } + + // 3. Resolve required aspects. + ListMultimap<SkyKey, Aspect> depAspects = resolveAspectDependencies( + env, depValues, depValueNames.values()); + if (depAspects == null) { + return null; + } + + // 3. Merge the dependent configured targets and aspects into a single map. + return mergeAspects(depValueNames, depValues, depAspects); + } + + /** + * Merges the each direct dependency configured target with the aspects associated with it. + * + * <p>Note that the combination of a configured target and its associated aspects are not + * represented by a Skyframe node. This is because there can possibly be many different + * combinations of aspects for a particular configured target, so it would result in a + * combinatiorial explosion of Skyframe nodes. + */ + private static ListMultimap<Attribute, ConfiguredTarget> mergeAspects( + ListMultimap<Attribute, Dependency> depValueNames, + Map<SkyKey, ConfiguredTarget> depConfiguredTargetMap, + ListMultimap<SkyKey, Aspect> depAspectMap) { + ListMultimap<Attribute, ConfiguredTarget> result = ArrayListMultimap.create(); + + for (Map.Entry<Attribute, Dependency> entry : depValueNames.entries()) { + Dependency dep = entry.getValue(); + SkyKey depKey = TO_KEYS.apply(dep); + ConfiguredTarget depConfiguredTarget = depConfiguredTargetMap.get(depKey); + result.put(entry.getKey(), + RuleConfiguredTarget.mergeAspects(depConfiguredTarget, depAspectMap.get(depKey))); + } + + return result; + } + + /** + * Given a list of {@link Dependency} objects, returns a multimap from the {@link SkyKey} of the + * dependency to the {@link Aspect} instances that should be merged into it. + * + * <p>Returns null if the required aspects are not computed yet. + */ + @Nullable + private static ListMultimap<SkyKey, Aspect> resolveAspectDependencies(Environment env, + Map<SkyKey, ConfiguredTarget> configuredTargetMap, Iterable<Dependency> deps) + throws DependencyEvaluationException { + ListMultimap<SkyKey, Aspect> result = ArrayListMultimap.create(); + Set<SkyKey> aspectKeys = new HashSet<>(); + for (Dependency dep : deps) { + for (Class<? extends ConfiguredAspectFactory> depAspect : dep.getAspects()) { + aspectKeys.add(AspectValue.key(dep.getLabel(), dep.getConfiguration(), depAspect)); + } + } + + Map<SkyKey, ValueOrException3< + AspectCreationException, NoSuchThingException, ConfiguredValueCreationException>> + depAspects = env.getValuesOrThrow(aspectKeys, AspectCreationException.class, + NoSuchThingException.class, ConfiguredValueCreationException.class); + + for (Dependency dep : deps) { + SkyKey depKey = TO_KEYS.apply(dep); + ConfiguredTarget depConfiguredTarget = configuredTargetMap.get(depKey); + List<AspectValue> aspects = new ArrayList<>(); + for (Class<? extends ConfiguredAspectFactory> depAspect : dep.getAspects()) { + if (!aspectMatchesConfiguredTarget(depConfiguredTarget, depAspect)) { + continue; + } + + SkyKey aspectKey = AspectValue.key(dep.getLabel(), dep.getConfiguration(), depAspect); + AspectValue aspectValue = null; + try { + aspectValue = (AspectValue) depAspects.get(aspectKey).get(); + } catch (ConfiguredValueCreationException e) { + // The configured target should have been created in resolveConfiguredTargetDependencies() + throw new IllegalStateException(e); + } catch (NoSuchThingException | AspectCreationException e) { + AspectFactory depAspectFactory = AspectFactory.Util.create(depAspect); + throw new DependencyEvaluationException(new ConfiguredValueCreationException( + String.format("Evaluation of aspect %s on %s failed: %s", + depAspectFactory.getDefinition().getName(), dep.getLabel(), e.toString()))); + } + + if (aspectValue == null) { + // Dependent aspect has either not been computed yet or is in error. + return null; + } + result.put(depKey, aspectValue.get()); + } + } + + return result; + } + + private static boolean aspectMatchesConfiguredTarget(ConfiguredTarget dep, + Class<? extends ConfiguredAspectFactory> aspectFactory) { + AspectDefinition aspectDefinition = AspectFactory.Util.create(aspectFactory).getDefinition(); + for (Class<?> provider : aspectDefinition.getRequiredProviders()) { + if (dep.getProvider((Class<? extends TransitiveInfoProvider>) provider) == null) { + return false; + } + } + + return true; + } + + /** + * Returns which aspects are computable based on the precise set of providers direct dependencies + * publish (and not the upper estimate in their rule definition). + * + * <p>An aspect is computable for a particular configured target if the configured target supplies + * all the providers the aspect requires. + * + * @param upperEstimate a multimap from attribute to the upper estimates computed by + * {@link com.google.devtools.build.lib.analysis.DependencyResolver}. + * @param configuredTargetDeps a multimap from attribute to the directly dependent configured + * targets + * @return a multimap from attribute to the more precise {@link Dependency} objects + */ + private static ListMultimap<Attribute, Dependency> getComputableAspects( + ListMultimap<Attribute, Dependency> upperEstimate, + Map<SkyKey, ConfiguredTarget> configuredTargetDeps) { + ListMultimap<Attribute, Dependency> result = ArrayListMultimap.create(); + for (Map.Entry<Attribute, Dependency> entry : upperEstimate.entries()) { + ConfiguredTarget dep = + configuredTargetDeps.get(TO_KEYS.apply(entry.getValue())); + List<Class<? extends ConfiguredAspectFactory>> depAspects = new ArrayList<>(); + for (Class<? extends ConfiguredAspectFactory> candidate : entry.getValue().getAspects()) { + boolean ok = true; + for (Class<?> requiredProvider : + AspectFactory.Util.create(candidate).getDefinition().getRequiredProviders()) { + if (dep.getProvider((Class<? extends TransitiveInfoProvider>) requiredProvider) == null) { + ok = false; + break; + } + } + + if (ok) { + depAspects.add(candidate); + } + } + + result.put(entry.getKey(), new Dependency( + entry.getValue().getLabel(), entry.getValue().getConfiguration(), + ImmutableSet.copyOf(depAspects))); + } + + return result; + } + + /** + * Returns the set of {@link ConfigMatchingProvider}s that key the configurable attributes + * used by this rule. + * + * <p>>If the configured targets supplying those providers aren't yet resolved by the + * dependency resolver, returns null. + */ + @Nullable + static Set<ConfigMatchingProvider> getConfigConditions(Target target, Environment env, + SkyframeDependencyResolver resolver, TargetAndConfiguration ctgValue) + throws DependencyEvaluationException { + if (!(target instanceof Rule)) { + return ImmutableSet.of(); + } + + ImmutableSet.Builder<ConfigMatchingProvider> configConditions = ImmutableSet.builder(); + + // Collect the labels of the configured targets we need to resolve. + ListMultimap<Attribute, LabelAndConfiguration> configLabelMap = ArrayListMultimap.create(); + RawAttributeMapper attributeMap = RawAttributeMapper.of(((Rule) target)); + for (Attribute a : ((Rule) target).getAttributes()) { + for (Label configLabel : attributeMap.getConfigurabilityKeys(a.getName(), a.getType())) { + if (!Type.Selector.isReservedLabel(configLabel)) { + configLabelMap.put(a, LabelAndConfiguration.of( + configLabel, ctgValue.getConfiguration())); + } + } + } + if (configLabelMap.isEmpty()) { + return ImmutableSet.of(); + } + + // Collect the corresponding Skyframe configured target values. Abort early if they haven't + // been computed yet. + Collection<Dependency> configValueNames = + resolver.resolveRuleLabels(ctgValue, null, configLabelMap); + Map<SkyKey, ConfiguredTarget> configValues = + resolveConfiguredTargetDependencies(env, configValueNames, target); + if (configValues == null) { + return null; + } + + // Get the configured targets as ConfigMatchingProvider interfaces. + for (Dependency entry : configValueNames) { + ConfiguredTarget value = configValues.get(TO_KEYS.apply(entry)); + // The code above guarantees that value is non-null here. + ConfigMatchingProvider provider = value.getProvider(ConfigMatchingProvider.class); + if (provider != null) { + configConditions.add(provider); + } else { + // Not a valid provider for configuration conditions. + String message = + entry.getLabel() + " is not a valid configuration key for " + target.getLabel(); + env.getListener().handle(Event.error(TargetUtils.getLocationMaybe(target), message)); + throw new DependencyEvaluationException(new ConfiguredValueCreationException(message)); + } + } + + return configConditions.build(); + } + + /*** + * Resolves the targets referenced in depValueNames and returns their ConfiguredTarget + * instances. + * + * <p>Returns null if not all instances are available yet. + * + */ + @Nullable + private static Map<SkyKey, ConfiguredTarget> resolveConfiguredTargetDependencies( + Environment env, Collection<Dependency> deps, Target target) + throws DependencyEvaluationException { + boolean ok = !env.valuesMissing(); + String message = null; + Iterable<SkyKey> depKeys = Iterables.transform(deps, TO_KEYS); + // TODO(bazel-team): maybe having a two-exception argument is better than typing a generic + // Exception here. + Map<SkyKey, ValueOrException2<NoSuchTargetException, + NoSuchPackageException>> depValuesOrExceptions = env.getValuesOrThrow(depKeys, + NoSuchTargetException.class, NoSuchPackageException.class); + Map<SkyKey, ConfiguredTarget> depValues = new HashMap<>(depValuesOrExceptions.size()); + SkyKey childKey = null; + NoSuchThingException transitiveChildException = null; + for (Map.Entry<SkyKey, ValueOrException2<NoSuchTargetException, NoSuchPackageException>> entry + : depValuesOrExceptions.entrySet()) { + ConfiguredTargetKey depKey = (ConfiguredTargetKey) entry.getKey().argument(); + LabelAndConfiguration depLabelAndConfiguration = LabelAndConfiguration.of( + depKey.getLabel(), depKey.getConfiguration()); + Label depLabel = depLabelAndConfiguration.getLabel(); + ConfiguredTargetValue depValue = null; + NoSuchThingException directChildException = null; + try { + depValue = (ConfiguredTargetValue) entry.getValue().get(); + } catch (NoSuchTargetException e) { + if (depLabel.equals(e.getLabel())) { + directChildException = e; + } else { + childKey = entry.getKey(); + transitiveChildException = e; + } + } catch (NoSuchPackageException e) { + if (depLabel.getPackageName().equals(e.getPackageName())) { + directChildException = e; + } else { + childKey = entry.getKey(); + transitiveChildException = e; + } + } + // If an exception wasn't caused by a direct child target value, we'll treat it the same + // as any other missing dep by setting ok = false below, and returning null at the end. + if (directChildException != null) { + // Only update messages for missing targets we depend on directly. + message = TargetUtils.formatMissingEdge(target, depLabel, directChildException); + env.getListener().handle(Event.error(TargetUtils.getLocationMaybe(target), message)); + } + + if (depValue == null) { + ok = false; + } else { + depValues.put(entry.getKey(), depValue.getConfiguredTarget()); + } + } + if (message != null) { + throw new DependencyEvaluationException(new NoSuchTargetException(message)); + } + if (childKey != null) { + throw new DependencyEvaluationException(childKey, transitiveChildException); + } + if (!ok) { + return null; + } else { + return depValues; + } + } + + + @Override + public String extractTag(SkyKey skyKey) { + return Label.print(((ConfiguredTargetKey) skyKey.argument()).getLabel()); + } + + @Nullable + private ConfiguredTargetValue createConfiguredTarget(SkyframeBuildView view, + Environment env, Target target, BuildConfiguration configuration, + ListMultimap<Attribute, ConfiguredTarget> depValueMap, + Set<ConfigMatchingProvider> configConditions) + throws ConfiguredTargetFunctionException, + InterruptedException { + boolean extendedSanityChecks = configuration != null && configuration.extendedSanityChecks(); + + StoredEventHandler events = new StoredEventHandler(); + BuildConfiguration ownerConfig = (configuration == null) + ? null : configuration.getArtifactOwnerConfiguration(); + boolean allowRegisteringActions = configuration == null || configuration.isActionsEnabled(); + CachingAnalysisEnvironment analysisEnvironment = view.createAnalysisEnvironment( + new ConfiguredTargetKey(target.getLabel(), ownerConfig), false, + extendedSanityChecks, events, env, allowRegisteringActions); + if (env.valuesMissing()) { + return null; + } + + ConfiguredTarget configuredTarget = view.createConfiguredTarget(target, configuration, + analysisEnvironment, depValueMap, configConditions); + + events.replayOn(env.getListener()); + if (events.hasErrors()) { + analysisEnvironment.disable(target); + throw new ConfiguredTargetFunctionException(new ConfiguredValueCreationException( + "Analysis of target '" + target.getLabel() + "' failed; build aborted")); + } + Preconditions.checkState(!analysisEnvironment.hasErrors(), + "Analysis environment hasError() but no errors reported"); + if (env.valuesMissing()) { + return null; + } + + analysisEnvironment.disable(target); + Preconditions.checkNotNull(configuredTarget, target); + + return new ConfiguredTargetValue(configuredTarget, + ImmutableList.copyOf(analysisEnvironment.getRegisteredActions())); + } + + /** + * An exception indicating that there was a problem during the construction of + * a ConfiguredTargetValue. + */ + public static final class ConfiguredValueCreationException extends Exception { + + public ConfiguredValueCreationException(String message) { + super(message); + } + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link ConfiguredTargetFunction#compute}. + */ + public static final class ConfiguredTargetFunctionException extends SkyFunctionException { + public ConfiguredTargetFunctionException(NoSuchTargetException e) { + super(e, Transience.PERSISTENT); + } + + private ConfiguredTargetFunctionException(ConfiguredValueCreationException error) { + super(error, Transience.PERSISTENT); + }; + + private ConfiguredTargetFunctionException( + @Nullable SkyKey childKey, Exception transitiveError) { + super(transitiveError, childKey); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java new file mode 100644 index 0000000..ea744c1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java
@@ -0,0 +1,96 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyFunctionName; + +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * A (Label, Configuration) pair. Note that this pair may be used to look up the generating action + * of an artifact. Callers may want to ensure that they have the correct configuration for this + * purpose by passing in {@link BuildConfiguration#getArtifactOwnerConfiguration} in preference to + * the raw configuration. + */ +public class ConfiguredTargetKey extends ActionLookupValue.ActionLookupKey { + private final Label label; + @Nullable + private final BuildConfiguration configuration; + + public ConfiguredTargetKey(Label label, @Nullable BuildConfiguration configuration) { + this.label = Preconditions.checkNotNull(label); + this.configuration = configuration; + } + + public ConfiguredTargetKey(ConfiguredTarget rule) { + this(rule.getTarget().getLabel(), rule.getConfiguration()); + } + + @Override + public Label getLabel() { + return label; + } + + @Override + SkyFunctionName getType() { + return SkyFunctions.CONFIGURED_TARGET; + } + + @Nullable + public BuildConfiguration getConfiguration() { + return configuration; + } + + @Override + public int hashCode() { + int configVal = configuration == null ? 79 : configuration.hashCode(); + return 31 * label.hashCode() + configVal; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof ConfiguredTargetKey)) { + return false; + } + ConfiguredTargetKey other = (ConfiguredTargetKey) obj; + return Objects.equals(label, other.label) && Objects.equals(configuration, other.configuration); + } + + public String prettyPrint() { + if (label == null) { + return "null"; + } + return (configuration != null && configuration.isHostConfiguration()) + ? (label.toString() + " (host)") : label.toString(); + } + + @Override + public String toString() { + return label + " " + (configuration == null ? "null" : configuration.shortCacheKey()); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetValue.java new file mode 100644 index 0000000..200e05f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetValue.java
@@ -0,0 +1,105 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyKey; + +import javax.annotation.Nullable; + +/** + * A configured target in the context of a Skyframe graph. + */ +@Immutable +@ThreadSafe +@VisibleForTesting +public final class ConfiguredTargetValue extends ActionLookupValue { + + // These variables are only non-final because they may be clear()ed to save memory. They are null + // only after they are cleared. + @Nullable private ConfiguredTarget configuredTarget; + + // We overload this variable to check whether the value has been clear()ed. We don't use a + // separate variable in order to save memory. + @Nullable private volatile Iterable<Action> actions; + + ConfiguredTargetValue(ConfiguredTarget configuredTarget, Iterable<Action> actions) { + super(actions); + this.configuredTarget = configuredTarget; + this.actions = actions; + } + + @VisibleForTesting + public ConfiguredTarget getConfiguredTarget() { + Preconditions.checkNotNull(actions, configuredTarget); + return configuredTarget; + } + + @VisibleForTesting + public Iterable<Action> getActions() { + return Preconditions.checkNotNull(actions, configuredTarget); + } + + /** + * Clears configured target data from this value, leaving only the artifact->generating action + * map. + * + * <p>Should only be used when user specifies --discard_analysis_cache. Must be called at most + * once per value, after which {@link #getConfiguredTarget} and {@link #getActions} cannot be + * called. + */ + public void clear() { + Preconditions.checkNotNull(actions, configuredTarget); + configuredTarget = null; + actions = null; + } + + @VisibleForTesting + public static SkyKey key(Label label, BuildConfiguration configuration) { + return key(new ConfiguredTargetKey(label, configuration)); + } + + static ImmutableList<SkyKey> keys(Iterable<ConfiguredTargetKey> lacs) { + ImmutableList.Builder<SkyKey> keys = ImmutableList.builder(); + for (ConfiguredTargetKey lac : lacs) { + keys.add(key(lac)); + } + return keys.build(); + } + + /** + * Returns a label of ConfiguredTargetValue. + */ + @ThreadSafe + static Label extractLabel(SkyKey value) { + Object valueName = value.argument(); + Preconditions.checkState(valueName instanceof ConfiguredTargetKey, valueName); + return ((ConfiguredTargetKey) valueName).getLabel(); + } + + @Override + public String toString() { + return "ConfiguredTargetValue: " + + configuredTarget + ", actions: " + (actions == null ? null : Iterables.toString(actions)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupFunction.java new file mode 100644 index 0000000..58cb67d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupFunction.java
@@ -0,0 +1,55 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import javax.annotation.Nullable; + +/** + * SkyFunction for {@link ContainingPackageLookupValue}s. + */ +public class ContainingPackageLookupFunction implements SkyFunction { + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + PackageIdentifier dir = (PackageIdentifier) skyKey.argument(); + PackageLookupValue pkgLookupValue = null; + pkgLookupValue = (PackageLookupValue) env.getValue(PackageLookupValue.key(dir)); + if (pkgLookupValue == null) { + return null; + } + + if (pkgLookupValue.packageExists()) { + return ContainingPackageLookupValue.withContainingPackage(dir, pkgLookupValue.getRoot()); + } + + PathFragment parentDir = dir.getPackageFragment().getParentDirectory(); + if (parentDir == null) { + return ContainingPackageLookupValue.noContainingPackage(); + } + PackageIdentifier parentId = new PackageIdentifier(dir.getRepository(), parentDir); + return env.getValue(ContainingPackageLookupValue.key(parentId)); + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupValue.java new file mode 100644 index 0000000..16516b5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupValue.java
@@ -0,0 +1,111 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * A value that represents the result of looking for the existence of a package that owns a + * specific directory path. Compare with {@link PackageLookupValue}, which deals with existence of + * a specific package. + */ +public abstract class ContainingPackageLookupValue implements SkyValue { + /** Returns whether there is a containing package. */ + public abstract boolean hasContainingPackage(); + + /** If there is a containing package, returns its name. */ + abstract PackageIdentifier getContainingPackageName(); + + /** If there is a containing package, returns its package root */ + public abstract Path getContainingPackageRoot(); + + public static SkyKey key(PackageIdentifier id) { + Preconditions.checkArgument(!id.getPackageFragment().isAbsolute(), id); + return new SkyKey(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, id); + } + + static ContainingPackageLookupValue noContainingPackage() { + return NoContainingPackage.INSTANCE; + } + + static ContainingPackageLookupValue withContainingPackage(PackageIdentifier pkgId, Path root) { + return new ContainingPackage(pkgId, root); + } + + private static class NoContainingPackage extends ContainingPackageLookupValue { + private static final NoContainingPackage INSTANCE = new NoContainingPackage(); + + @Override + public boolean hasContainingPackage() { + return false; + } + + @Override + public PackageIdentifier getContainingPackageName() { + throw new IllegalStateException(); + } + + @Override + public Path getContainingPackageRoot() { + throw new IllegalStateException(); + } + } + + private static class ContainingPackage extends ContainingPackageLookupValue { + private final PackageIdentifier containingPackage; + private final Path containingPackageRoot; + + private ContainingPackage(PackageIdentifier pkgId, Path containingPackageRoot) { + this.containingPackage = pkgId; + this.containingPackageRoot = containingPackageRoot; + } + + @Override + public boolean hasContainingPackage() { + return true; + } + + @Override + public PackageIdentifier getContainingPackageName() { + return containingPackage; + } + + @Override + public Path getContainingPackageRoot() { + return containingPackageRoot; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof ContainingPackage)) { + return false; + } + ContainingPackage other = (ContainingPackage) obj; + return containingPackage.equals(other.containingPackage) + && containingPackageRoot.equals(other.containingPackageRoot); + } + + @Override + public int hashCode() { + return containingPackage.hashCode(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportFunction.java new file mode 100644 index 0000000..8d6fe3d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportFunction.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * A Skyframe function to calculate the coverage report Action and Artifacts. + */ +public class CoverageReportFunction implements SkyFunction { + CoverageReportFunction() {} + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + Preconditions.checkState( + CoverageReportValue.SKY_KEY.equals(skyKey), String.format( + "Expected %s for SkyKey but got %s instead", CoverageReportValue.SKY_KEY, skyKey)); + + Action action = PrecomputedValue.COVERAGE_REPORT_KEY.get(env); + if (action == null) { + return null; + } + + return new CoverageReportValue( + action.getOutputs(), + action); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportValue.java new file mode 100644 index 0000000..862e381 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/CoverageReportValue.java
@@ -0,0 +1,55 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; + +/** + * A SkyValue to store the coverage report Action and Artifacts. + */ +public class CoverageReportValue extends ActionLookupValue { + private final ImmutableSet<Artifact> coverageReportArtifacts; + + // There should only ever be one CoverageReportValue value in the graph. + public static final SkyKey SKY_KEY = new SkyKey(SkyFunctions.COVERAGE_REPORT, "COVERAGE_REPORT"); + public static final ArtifactOwner ARTIFACT_OWNER = new CoverageReportKey(); + + public CoverageReportValue(ImmutableSet<Artifact> coverageReportArtifacts, + Action coverageReportAction) { + super(coverageReportAction); + this.coverageReportArtifacts = coverageReportArtifacts; + } + + public ImmutableSet<Artifact> getCoverageReportArtifacts() { + return coverageReportArtifacts; + } + + private static class CoverageReportKey extends ActionLookupKey { + @Override + SkyFunctionName getType() { + throw new UnsupportedOperationException(); + } + + @Override + SkyKey getSkyKey() { + return SKY_KEY; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwareness.java b/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwareness.java new file mode 100644 index 0000000..d0f4c99 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwareness.java
@@ -0,0 +1,82 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.vfs.ModifiedFileSet; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.Closeable; + +import javax.annotation.Nullable; + +/** + * Interface for computing modifications of files under a package path entry. + * + * <p> Skyframe has a {@link DiffAwareness} instance per package-path entry, and each instance is + * responsible for all files under its path entry. At the beginning of each incremental build, + * skyframe queries for changes using {@link #getDiff}. Ideally, {@link #getDiff} should be + * constant-time; if it were linear in the number of files of interest, we might as well just + * detect modifications manually. + */ +public interface DiffAwareness extends Closeable { + + /** Factory for creating {@link DiffAwareness} instances. */ + public interface Factory { + /** + * Returns a {@link DiffAwareness} instance suitable for managing changes to files under the + * given package path entry, or {@code null} if this factory cannot create such an instance. + * + * <p> Skyframe has a collection of factories, and will create a {@link DiffAwareness} instance + * per package path entry using one of the factories that returns a non-null value. + */ + @Nullable + DiffAwareness maybeCreate(Path pathEntry); + } + + /** Opaque view of the filesystem under a package path entry at a specific point in time. */ + interface View { + } + + /** + * Returns the live view of the filesystem under the package path entry. + * + * @throws BrokenDiffAwarenessException if something is wrong and the caller should discard this + * {@link DiffAwareness} instance. The {@link DiffAwareness} is expected to close itself in + * this case. + */ + View getCurrentView() throws BrokenDiffAwarenessException; + + /** + * Returns the set of files of interest that have been modified between the given two views. + * + * <p>The given views must have come from previous calls to {@link #getCurrentView} on the + * {@link DiffAwareness} instance (i.e. using a {@link View} from another instance is not + * supported). + * + * @throws IncompatibleViewException if the given views are not compatible with this + * {@link DiffAwareness} instance. This probably indicates a bug. + * @throws BrokenDiffAwarenessException if something is wrong and the caller should discard this + * {@link DiffAwareness} instance. The {@link DiffAwareness} is expected to close itself in + * this case. + */ + ModifiedFileSet getDiff(View oldView, View newView) + throws IncompatibleViewException, BrokenDiffAwarenessException; + + /** + * Must be called whenever the {@link DiffAwareness} object is to be discarded. Using a + * {@link DiffAwareness} instance after calling {@link #close} on it is unspecified behavior. + */ + @Override + void close(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwarenessManager.java b/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwarenessManager.java new file mode 100644 index 0000000..d1bb81e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/DiffAwarenessManager.java
@@ -0,0 +1,188 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.skyframe.DiffAwareness.View; +import com.google.devtools.build.lib.vfs.ModifiedFileSet; +import com.google.devtools.build.lib.vfs.Path; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Helper class to make it easier to correctly use the {@link DiffAwareness} interface in a + * sequential manner. + */ +public final class DiffAwarenessManager { + + private final ImmutableSet<? extends DiffAwareness.Factory> diffAwarenessFactories; + private Map<Path, DiffAwarenessState> currentDiffAwarenessStates = Maps.newHashMap(); + private final Reporter reporter; + + public DiffAwarenessManager(Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories, + Reporter reporter) { + this.diffAwarenessFactories = ImmutableSet.copyOf(diffAwarenessFactories); + this.reporter = reporter; + } + + private static class DiffAwarenessState { + private final DiffAwareness diffAwareness; + /** + * The {@link View} that should be the baseline for the next {@link #getDiff} call, or + * {@code null} if the next {@link #getDiff} will be the first incremental one. + */ + @Nullable + private View baselineView; + + private DiffAwarenessState(DiffAwareness diffAwareness, @Nullable View baselineView) { + this.diffAwareness = diffAwareness; + this.baselineView = baselineView; + } + } + + /** Reset internal {@link DiffAwareness} state. */ + public void reset() { + for (DiffAwarenessState diffAwarenessState : currentDiffAwarenessStates.values()) { + diffAwarenessState.diffAwareness.close(); + } + currentDiffAwarenessStates.clear(); + } + + /** A set of modified files that should be marked as processed. */ + public interface ProcessableModifiedFileSet { + ModifiedFileSet getModifiedFileSet(); + + /** + * This should be called when the changes have been noted. Otherwise, the result from the next + * call to {@link #getDiff} will be from the baseline of the old, unprocessed, diff. + */ + void markProcessed(); + } + + /** + * Gets the set of changed files since the last call with this path entry, or + * {@code ModifiedFileSet.EVERYTHING_MODIFIED} if this is the first such call. + */ + public ProcessableModifiedFileSet getDiff(Path pathEntry) { + DiffAwarenessState diffAwarenessState = maybeGetDiffAwarenessState(pathEntry); + if (diffAwarenessState == null) { + return BrokenProcessableModifiedFileSet.INSTANCE; + } + DiffAwareness diffAwareness = diffAwarenessState.diffAwareness; + View newView; + try { + newView = diffAwareness.getCurrentView(); + } catch (BrokenDiffAwarenessException e) { + handleBrokenDiffAwareness(pathEntry, e); + return BrokenProcessableModifiedFileSet.INSTANCE; + } + + View baselineView = diffAwarenessState.baselineView; + if (baselineView == null) { + diffAwarenessState.baselineView = newView; + return BrokenProcessableModifiedFileSet.INSTANCE; + } + + ModifiedFileSet diff; + try { + diff = diffAwareness.getDiff(baselineView, newView); + } catch (BrokenDiffAwarenessException e) { + handleBrokenDiffAwareness(pathEntry, e); + return BrokenProcessableModifiedFileSet.INSTANCE; + } catch (IncompatibleViewException e) { + throw new IllegalStateException(pathEntry + " " + baselineView + " " + newView, e); + } + ProcessableModifiedFileSet result = new ProcessableModifiedFileSetImpl(diff, pathEntry, + newView); + return result; + } + + private void handleBrokenDiffAwareness(Path pathEntry, BrokenDiffAwarenessException e) { + currentDiffAwarenessStates.remove(pathEntry); + reporter.handle(Event.warn(e.getMessage() + "... temporarily falling back to manually " + + "checking files for changes")); + } + + /** + * Returns the current diff awareness for the given path entry, or a fresh one if there is no + * current one, or otherwise {@code null} if no factory could make a fresh one. + */ + @Nullable + private DiffAwarenessState maybeGetDiffAwarenessState(Path pathEntry) { + DiffAwarenessState diffAwarenessState = currentDiffAwarenessStates.get(pathEntry); + if (diffAwarenessState != null) { + return diffAwarenessState; + } + for (DiffAwareness.Factory factory : diffAwarenessFactories) { + DiffAwareness newDiffAwareness = factory.maybeCreate(pathEntry); + if (newDiffAwareness != null) { + diffAwarenessState = new DiffAwarenessState(newDiffAwareness, /*previousView=*/null); + currentDiffAwarenessStates.put(pathEntry, diffAwarenessState); + return diffAwarenessState; + } + } + return null; + } + + private class ProcessableModifiedFileSetImpl implements ProcessableModifiedFileSet { + + private final ModifiedFileSet modifiedFileSet; + private final Path pathEntry; + /** + * The {@link View} that should be the baseline on the next {@link #getDiff} call after + * {@link #markProcessed} is called. + */ + private final View nextView; + + private ProcessableModifiedFileSetImpl(ModifiedFileSet modifiedFileSet, Path pathEntry, + View nextView) { + this.modifiedFileSet = modifiedFileSet; + this.pathEntry = pathEntry; + this.nextView = nextView; + } + + @Override + public ModifiedFileSet getModifiedFileSet() { + return modifiedFileSet; + } + + @Override + public void markProcessed() { + DiffAwarenessState diffAwarenessState = currentDiffAwarenessStates.get(pathEntry); + if (diffAwarenessState != null) { + diffAwarenessState.baselineView = nextView; + } + } + } + + private static class BrokenProcessableModifiedFileSet implements ProcessableModifiedFileSet { + + private static final BrokenProcessableModifiedFileSet INSTANCE = + new BrokenProcessableModifiedFileSet(); + + @Override + public ModifiedFileSet getModifiedFileSet() { + return ModifiedFileSet.EVERYTHING_MODIFIED; + } + + @Override + public void markProcessed() { + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingFunction.java new file mode 100644 index 0000000..93c3d75 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingFunction.java
@@ -0,0 +1,72 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import javax.annotation.Nullable; + +/** + * A {@link SkyFunction} for {@link DirectoryListingValue}s. + */ +final class DirectoryListingFunction implements SkyFunction { + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) + throws DirectoryListingFunctionException { + RootedPath dirRootedPath = (RootedPath) skyKey.argument(); + + FileValue dirFileValue = (FileValue) env.getValue(FileValue.key(dirRootedPath)); + if (dirFileValue == null) { + return null; + } + + RootedPath realDirRootedPath = dirFileValue.realRootedPath(); + if (!dirFileValue.isDirectory()) { + // Recall that the directory is assumed to exist (see DirectoryListingValue#key). + throw new DirectoryListingFunctionException(new InconsistentFilesystemException( + dirRootedPath.asPath() + " is no longer an existing directory. Did you delete it during " + + "the build?")); + } + + DirectoryListingStateValue directoryListingStateValue = + (DirectoryListingStateValue) env.getValue(DirectoryListingStateValue.key( + realDirRootedPath)); + if (directoryListingStateValue == null) { + return null; + } + + return DirectoryListingValue.value(dirRootedPath, dirFileValue, directoryListingStateValue); + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link DirectoryListingFunction#compute}. + */ + private static final class DirectoryListingFunctionException extends SkyFunctionException { + public DirectoryListingFunctionException(InconsistentFilesystemException e) { + super(e, Transience.TRANSIENT); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java new file mode 100644 index 0000000..6e47a2d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateFunction.java
@@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; + +/** + * A {@link SkyFunction} for {@link DirectoryListingStateValue}s. + * + * <p>Merely calls DirectoryListingStateValue#create, but also has special handling for + * directories outside the package roots (see {@link ExternalFilesHelper}). + */ +public class DirectoryListingStateFunction implements SkyFunction { + + private final ExternalFilesHelper externalFilesHelper; + + public DirectoryListingStateFunction(ExternalFilesHelper externalFilesHelper) { + this.externalFilesHelper = externalFilesHelper; + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) + throws DirectoryListingStateFunctionException { + RootedPath dirRootedPath = (RootedPath) skyKey.argument(); + externalFilesHelper.maybeAddDepOnBuildId(dirRootedPath, env); + if (env.valuesMissing()) { + return null; + } + try { + return DirectoryListingStateValue.create(dirRootedPath); + } catch (IOException e) { + throw new DirectoryListingStateFunctionException(e); + } + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link DirectoryListingStateFunction#compute}. + */ + private static final class DirectoryListingStateFunctionException + extends SkyFunctionException { + public DirectoryListingStateFunctionException(IOException e) { + super(e, Transience.TRANSIENT); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateValue.java new file mode 100644 index 0000000..87b9748 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateValue.java
@@ -0,0 +1,214 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.vfs.Dirent; +import com.google.devtools.build.lib.vfs.Dirent.Type; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.io.Serializable; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Objects; + +/** + * Encapsulates the filesystem operations needed to get the directory entries of a directory. + * + * <p>This class is an implementation detail of {@link DirectoryListingValue}. + */ +final class DirectoryListingStateValue implements SkyValue { + + private final CompactSortedDirents compactSortedDirents; + + private DirectoryListingStateValue(Collection<Dirent> dirents) { + this.compactSortedDirents = CompactSortedDirents.create(dirents); + } + + @VisibleForTesting + public static DirectoryListingStateValue createForTesting(Collection<Dirent> dirents) { + return new DirectoryListingStateValue(dirents); + } + + public static DirectoryListingStateValue create(RootedPath dirRootedPath) throws IOException { + Collection<Dirent> dirents = dirRootedPath.asPath().readdir(Symlinks.NOFOLLOW); + return new DirectoryListingStateValue(dirents); + } + + @ThreadSafe + public static SkyKey key(RootedPath rootedPath) { + return new SkyKey(SkyFunctions.DIRECTORY_LISTING_STATE, rootedPath); + } + + /** + * Returns the directory entries for this directory, in a stable order. + * + * <p>Symlinks are not expanded. + */ + public Iterable<Dirent> getDirents() { + return compactSortedDirents; + } + + @Override + public int hashCode() { + return compactSortedDirents.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof DirectoryListingStateValue)) { + return false; + } + DirectoryListingStateValue other = (DirectoryListingStateValue) obj; + return compactSortedDirents.equals(other.compactSortedDirents); + } + + /** A space-efficient, sorted, immutable dirent structure. */ + private static class CompactSortedDirents implements Iterable<Dirent>, Serializable { + + private final String[] names; + private final BitSet packedTypes; + + private CompactSortedDirents(String[] names, BitSet packedTypes) { + this.names = names; + this.packedTypes = packedTypes; + } + + public static CompactSortedDirents create(Collection<Dirent> dirents) { + final Dirent[] direntArray = dirents.toArray(new Dirent[dirents.size()]); + Integer[] indices = new Integer[dirents.size()]; + for (int i = 0; i < dirents.size(); i++) { + indices[i] = i; + } + Arrays.sort(indices, + new Comparator<Integer>() { + @Override + public int compare(Integer o1, Integer o2) { + return direntArray[o1].getName().compareTo(direntArray[o2].getName()); + } + }); + String[] names = new String[dirents.size()]; + BitSet packedTypes = new BitSet(dirents.size() * 2); + for (int i = 0; i < dirents.size(); i++) { + Dirent dirent = direntArray[indices[i]]; + names[i] = dirent.getName(); + packType(packedTypes, dirent.getType(), i); + } + return new CompactSortedDirents(names, packedTypes); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof CompactSortedDirents)) { + return false; + } + if (this == obj) { + return true; + } + CompactSortedDirents other = (CompactSortedDirents) obj; + return Arrays.equals(names, other.names) && packedTypes.equals(other.packedTypes); + } + + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(names), packedTypes); + } + + @Override + public Iterator<Dirent> iterator() { + return new Iterator<Dirent>() { + + private int i = 0; + + @Override + public boolean hasNext() { + return i < size(); + } + + @Override + public Dirent next() { + return direntAt(i++); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + + private int size() { + return names.length; + } + + /** Returns the type of the ith dirent. */ + private Dirent.Type unpackType(int i) { + int start = i * 2; + boolean upper = packedTypes.get(start); + boolean lower = packedTypes.get(start + 1); + if (!upper && !lower) { + return Type.FILE; + } else if (!upper && lower){ + return Type.DIRECTORY; + } else if (upper && !lower) { + return Type.SYMLINK; + } else { + return Type.UNKNOWN; + } + } + + /** Sets the type of the ith dirent. */ + private static void packType(BitSet bitSet, Dirent.Type type, int i) { + int start = i * 2; + switch (type) { + case FILE: + pack(bitSet, start, false, false); + break; + case DIRECTORY: + pack(bitSet, start, false, true); + break; + case SYMLINK: + pack(bitSet, start, true, false); + break; + case UNKNOWN: + pack(bitSet, start, true, true); + break; + default: + throw new IllegalStateException("Unknown dirent type: " + type); + } + } + + private static void pack(BitSet bitSet, int start, boolean upper, boolean lower) { + bitSet.set(start, upper); + bitSet.set(start + 1, lower); + } + + private Dirent direntAt(int i) { + Preconditions.checkState(i >= 0 && i < size(), "i: %s, size: %s", i, size()); + return new Dirent(names[i], unpackType(i)); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingValue.java new file mode 100644 index 0000000..3fe6dba --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingValue.java
@@ -0,0 +1,134 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.vfs.Dirent; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.Objects; + +/** + * A value that represents the list of files in a given directory under a given package path root. + * Anything in Skyframe that cares about the contents of a directory should have a dependency + * on the corresponding {@link DirectoryListingValue}. + * + * <p>This value only depends on the FileValue corresponding to the directory. In particular, note + * that it does not depend on any of its child entries. + * + * <p>Note that symlinks in dirents are <b>not</b> expanded. Dependents of the value are responsible + * for expanding the symlink entries by referring to FileValues that correspond to the symlinks. + * This is a little onerous, but correct: we do not need to reread the directory when a symlink + * inside it changes, therefore this value should not be invalidated in that case. + */ +@Immutable +@ThreadSafe +abstract class DirectoryListingValue implements SkyValue { + + /** + * Returns the directory entries for this directory, in a stable order. + * + * <p>Symlinks are not expanded. + */ + public abstract Iterable<Dirent> getDirents(); + + /** + * Returns a {@link SkyKey} for getting the directory entries of the given directory. The + * given path is assumed to be an existing directory (e.g. via {@link FileValue#isDirectory} or + * from a directory listing on its parent directory). + */ + @ThreadSafe + static SkyKey key(RootedPath directoryUnderRoot) { + return new SkyKey(SkyFunctions.DIRECTORY_LISTING, directoryUnderRoot); + } + + static DirectoryListingValue value(RootedPath dirRootedPath, FileValue dirFileValue, + DirectoryListingStateValue realDirectoryListingStateValue) { + return dirFileValue.realRootedPath().equals(dirRootedPath) + ? new RegularDirectoryListingValue(realDirectoryListingStateValue) + : new DifferentRealPathDirectoryListingValue(dirFileValue.realRootedPath(), + realDirectoryListingStateValue); + } + + @ThreadSafe + private static final class RegularDirectoryListingValue extends DirectoryListingValue { + + private final DirectoryListingStateValue directoryListingStateValue; + + private RegularDirectoryListingValue(DirectoryListingStateValue directoryListingStateValue) { + this.directoryListingStateValue = directoryListingStateValue; + } + + @Override + public Iterable<Dirent> getDirents() { + return directoryListingStateValue.getDirents(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof RegularDirectoryListingValue)) { + return false; + } + RegularDirectoryListingValue other = (RegularDirectoryListingValue) obj; + return directoryListingStateValue.equals(other.directoryListingStateValue); + } + + @Override + public int hashCode() { + return directoryListingStateValue.hashCode(); + } + } + + @ThreadSafe + private static final class DifferentRealPathDirectoryListingValue extends DirectoryListingValue { + + private final RootedPath realDirRootedPath; + private final DirectoryListingStateValue directoryListingStateValue; + + private DifferentRealPathDirectoryListingValue(RootedPath realDirRootedPath, + DirectoryListingStateValue directoryListingStateValue) { + this.realDirRootedPath = realDirRootedPath; + this.directoryListingStateValue = directoryListingStateValue; + } + + @Override + public Iterable<Dirent> getDirents() { + return directoryListingStateValue.getDirents(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof DifferentRealPathDirectoryListingValue)) { + return false; + } + DifferentRealPathDirectoryListingValue other = (DifferentRealPathDirectoryListingValue) obj; + return realDirRootedPath.equals(other.realDirRootedPath) + && directoryListingStateValue.equals(other.directoryListingStateValue); + } + + @Override + public int hashCode() { + return Objects.hash(realDirRootedPath, directoryListingStateValue); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ErrorReadingSkylarkExtensionException.java b/src/main/java/com/google/devtools/build/lib/skyframe/ErrorReadingSkylarkExtensionException.java new file mode 100644 index 0000000..8593afb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ErrorReadingSkylarkExtensionException.java
@@ -0,0 +1,21 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +/** Indicates some sort of IO error while dealing with a Skylark extension. */ +public class ErrorReadingSkylarkExtensionException extends Exception { + public ErrorReadingSkylarkExtensionException(String message) { + super(message); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ExternalFilesHelper.java b/src/main/java/com/google/devtools/build/lib/skyframe/ExternalFilesHelper.java new file mode 100644 index 0000000..ce858de --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ExternalFilesHelper.java
@@ -0,0 +1,97 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; + +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +/** Common utilities for dealing with files outside the package roots. */ +class ExternalFilesHelper { + + private final AtomicReference<PathPackageLocator> pkgLocator; + private final Set<Path> immutableDirs; + + ExternalFilesHelper(AtomicReference<PathPackageLocator> pkgLocator) { + this(pkgLocator, ImmutableSet.<Path>of()); + } + + ExternalFilesHelper(AtomicReference<PathPackageLocator> pkgLocator, Set<Path> immutableDirs) { + this.pkgLocator = pkgLocator; + this.immutableDirs = immutableDirs; + } + + private enum FileType { + // A file inside the package roots. + INTERNAL_FILE, + + // A file outside the package roots that we may pretends is immutable. + EXTERNAL_IMMUTABLE_FILE, + + // A file outside the package roots about which we may make no other assumptions. + EXTERNAL_MUTABLE_FILE, + } + + private FileType getFileType(RootedPath rootedPath) { + // TODO(bazel-team): This is inefficient when there are a lot of package roots or there are a + // lot of immutable directories. Consider either explicitly preventing this case or using a more + // efficient approach here (e.g. use a trie for determing if a file is under an immutable + // directory). + if (!pkgLocator.get().getPathEntries().contains(rootedPath.getRoot())) { + Path path = rootedPath.asPath(); + for (Path immutableDir : immutableDirs) { + if (path.startsWith(immutableDir)) { + return FileType.EXTERNAL_IMMUTABLE_FILE; + } + } + return FileType.EXTERNAL_MUTABLE_FILE; + } + return FileType.INTERNAL_FILE; + } + + public boolean shouldAssumeImmutable(RootedPath rootedPath) { + return getFileType(rootedPath) == FileType.EXTERNAL_IMMUTABLE_FILE; + } + + public void maybeAddDepOnBuildId(RootedPath rootedPath, SkyFunction.Environment env) { + if (getFileType(rootedPath) == FileType.EXTERNAL_MUTABLE_FILE) { + // For files outside the package roots that are not assumed to be immutable, add a dependency + // on the build_id. This is sufficient for correctness; all other files will be handled by + // diff awareness of their respective package path, but these files need to be addressed + // separately. + // + // Using the build_id here seems to introduce a performance concern because the upward + // transitive closure of these external files will get eagerly invalidated on each + // incremental build (e.g. if every file had a transitive dependency on the filesystem root, + // then we'd have a big performance problem). But this a non-issue by design: + // - We don't add a dependency on the parent directory at the package root boundary, so the + // only transitive dependencies from files inside the package roots to external files are + // through symlinks. So the upwards transitive closure of external files is small. + // - The only way external source files get into the skyframe graph in the first place is + // through symlinks outside the package roots, which we neither want to encourage nor + // optimize for since it is not common. So the set of external files is small. + // + // The above reasoning doesn't hold for bazel, because external repositories + // (e.g. new_local_repository) cause lots of external symlinks to be present in the build. + // So bazel pretends that these external repositories are immutable to avoid the performance + // penalty described above. + PrecomputedValue.dependOnBuildId(env); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileAndMetadataCache.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileAndMetadataCache.java new file mode 100644 index 0000000..2a0de78 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileAndMetadataCache.java
@@ -0,0 +1,466 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; +import com.google.common.collect.Sets; +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.cache.Digest; +import com.google.devtools.build.lib.actions.cache.DigestUtils; +import com.google.devtools.build.lib.actions.cache.Metadata; +import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.FileStatus; +import com.google.devtools.build.lib.vfs.FileStatusWithDigest; +import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.protobuf.ByteString; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import javax.annotation.Nullable; + +/** + * Cache provided by an {@link ActionExecutionFunction}, allowing Blaze to obtain data from the + * graph and to inject data (e.g. file digests) back into the graph. + * + * <p>Data for the action's inputs is injected into this cache on construction, using the graph as + * the source of truth. + * + * <p>As well, this cache collects data about the action's output files, which is used in three + * ways. First, it is served as requested during action execution, primarily by the {@code + * ActionCacheChecker} when determining if the action must be rerun, and then after the action is + * run, to gather information about the outputs. Second, it is accessed by {@link + * ArtifactFunction}s in order to construct {@link ArtifactValue}s. Third, the {@link + * FilesystemValueChecker} uses it to determine the set of output files to check for inter-build + * modifications. Because all these use cases are slightly different, we must occasionally store two + * versions of the data for a value (see {@link #getAdditionalOutputData} for more. + */ +@VisibleForTesting +public class FileAndMetadataCache implements ActionInputFileCache, MetadataHandler { + /** This should never be read directly. Use {@link #getInputFileArtifactValue} instead. */ + private final Map<Artifact, FileArtifactValue> inputArtifactData; + private final Map<Artifact, Collection<Artifact>> expandedInputMiddlemen; + private final File execRoot; + private final Map<ByteString, Artifact> reverseMap = new ConcurrentHashMap<>(); + private final ConcurrentMap<Artifact, FileValue> outputArtifactData = + new ConcurrentHashMap<>(); + // See #getAdditionalOutputData for documentation of this field. + private final ConcurrentMap<Artifact, FileArtifactValue> additionalOutputData = + new ConcurrentHashMap<>(); + private final Set<Artifact> injectedArtifacts = Sets.newConcurrentHashSet(); + private final ImmutableSet<Artifact> outputs; + @Nullable private final SkyFunction.Environment env; + private final TimestampGranularityMonitor tsgm; + + private static final Interner<ByteString> BYTE_INTERNER = Interners.newWeakInterner(); + + public FileAndMetadataCache(Map<Artifact, FileArtifactValue> inputArtifactData, + Map<Artifact, Collection<Artifact>> expandedInputMiddlemen, File execRoot, + Iterable<Artifact> outputs, @Nullable SkyFunction.Environment env, + TimestampGranularityMonitor tsgm) { + this.inputArtifactData = Preconditions.checkNotNull(inputArtifactData); + this.expandedInputMiddlemen = Preconditions.checkNotNull(expandedInputMiddlemen); + this.execRoot = Preconditions.checkNotNull(execRoot); + this.outputs = ImmutableSet.copyOf(outputs); + this.env = env; + this.tsgm = tsgm; + } + + @Override + public Metadata getMetadataMaybe(Artifact artifact) { + try { + return getMetadata(artifact); + } catch (IOException e) { + return null; + } + } + + private static Metadata metadataFromValue(FileArtifactValue value) throws FileNotFoundException { + if (value == FileArtifactValue.MISSING_FILE_MARKER) { + throw new FileNotFoundException(); + } + // If the file is empty or a directory, we need to return the mtime because the action cache + // uses mtime to determine if this artifact has changed. We do not optimize for this code + // path (by storing the mtime somewhere) because we eventually may be switching to use digests + // for empty files. We want this code path to go away somehow too for directories (maybe by + // implementing FileSet in Skyframe). + return value.getSize() > 0 + ? new Metadata(value.getDigest()) + : new Metadata(value.getModifiedTime()); + } + + @Override + public Metadata getMetadata(Artifact artifact) throws IOException { + Metadata metadata = getRealMetadata(artifact); + return artifact.isConstantMetadata() ? Metadata.CONSTANT_METADATA : metadata; + } + + @Nullable + private FileArtifactValue getInputFileArtifactValue(ActionInput input) { + FileArtifactValue value = inputArtifactData.get(input); + if (value != null) { + return value; + } + if (outputs.contains(input)) { + // When this method is called to calculate the metadata of an artifact, the artifact may be an + // output artifact. Don't try to do anything then. + return null; + } + if (!(input instanceof Artifact)) { + // Maybe we're being asked for some strange constructed ActionInput coming from runfiles or + // similar. We have no information about such things. + return null; + } + // TODO(bazel-team): Remove this codepath once Skyframe has native input discovery, so all + // inputs will already have metadata known. + // ActionExecutionFunction may have passed in null environment if this action does not + // discover inputs. In which case we should not have gotten here. + Preconditions.checkNotNull(env, input); + Artifact artifact = (Artifact) input; + if (artifact.isSourceArtifact()) { + // We might have no artifact data for discovered source inputs, and it's not worth storing + // it in this cache, because it won't be reused across actions -- while we could request an + // artifact from the graph, we would have to be tolerant to it not yet being present in the + // graph yet, which adds complexity. Instead, we let the undeclared inputs handler stat it, so + // it can be reused. + return null; + } else { + // This getValue call is not expected to return null, because if the artifact is a + // transitive dependency of this action (as it should be), it will already have been built, + // so this call will return a value. + // This getValue call is theoretically less efficient for a subsequent incremental build + // than it would be to do a bulk getValues call after action execution, as is done for + // source inputs. However, since almost all nodes requested here are transitive deps of an + // already-declared dependency, change pruning will request these values serially, but they + // will already have been built. So the only penalty is restarting ParallelEvaluator#run, as + // opposed to traversing the entire downward transitive closure on a single thread. + value = (FileArtifactValue) env.getValue( + FileArtifactValue.key(artifact, /*argument ignored for derived artifacts*/false)); + return value; + } + } + + /** + * We cache data for constant-metadata artifacts, even though it is technically unnecessary, + * because the data stored in this cache is consumed by various parts of Blaze via the {@link + * ActionExecutionValue} (for now, {@link FilesystemValueChecker} and {@link ArtifactFunction}). + * It is simpler for those parts if every output of the action is present in the cache. However, + * we must not return the actual metadata for a constant-metadata artifact. + */ + private Metadata getRealMetadata(Artifact artifact) throws IOException { + FileArtifactValue value = getInputFileArtifactValue(artifact); + if (value != null) { + return metadataFromValue(value); + } + if (artifact.isSourceArtifact()) { + // A discovered input we didn't have data for. + // TODO(bazel-team): Change this to an assertion once Skyframe has native input discovery, so + // all inputs will already have metadata known. + return null; + } else if (artifact.isMiddlemanArtifact()) { + // A middleman artifact's data was either already injected from the action cache checker using + // #setDigestForVirtualArtifact, or it has the default middleman value. + value = additionalOutputData.get(artifact); + if (value != null) { + return metadataFromValue(value); + } + value = FileArtifactValue.DEFAULT_MIDDLEMAN; + FileArtifactValue oldValue = additionalOutputData.putIfAbsent(artifact, value); + checkInconsistentData(artifact, oldValue, value); + return metadataFromValue(value); + } + FileValue fileValue = outputArtifactData.get(artifact); + if (fileValue != null) { + // Non-middleman artifacts should only have additionalOutputData if they have + // outputArtifactData. We don't assert this because of concurrency possibilities, but at least + // we don't check additionalOutputData unless we expect that we might see the artifact there. + value = additionalOutputData.get(artifact); + // If additional output data is present for this artifact, we use it in preference to the + // usual calculation. + if (value != null) { + return metadataFromValue(value); + } + if (!fileValue.exists()) { + throw new FileNotFoundException(artifact.prettyPrint() + " does not exist"); + } + return new Metadata(Preconditions.checkNotNull(fileValue.getDigest(), artifact)); + } + // We do not cache exceptions besides nonexistence here, because it is unlikely that the file + // will be requested from this cache too many times. + fileValue = fileValueFromArtifact(artifact, null, tsgm); + FileValue oldFileValue = outputArtifactData.putIfAbsent(artifact, fileValue); + checkInconsistentData(artifact, oldFileValue, value); + return maybeStoreAdditionalData(artifact, fileValue, null); + } + + /** Expands one of the input middlemen artifacts of the corresponding action. */ + public Collection<Artifact> expandInputMiddleman(Artifact middlemanArtifact) { + Preconditions.checkState(middlemanArtifact.isMiddlemanArtifact(), middlemanArtifact); + Collection<Artifact> result = expandedInputMiddlemen.get(middlemanArtifact); + // Note that result may be null for non-aggregating middlemen. + return result == null ? ImmutableSet.<Artifact>of() : result; + } + + /** + * Check that the new {@code data} we just calculated for an {@code artifact} agrees with the + * {@code oldData} (presumably calculated concurrently), if it was present. + */ + // Not private only because used by SkyframeActionExecutor's metadata handler. + static void checkInconsistentData(Artifact artifact, + @Nullable Object oldData, Object data) throws IOException { + if (oldData != null && !oldData.equals(data)) { + // Another thread checked this file since we looked at the map, and got a different answer + // than we did. Presumably the user modified the file between reads. + throw new IOException("Data for " + artifact.prettyPrint() + " changed to " + data + + " after it was calculated as " + oldData); + } + } + + /** + * See {@link #getAdditionalOutputData} for why we sometimes need to store additional data, even + * for normal (non-middleman) artifacts. + */ + @Nullable + private Metadata maybeStoreAdditionalData(Artifact artifact, FileValue data, + @Nullable byte[] injectedDigest) throws IOException { + if (!data.exists()) { + // Nonexistent files should only occur before executing an action. + throw new FileNotFoundException(artifact.prettyPrint() + " does not exist"); + } + boolean isFile = data.isFile(); + boolean useDigest = DigestUtils.useFileDigest(artifact, isFile, isFile ? data.getSize() : 0); + if (useDigest && data.getDigest() != null) { + // We do not need to store the FileArtifactValue separately -- the digest is in the file value + // and that is all that is needed for this file's metadata. + return new Metadata(data.getDigest()); + } + // Unfortunately, the FileValue does not contain enough information for us to calculate the + // corresponding FileArtifactValue -- either the metadata must use the modified time, which we + // do not expose in the FileValue, or the FileValue didn't store the digest So we store the + // metadata separately. + // Use the FileValue's digest if no digest was injected, or if the file can't be digested. + injectedDigest = injectedDigest != null || !isFile ? injectedDigest : data.getDigest(); + FileArtifactValue value = + FileArtifactValue.create(artifact, isFile, isFile ? data.getSize() : 0, injectedDigest); + FileArtifactValue oldValue = additionalOutputData.putIfAbsent(artifact, value); + checkInconsistentData(artifact, oldValue, value); + return metadataFromValue(value); + } + + @Override + public void setDigestForVirtualArtifact(Artifact artifact, Digest digest) { + Preconditions.checkState(artifact.isMiddlemanArtifact(), artifact); + Preconditions.checkNotNull(digest, artifact); + additionalOutputData.put(artifact, + FileArtifactValue.createMiddleman(digest.asMetadata().digest)); + } + + @Override + public void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest) { + if (output instanceof Artifact) { + Artifact artifact = (Artifact) output; + Preconditions.checkState(injectedArtifacts.add(artifact), artifact); + FileValue fileValue; + try { + // This call may do an unnecessary call to Path#getFastDigest to see if the digest is + // readily available. We cannot pass the digest in, though, because if it is not available + // from the filesystem, this FileValue will not compare equal to another one created for the + // same file, because the other one will be missing its digest. + fileValue = fileValueFromArtifact(artifact, FileStatusWithDigestAdapter.adapt(statNoFollow), + tsgm); + byte[] fileDigest = fileValue.getDigest(); + Preconditions.checkState(fileDigest == null || Arrays.equals(digest, fileDigest), + "%s %s %s", artifact, digest, fileDigest); + outputArtifactData.put(artifact, fileValue); + } catch (IOException e) { + // Do nothing - we just failed to inject metadata. Real error handling will be done later, + // when somebody will try to access that file. + return; + } + // If needed, insert additional data. Note that this can only be true if the file is empty or + // the filesystem does not support fast digests. Since we usually only inject digests when + // running with a filesystem that supports fast digests, this is fairly unlikely. + try { + maybeStoreAdditionalData(artifact, fileValue, digest); + } catch (IOException e) { + if (fileValue.getSize() != 0) { + // Empty files currently have their mtimes examined, and so could throw. No other files + // should throw, since all filesystem access has already been done. + throw new IllegalStateException( + "Filesystem should not have been accessed while injecting data for " + + artifact.prettyPrint(), e); + } + // Ignore exceptions for empty files, as above. + } + } + } + + @Override + public void discardMetadata(Collection<Artifact> artifactList) { + Preconditions.checkState(injectedArtifacts.isEmpty(), + "Artifacts cannot be injected before action execution: %s", injectedArtifacts); + outputArtifactData.keySet().removeAll(artifactList); + additionalOutputData.keySet().removeAll(artifactList); + } + + @Override + public boolean artifactExists(Artifact artifact) { + return getMetadataMaybe(artifact) != null; + } + + @Override + public boolean isRegularFile(Artifact artifact) { + // Currently this method is used only for genrule input directory checks. If we need to call + // this on output artifacts too, this could be more efficient. + FileArtifactValue value = getInputFileArtifactValue(artifact); + if (value != null && value.getDigest() != null) { + return true; + } + return artifact.getPath().isFile(); + } + + @Override + public boolean isInjected(Artifact artifact) { + return injectedArtifacts.contains(artifact); + } + + /** + * @return data for output files that was computed during execution. Should include data for all + * non-middleman artifacts. + */ + Map<Artifact, FileValue> getOutputData() { + return outputArtifactData; + } + + /** + * Returns data for any output files whose metadata was not computable from the corresponding + * entry in {@link #getOutputData}. + * + * <p>There are three reasons why we might not be able to compute metadata for an artifact from + * the FileValue. First, middleman artifacts have no corresponding FileValues. Second, if + * computing a file's digest is not fast, the FileValue does not do so, so a file on a filesystem + * without fast digests has to have its metadata stored separately. Third, some files' metadata + * (directories, empty files) contain their mtimes, which the FileValue does not expose, so that + * has to be stored separately. + * + * <p>Note that for files that need digests, we can't easily inject the digest in the FileValue + * because it would complicate equality-checking on subsequent builds -- if our filesystem doesn't + * do fast digests, the comparison value would not have a digest. + */ + Map<Artifact, FileArtifactValue> getAdditionalOutputData() { + return additionalOutputData; + } + + @Override + public long getSizeInBytes(ActionInput input) throws IOException { + FileArtifactValue metadata = getInputFileArtifactValue(input); + if (metadata != null) { + return metadata.getSize(); + } + return -1; + } + + @Nullable + @Override + public File getFileFromDigest(ByteString digest) throws IOException { + Artifact artifact = reverseMap.get(digest); + if (artifact != null) { + String relPath = artifact.getExecPathString(); + return relPath.startsWith("/") ? new File(relPath) : new File(execRoot, relPath); + } + return null; + } + + @Nullable + @Override + public ByteString getDigest(ActionInput input) throws IOException { + FileArtifactValue value = getInputFileArtifactValue(input); + if (value != null) { + byte[] bytes = value.getDigest(); + if (bytes != null) { + ByteString digest = ByteString.copyFrom(BaseEncoding.base16().lowerCase().encode(bytes) + .getBytes(StandardCharsets.US_ASCII)); + reverseMap.put(BYTE_INTERNER.intern(digest), (Artifact) input); + return digest; + } + } + return null; + } + + @Override + public boolean contentsAvailableLocally(ByteString digest) { + return reverseMap.containsKey(digest); + } + + static FileValue fileValueFromArtifact(Artifact artifact, + @Nullable FileStatusWithDigest statNoFollow, TimestampGranularityMonitor tsgm) + throws IOException { + Path path = artifact.getPath(); + RootedPath rootedPath = + RootedPath.toRootedPath(artifact.getRoot().getPath(), artifact.getRootRelativePath()); + if (statNoFollow == null) { + statNoFollow = FileStatusWithDigestAdapter.adapt(path.statIfFound(Symlinks.NOFOLLOW)); + if (statNoFollow == null) { + return FileValue.value(rootedPath, FileStateValue.NONEXISTENT_FILE_STATE_NODE, + rootedPath, FileStateValue.NONEXISTENT_FILE_STATE_NODE); + } + } + Path realPath = path; + // We use FileStatus#isSymbolicLink over Path#isSymbolicLink to avoid the unnecessary stat + // done by the latter. + if (statNoFollow.isSymbolicLink()) { + realPath = path.resolveSymbolicLinks(); + // We need to protect against symlink cycles since FileValue#value assumes it's dealing with a + // file that's not in a symlink cycle. + if (realPath.equals(path)) { + throw new IOException("symlink cycle"); + } + } + RootedPath realRootedPath = RootedPath.toRootedPathMaybeUnderRoot(realPath, + ImmutableList.of(artifact.getRoot().getPath())); + FileStateValue fileStateValue; + FileStateValue realFileStateValue; + try { + fileStateValue = FileStateValue.createWithStatNoFollow(rootedPath, statNoFollow, tsgm); + // TODO(bazel-team): consider avoiding a 'stat' here when the symlink target hasn't changed + // and is a source file (since changes to those are checked separately). + realFileStateValue = realPath.equals(path) ? fileStateValue + : FileStateValue.create(realRootedPath, tsgm); + } catch (InconsistentFilesystemException e) { + throw new IOException(e); + } + return FileValue.value(rootedPath, fileStateValue, realRootedPath, realFileStateValue); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java new file mode 100644 index 0000000..7acc38c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileArtifactValue.java
@@ -0,0 +1,148 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.cache.DigestUtils; +import com.google.devtools.build.lib.vfs.FileStatus; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; +import java.util.Arrays; + +import javax.annotation.Nullable; + +/** + * Stores the data of an artifact corresponding to a file. This file may be an ordinary file, in + * which case we would expect to see a digest and size; a directory, in which case we would expect + * to see an mtime; or an empty file, where we would expect to see a size (=0), mtime, and digest + */ +public class FileArtifactValue extends ArtifactValue { + /** Data for Middleman artifacts that did not have data specified. */ + static final FileArtifactValue DEFAULT_MIDDLEMAN = new FileArtifactValue(null, 0, 0); + /** Data that marks that a file is not present on the filesystem. */ + static final FileArtifactValue MISSING_FILE_MARKER = new FileArtifactValue(null, 1, 0); + + @Nullable private final byte[] digest; + private final long mtime; + private final long size; + + private FileArtifactValue(byte[] digest, long size) { + Preconditions.checkState(size >= 0, "size must be non-negative: %s %s", digest, size); + this.digest = Preconditions.checkNotNull(digest, size); + this.size = size; + this.mtime = -1; + } + + // Only used by empty files (non-null digest) and directories (null digest). + private FileArtifactValue(byte[] digest, long mtime, long size) { + Preconditions.checkState(mtime >= 0, "mtime must be non-negative: %s %s", mtime, size); + Preconditions.checkState(size == 0, "size must be zero: %s %s", mtime, size); + this.digest = digest; + this.size = size; + this.mtime = mtime; + } + + static FileArtifactValue create(Artifact artifact) throws IOException { + Path path = artifact.getPath(); + FileStatus stat = path.stat(); + boolean isFile = stat.isFile(); + return create(artifact, isFile, isFile ? stat.getSize() : 0, null); + } + + static FileArtifactValue create(Artifact artifact, FileValue fileValue) throws IOException { + boolean isFile = fileValue.isFile(); + return create(artifact, isFile, isFile ? fileValue.getSize() : 0, + isFile ? fileValue.getDigest() : null); + } + + static FileArtifactValue create(Artifact artifact, boolean isFile, long size, + @Nullable byte[] digest) throws IOException { + if (isFile && digest == null) { + digest = DigestUtils.getDigestOrFail(artifact.getPath(), size); + } + if (!DigestUtils.useFileDigest(artifact, isFile, size)) { + // In this case, we need to store the mtime because the action cache uses mtime to determine + // if this artifact has changed. This is currently true for empty files and directories. We + // do not optimize for this code path (by storing the mtime in a FileValue) because we do not + // like it and may remove this special-casing for empty files in the future. We want this code + // path to go away somehow too for directories (maybe by implementing FileSet + // in Skyframe) + return new FileArtifactValue(digest, artifact.getPath().getLastModifiedTime(), size); + } + Preconditions.checkState(digest != null, artifact); + return new FileArtifactValue(digest, size); + } + + static FileArtifactValue createMiddleman(byte[] digest) { + Preconditions.checkNotNull(digest); + // The Middleman artifact values have size 1 because we want their digests to be used. This hack + // can be removed once empty files are digested. + return new FileArtifactValue(digest, /*size=*/1); + } + + @Nullable + byte[] getDigest() { + return digest; + } + + /** Gets the size of the file. Directories have size 0. */ + long getSize() { + return size; + } + + /** + * Gets last modified time of file. Should only be called if {@link DigestUtils#useFileDigest} was + * false for this artifact -- namely, either it is a directory or an empty file. Note that since + * we store directory sizes as 0, all files for which this method can be called have size 0. + */ + long getModifiedTime() { + Preconditions.checkState(size == 0, "%s %s %s", digest, mtime, size); + return mtime; + } + + @Override + public int hashCode() { + // Hash digest by content, not reference. Note that digest is the only array in this array. + return Arrays.deepHashCode(new Object[] {size, mtime, digest}); + } + + /** + * Two FileArtifactValues will only compare equal if they have the same content. This differs + * from the {@code Metadata#equivalence} method, which allows for comparison using mtime if + * one object does not have a digest available. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof FileArtifactValue)) { + return false; + } + FileArtifactValue that = (FileArtifactValue) other; + return this.mtime == that.mtime && this.size == that.size + && Arrays.equals(this.digest, that.digest); + } + + @Override + public String toString() { + return Objects.toStringHelper(FileArtifactValue.class) + .add("digest", digest) + .add("mtime", mtime) + .add("size", size).toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileContentsProxy.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileContentsProxy.java new file mode 100644 index 0000000..344c364 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileContentsProxy.java
@@ -0,0 +1,66 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import java.io.Serializable; +import java.util.Objects; + +/** + * In case we can't get a fast digest from the filesystem, we store this metadata as a proxy to + * the file contents. Currently it is a pair of the mtime and "value id" (which is right now just + * the ivalue number). We may wish to augment this object with the following data: + * a. the device number + * b. the ctime, which cannot be tampered with in userspace + * + * <p>For an example of why mtime alone is insufficient, note that 'mv' preserves timestamps. So if + * files 'a' and 'b' initially have the same timestamp, then we would think 'b' is unchanged after + * the user executes `mv a b` between two builds. + */ +public final class FileContentsProxy implements Serializable { + private final long mtime; + private final long valueId; + + private FileContentsProxy(long mtime, long valueId) { + this.mtime = mtime; + this.valueId = valueId; + } + + public static FileContentsProxy create(long mtime, long valueId) { + return new FileContentsProxy(mtime, valueId); + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + + if (!(other instanceof FileContentsProxy)) { + return false; + } + + FileContentsProxy that = (FileContentsProxy) other; + return mtime == that.mtime && valueId == that.valueId; + } + + @Override + public int hashCode() { + return Objects.hash(mtime, valueId); + } + + @Override + public String toString() { + return "mtime: " + mtime + " valueId: " + valueId; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileFunction.java new file mode 100644 index 0000000..ad3cb74 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileFunction.java
@@ -0,0 +1,217 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.LinkedHashSet; +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.Nullable; + +/** + * A {@link SkyFunction} for {@link FileValue}s. + * + * <p>Most of the complexity in the implementation is associated to handling symlinks. Namely, + * this class makes sure that {@code FileValue}s corresponding to symlinks are correctly invalidated + * if the destination of the symlink is invalidated. Directory symlinks are also covered. + */ +public class FileFunction implements SkyFunction { + + private final AtomicReference<PathPackageLocator> pkgLocator; + private final ExternalFilesHelper externalFilesHelper; + + public FileFunction(AtomicReference<PathPackageLocator> pkgLocator, + ExternalFilesHelper externalFilesHelper) { + this.pkgLocator = pkgLocator; + this.externalFilesHelper = externalFilesHelper; + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws FileFunctionException { + RootedPath rootedPath = (RootedPath) skyKey.argument(); + RootedPath realRootedPath = rootedPath; + FileStateValue realFileStateValue = null; + PathFragment relativePath = rootedPath.getRelativePath(); + + // Resolve ancestor symlinks, but only if the current file is not the filesystem root (has no + // parent) or a package path root (treated opaquely and handled by skyframe's DiffAwareness + // interface) or otherwise assumed to be immutable (handling ancestors would add dependencies + // too aggressively). Note that this is the first thing we do - if an ancestor is part of a + // symlink cycle, we want to detect that quickly as it gives a more informative error message + // than we'd get doing bogus filesystem operations. + if (!relativePath.equals(PathFragment.EMPTY_FRAGMENT) + && !externalFilesHelper.shouldAssumeImmutable(rootedPath)) { + Pair<RootedPath, FileStateValue> resolvedState = + resolveFromAncestors(rootedPath, env); + if (resolvedState == null) { + return null; + } + realRootedPath = resolvedState.getFirst(); + realFileStateValue = resolvedState.getSecond(); + } + + FileStateValue fileStateValue = (FileStateValue) env.getValue(FileStateValue.key(rootedPath)); + if (fileStateValue == null) { + return null; + } + if (realFileStateValue == null) { + realFileStateValue = fileStateValue; + } + + LinkedHashSet<RootedPath> seenPaths = Sets.newLinkedHashSet(); + while (realFileStateValue.getType().equals(FileStateValue.Type.SYMLINK)) { + if (!seenPaths.add(realRootedPath)) { + FileSymlinkCycleException fileSymlinkCycleException = + makeFileSymlinkCycleException(realRootedPath, seenPaths); + if (env.getValue(FileSymlinkCycleUniquenessValue.key(fileSymlinkCycleException.getCycle())) + == null) { + // Note that this dependency is merely to ensure that each unique cycle gets reported + // exactly once. + return null; + } + throw new FileFunctionException(fileSymlinkCycleException); + } + Pair<RootedPath, FileStateValue> resolvedState = getSymlinkTargetRootedPath(realRootedPath, + realFileStateValue.getSymlinkTarget(), env); + if (resolvedState == null) { + return null; + } + realRootedPath = resolvedState.getFirst(); + realFileStateValue = resolvedState.getSecond(); + } + return FileValue.value(rootedPath, fileStateValue, realRootedPath, realFileStateValue); + } + + /** + * Returns the path and file state of {@code rootedPath}, accounting for ancestor symlinks, or + * {@code null} if there was a missing dep. + */ + @Nullable + private Pair<RootedPath, FileStateValue> resolveFromAncestors(RootedPath rootedPath, + Environment env) throws FileFunctionException { + PathFragment relativePath = rootedPath.getRelativePath(); + RootedPath realRootedPath = rootedPath; + FileValue parentFileValue = null; + if (!relativePath.equals(PathFragment.EMPTY_FRAGMENT)) { + RootedPath parentRootedPath = RootedPath.toRootedPath(rootedPath.getRoot(), + relativePath.getParentDirectory()); + parentFileValue = (FileValue) env.getValue(FileValue.key(parentRootedPath)); + if (parentFileValue == null) { + return null; + } + PathFragment baseName = new PathFragment(relativePath.getBaseName()); + RootedPath parentRealRootedPath = parentFileValue.realRootedPath(); + realRootedPath = RootedPath.toRootedPath(parentRealRootedPath.getRoot(), + parentRealRootedPath.getRelativePath().getRelative(baseName)); + } + FileStateValue realFileStateValue = + (FileStateValue) env.getValue(FileStateValue.key(realRootedPath)); + if (realFileStateValue == null) { + return null; + } + if (realFileStateValue.getType() != FileStateValue.Type.NONEXISTENT + && parentFileValue != null && !parentFileValue.isDirectory()) { + String type = realFileStateValue.getType().toString().toLowerCase(); + String message = type + " " + rootedPath.asPath() + " exists but its parent " + + "directory " + parentFileValue.realRootedPath().asPath() + " doesn't exist."; + throw new FileFunctionException(new InconsistentFilesystemException(message), + Transience.TRANSIENT); + } + return Pair.of(realRootedPath, realFileStateValue); + } + + /** + * Returns the symlink target and file state of {@code rootedPath}'s symlink to + * {@code symlinkTarget}, accounting for ancestor symlinks, or {@code null} if there was a + * missing dep. + */ + @Nullable + private Pair<RootedPath, FileStateValue> getSymlinkTargetRootedPath(RootedPath rootedPath, + PathFragment symlinkTarget, Environment env) throws FileFunctionException { + if (symlinkTarget.isAbsolute()) { + Path path = rootedPath.asPath().getFileSystem().getRootDirectory().getRelative( + symlinkTarget); + return resolveFromAncestors( + RootedPath.toRootedPathMaybeUnderRoot(path, pkgLocator.get().getPathEntries()), env); + } + Path path = rootedPath.asPath(); + Path symlinkTargetPath; + if (path.getParentDirectory() != null) { + RootedPath parentRootedPath = RootedPath.toRootedPathMaybeUnderRoot( + path.getParentDirectory(), pkgLocator.get().getPathEntries()); + FileValue parentFileValue = (FileValue) env.getValue(FileValue.key(parentRootedPath)); + if (parentFileValue == null) { + return null; + } + symlinkTargetPath = parentFileValue.realRootedPath().asPath().getRelative(symlinkTarget); + } else { + // This means '/' is a symlink to 'symlinkTarget'. + symlinkTargetPath = path.getRelative(symlinkTarget); + } + RootedPath symlinkTargetRootedPath = RootedPath.toRootedPathMaybeUnderRoot(symlinkTargetPath, + pkgLocator.get().getPathEntries()); + return resolveFromAncestors(symlinkTargetRootedPath, env); + } + + private FileSymlinkCycleException makeFileSymlinkCycleException(RootedPath startOfCycle, + Iterable<RootedPath> symlinkPaths) { + boolean inPathToCycle = true; + ImmutableList.Builder<RootedPath> pathToCycleBuilder = ImmutableList.builder(); + ImmutableList.Builder<RootedPath> cycleBuilder = ImmutableList.builder(); + for (RootedPath path : symlinkPaths) { + if (path.equals(startOfCycle)) { + inPathToCycle = false; + } + if (inPathToCycle) { + pathToCycleBuilder.add(path); + } else { + cycleBuilder.add(path); + } + } + return new FileSymlinkCycleException(pathToCycleBuilder.build(), cycleBuilder.build()); + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link FileFunction#compute}. + */ + private static final class FileFunctionException extends SkyFunctionException { + + public FileFunctionException(InconsistentFilesystemException e, Transience transience) { + super(e, transience); + } + + public FileFunctionException(FileSymlinkCycleException e) { + super(e, Transience.PERSISTENT); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java new file mode 100644 index 0000000..ec2e871 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateFunction.java
@@ -0,0 +1,76 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; + +/** + * A {@link SkyFunction} for {@link FileStateValue}s. + * + * <p>Merely calls FileStateValue#create, but also has special handling for files outside the + * package roots (see {@link ExternalFilesHelper}). + */ +public class FileStateFunction implements SkyFunction { + + private final TimestampGranularityMonitor tsgm; + private final ExternalFilesHelper externalFilesHelper; + + public FileStateFunction(TimestampGranularityMonitor tsgm, + ExternalFilesHelper externalFilesHelper) { + this.tsgm = tsgm; + this.externalFilesHelper = externalFilesHelper; + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws FileStateFunctionException { + RootedPath rootedPath = (RootedPath) skyKey.argument(); + externalFilesHelper.maybeAddDepOnBuildId(rootedPath, env); + if (env.valuesMissing()) { + return null; + } + try { + return FileStateValue.create(rootedPath, tsgm); + } catch (IOException e) { + throw new FileStateFunctionException(e); + } catch (InconsistentFilesystemException e) { + throw new FileStateFunctionException(e); + } + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link FileStateFunction#compute}. + */ + private static final class FileStateFunctionException extends SkyFunctionException { + public FileStateFunctionException(IOException e) { + super(e, Transience.TRANSIENT); + } + + public FileStateFunctionException(InconsistentFilesystemException e) { + super(e, Transience.TRANSIENT); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java new file mode 100644 index 0000000..8631ff3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java
@@ -0,0 +1,317 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.FileStatus; +import com.google.devtools.build.lib.vfs.FileStatusWithDigest; +import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * Encapsulates the filesystem operations needed to get state for a path. This is at least a + * 'lstat' to determine what type of file the path is. + * <ul> + * <li> For a non-existent file, the non existence is noted. + * <li> For a symlink, the symlink target is noted. + * <li> For a directory, the existence is noted. + * <li> For a file, the existence is noted, along with metadata about the file (e.g. + * file digest). See {@link FileFileStateValue}. + * <ul> + * + * <p>This class is an implementation detail of {@link FileValue} and should not be used outside of + * {@link FileFunction}. Instead, {@link FileValue} should be used by consumers that care about + * files. + * + * <p>All subclasses must implement {@link #equals} and {@link #hashCode} properly. + */ +abstract class FileStateValue implements SkyValue { + + public static final FileStateValue DIRECTORY_FILE_STATE_NODE = DirectoryFileStateValue.INSTANCE; + public static final FileStateValue NONEXISTENT_FILE_STATE_NODE = + NonexistentFileStateValue.INSTANCE; + + public enum Type { + FILE, + DIRECTORY, + SYMLINK, + NONEXISTENT, + } + + protected FileStateValue() { + } + + public static FileStateValue create(RootedPath rootedPath, + @Nullable TimestampGranularityMonitor tsgm) throws InconsistentFilesystemException, + IOException { + Path path = rootedPath.asPath(); + // Stat, but don't throw an exception for the common case of a nonexistent file. This still + // throws an IOException in case any other IO error is encountered. + FileStatus stat = path.statIfFound(Symlinks.NOFOLLOW); + if (stat == null) { + return NONEXISTENT_FILE_STATE_NODE; + } + return createWithStatNoFollow(rootedPath, FileStatusWithDigestAdapter.adapt(stat), tsgm); + } + + public static FileStateValue createWithStatNoFollow(RootedPath rootedPath, + FileStatusWithDigest statNoFollow, @Nullable TimestampGranularityMonitor tsgm) + throws InconsistentFilesystemException, IOException { + Path path = rootedPath.asPath(); + if (statNoFollow.isFile()) { + return FileFileStateValue.fromPath(path, statNoFollow, tsgm); + } else if (statNoFollow.isDirectory()) { + return DIRECTORY_FILE_STATE_NODE; + } else if (statNoFollow.isSymbolicLink()) { + return new SymlinkFileStateValue(path.readSymbolicLink()); + } + throw new InconsistentFilesystemException("according to stat, existing path " + path + " is " + + "neither a file nor directory nor symlink."); + } + + @ThreadSafe + static SkyKey key(RootedPath rootedPath) { + return new SkyKey(SkyFunctions.FILE_STATE, rootedPath); + } + + abstract Type getType(); + + PathFragment getSymlinkTarget() { + throw new IllegalStateException(); + } + + long getSize() { + throw new IllegalStateException(); + } + + @Nullable + byte[] getDigest() { + throw new IllegalStateException(); + } + + /** + * Implementation of {@link FileStateValue} for files that exist. + * + * <p>A union of (digest, mtime). We use digests only if a fast digest lookup is available from + * the filesystem. If not, we fall back to mtime-based digests. This avoids the case where Blaze + * must read all files involved in the build in order to check for modifications in the case + * where fast digest lookups are not available. + */ + @ThreadSafe + private static final class FileFileStateValue extends FileStateValue { + private final long size; + // Only needed for empty-file equality-checking. Otherwise is always -1. + // TODO(bazel-team): Consider getting rid of this special case for empty files. + private final long mtime; + @Nullable private final byte[] digest; + @Nullable private final FileContentsProxy contentsProxy; + + private FileFileStateValue(long size, long mtime, byte[] digest, + FileContentsProxy contentsProxy) { + Preconditions.checkState((digest == null) != (contentsProxy == null)); + this.size = size; + // mtime is forced to be -1 so that we do not accidentally depend on it for non-empty files, + // which should only be compared using digests. + this.mtime = size == 0 ? mtime : -1; + this.digest = digest; + this.contentsProxy = contentsProxy; + } + + /** + * Create a FileFileStateValue instance corresponding to the given existing file. + * @param stat must be of type "File". (Not a symlink). + */ + public static FileFileStateValue fromPath(Path path, FileStatusWithDigest stat, + @Nullable TimestampGranularityMonitor tsgm) + throws InconsistentFilesystemException { + Preconditions.checkState(stat.isFile(), path); + try { + byte[] digest = stat.getDigest(); + if (digest == null) { + digest = path.getFastDigest(); + } + if (digest == null) { + long mtime = stat.getLastModifiedTime(); + // Note that TimestampGranularityMonitor#notifyDependenceOnFileTime is a thread-safe + // method. + if (tsgm != null) { + tsgm.notifyDependenceOnFileTime(mtime); + } + return new FileFileStateValue(stat.getSize(), stat.getLastModifiedTime(), null, + FileContentsProxy.create(mtime, stat.getNodeId())); + } else { + // We are careful here to avoid putting the value ID into FileMetadata if we already have + // a digest. Arbitrary filesystems may do weird things with the value ID; a digest is more + // robust. + return new FileFileStateValue(stat.getSize(), stat.getLastModifiedTime(), digest, null); + } + } catch (IOException e) { + String errorMessage = e.getMessage() != null + ? "error '" + e.getMessage() + "'" : "an error"; + throw new InconsistentFilesystemException("'stat' said " + path + " is a file but then we " + + "later encountered " + errorMessage + " which indicates that " + path + " no longer " + + "exists. Did you delete it during the build?"); + } + } + + @Override + public Type getType() { + return Type.FILE; + } + + @Override + public long getSize() { + return size; + } + + @Override + @Nullable + public byte[] getDigest() { + return digest; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof FileFileStateValue) { + FileFileStateValue other = (FileFileStateValue) obj; + return size == other.size && mtime == other.mtime && Arrays.equals(digest, other.digest) + && Objects.equals(contentsProxy, other.contentsProxy); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(size, mtime, Arrays.hashCode(digest), contentsProxy); + } + + @Override + public String toString() { + return "[size: " + size + " " + (mtime != -1 ? "mtime: " + mtime : "") + + (digest != null ? "digest: " + Arrays.toString(digest) : contentsProxy) + "]"; + } + } + + /** Implementation of {@link FileStateValue} for directories that exist. */ + private static final class DirectoryFileStateValue extends FileStateValue { + + public static final DirectoryFileStateValue INSTANCE = new DirectoryFileStateValue(); + + private DirectoryFileStateValue() { + } + + @Override + public Type getType() { + return Type.DIRECTORY; + } + + @Override + public String toString() { + return "directory"; + } + + // This object is normally a singleton, but deserialization produces copies. + @Override + public boolean equals(Object obj) { + return obj instanceof DirectoryFileStateValue; + } + + @Override + public int hashCode() { + return 7654321; + } + } + + /** Implementation of {@link FileStateValue} for symlinks. */ + private static final class SymlinkFileStateValue extends FileStateValue { + + private final PathFragment symlinkTarget; + + private SymlinkFileStateValue(PathFragment symlinkTarget) { + this.symlinkTarget = symlinkTarget; + } + + @Override + public Type getType() { + return Type.SYMLINK; + } + + @Override + public PathFragment getSymlinkTarget() { + return symlinkTarget; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof SymlinkFileStateValue)) { + return false; + } + SymlinkFileStateValue other = (SymlinkFileStateValue) obj; + return symlinkTarget.equals(other.symlinkTarget); + } + + @Override + public int hashCode() { + return symlinkTarget.hashCode(); + } + + @Override + public String toString() { + return "symlink to " + symlinkTarget; + } + } + + /** Implementation of {@link FileStateValue} for nonexistent files. */ + private static final class NonexistentFileStateValue extends FileStateValue { + + public static final NonexistentFileStateValue INSTANCE = new NonexistentFileStateValue(); + + private NonexistentFileStateValue() { + } + + @Override + public Type getType() { + return Type.NONEXISTENT; + } + + @Override + public String toString() { + return "nonexistent"; + } + + // This object is normally a singleton, but deserialization produces copies. + @Override + public boolean equals(Object obj) { + return obj instanceof NonexistentFileStateValue; + } + + @Override + public int hashCode() { + return 8765432; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleException.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleException.java new file mode 100644 index 0000000..d57fc42 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleException.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.vfs.RootedPath; + +/** Exception indicating that a cycle was found in the filesystem. */ +public class FileSymlinkCycleException extends Exception { + + private final ImmutableList<RootedPath> pathToCycle; + private final ImmutableList<RootedPath> cycle; + + FileSymlinkCycleException(ImmutableList<RootedPath> pathToCycle, + ImmutableList<RootedPath> cycle) { + // The cycle itself has already been reported by FileSymlinkCycleUniquenessValue, but we still + // want to have a readable #getMessage. + super("Symlink cycle"); + this.pathToCycle = pathToCycle; + this.cycle = cycle; + } + + /** + * The symlink path to the symlink cycle. For example, suppose 'a' -> 'b' -> 'c' -> 'd' -> 'c'. + * The path to the cycle is 'a', 'b'. + */ + ImmutableList<RootedPath> getPathToCycle() { + return pathToCycle; + } + + /** + * The symlink cycle. For example, suppose 'a' -> 'b' -> 'c' -> 'd' -> 'c'. + * The cycle is 'c', 'd'. + */ + ImmutableList<RootedPath> getCycle() { + return cycle; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessFunction.java new file mode 100644 index 0000000..a0604b5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessFunction.java
@@ -0,0 +1,45 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** A value builder that has the side effect of reporting a file symlink cycle. */ +public class FileSymlinkCycleUniquenessFunction implements SkyFunction { + + @SuppressWarnings("unchecked") // Cast from Object to ImmutableList<RootedPath>. + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + StringBuilder cycleMessage = new StringBuilder("circular symlinks detected\n"); + cycleMessage.append("[start of symlink cycle]\n"); + for (RootedPath rootedPath : (ImmutableList<RootedPath>) skyKey.argument()) { + cycleMessage.append(rootedPath.asPath() + "\n"); + } + cycleMessage.append("[end of symlink cycle]"); + // The purpose of this value builder is the side effect of emitting an error message exactly + // once per build per unique cycle. + env.getListener().handle(Event.error(cycleMessage.toString())); + return FileSymlinkCycleUniquenessValue.INSTANCE; + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessValue.java new file mode 100644 index 0000000..627276d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessValue.java
@@ -0,0 +1,57 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * A value for ensuring that a file symlink cycle is reported exactly once. This is achieved by + * forcing the same value key for two logically equivalent cycles (e.g. ['a' -> 'b' -> 'c' -> 'a'] + * and ['b' -> 'c' -> 'a' -> 'b'], and letting Skyframe do its magic. + */ +class FileSymlinkCycleUniquenessValue implements SkyValue { + + public static final FileSymlinkCycleUniquenessValue INSTANCE = + new FileSymlinkCycleUniquenessValue(); + + private FileSymlinkCycleUniquenessValue() { + } + + static SkyKey key(ImmutableList<RootedPath> cycle) { + Preconditions.checkState(!cycle.isEmpty()); + return new SkyKey(SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS, canonicalize(cycle)); + } + + private static ImmutableList<RootedPath> canonicalize(ImmutableList<RootedPath> cycle) { + int minPos = 0; + String minString = cycle.get(0).toString(); + for (int i = 1; i < cycle.size(); i++) { + String candidateString = cycle.get(i).toString(); + if (candidateString.compareTo(minString) < 0) { + minPos = i; + minString = candidateString; + } + } + ImmutableList.Builder<RootedPath> builder = ImmutableList.builder(); + for (int i = 0; i < cycle.size(); i++) { + int pos = (minPos + i) % cycle.size(); + builder.add(cycle.get(pos)); + } + return builder.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileValue.java new file mode 100644 index 0000000..1850fd9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileValue.java
@@ -0,0 +1,279 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.skyframe.FileStateValue.Type; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * A value that corresponds to a file (or directory or symlink or non-existent file), fully + * accounting for symlinks (e.g. proper dependencies on ancestor symlinks so as to be incrementally + * correct). Anything in Skyframe that cares about the fully resolved path of a file (e.g. anything + * that cares about the contents of a file) should have a dependency on the corresponding + * {@link FileValue}. + * + * <p> + * Note that the existence of a file value does not imply that the file exists on the filesystem. + * File values for missing files will be created on purpose in order to facilitate incremental + * builds in the case those files have reappeared. + * + * <p> + * This class contains the relevant metadata for a file, although not the contents. Note that + * since a FileValue doesn't store its corresponding SkyKey, it's possible for the FileValues for + * two different paths to be the same. + * + * <p> + * This should not be used for build outputs; use {@link ArtifactValue} for those. + */ +@Immutable +@ThreadSafe +public abstract class FileValue implements SkyValue { + + boolean exists() { + return realFileStateValue().getType() != Type.NONEXISTENT; + } + + public boolean isSymlink() { + return false; + } + + /** + * Returns true if this value corresponds to a file or symlink to an existing file. If so, its + * parent directory is guaranteed to exist. + */ + public boolean isFile() { + return realFileStateValue().getType() == Type.FILE; + } + + /** + * Returns true if the file is a directory or a symlink to an existing directory. If so, its + * parent directory is guaranteed to exist. + */ + public boolean isDirectory() { + return realFileStateValue().getType() == Type.DIRECTORY; + } + + /** + * Returns the real rooted path of the file, taking ancestor symlinks into account. For example, + * the rooted path ['root']/['a/b'] is really ['root']/['c/b'] if 'a' is a symlink to 'b'. Note + * that ancestor symlinks outside the root boundary are not taken into consideration. + */ + public abstract RootedPath realRootedPath(); + + abstract FileStateValue realFileStateValue(); + + /** + * Returns the unresolved link target if {@link #isSymlink()}. + * + * <p>This is useful if the caller wants to, for example, duplicate a relative symlink. An actual + * example could be a build rule that copies a set of input files to the output directory, but + * upon encountering symbolic links it can decide between copying or following them. + */ + PathFragment getUnresolvedLinkTarget() { + throw new IllegalStateException(this.toString()); + } + + long getSize() { + Preconditions.checkState(isFile(), this); + return realFileStateValue().getSize(); + } + + @Nullable + byte[] getDigest() { + Preconditions.checkState(isFile(), this); + return realFileStateValue().getDigest(); + } + + /** + * Returns a key for building a file value for the given root-relative path. + */ + @ThreadSafe + public static SkyKey key(RootedPath rootedPath) { + return new SkyKey(SkyFunctions.FILE, rootedPath); + } + + /** + * Only intended to be used by {@link FileFunction}. Should not be used for symlink cycles. + */ + static FileValue value(RootedPath rootedPath, FileStateValue fileStateValue, + RootedPath realRootedPath, FileStateValue realFileStateValue) { + if (rootedPath.equals(realRootedPath)) { + Preconditions.checkState(fileStateValue.getType() != FileStateValue.Type.SYMLINK, + "rootedPath: %s, fileStateValue: %s, realRootedPath: %s, realFileStateValue: %s", + rootedPath, fileStateValue, realRootedPath, realFileStateValue); + return new RegularFileValue(rootedPath, fileStateValue); + } else { + if (fileStateValue.getType() == FileStateValue.Type.SYMLINK) { + return new SymlinkFileValue(realRootedPath, realFileStateValue, + fileStateValue.getSymlinkTarget()); + } else { + return new DifferentRealPathFileValue(realRootedPath, realFileStateValue); + } + } + } + + /** + * Implementation of {@link FileValue} for files whose fully resolved path is the same as the + * requested path. For example, this is the case for the path "foo/bar/baz" if neither 'foo' nor + * 'foo/bar' nor 'foo/bar/baz' are symlinks. + */ + private static final class RegularFileValue extends FileValue { + + private final RootedPath rootedPath; + private final FileStateValue fileStateValue; + + private RegularFileValue(RootedPath rootedPath, FileStateValue fileState) { + this.rootedPath = Preconditions.checkNotNull(rootedPath); + this.fileStateValue = Preconditions.checkNotNull(fileState); + } + + @Override + public RootedPath realRootedPath() { + return rootedPath; + } + + @Override + FileStateValue realFileStateValue() { + return fileStateValue; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != RegularFileValue.class) { + return false; + } + RegularFileValue other = (RegularFileValue) obj; + return rootedPath.equals(other.rootedPath) && fileStateValue.equals(other.fileStateValue); + } + + @Override + public int hashCode() { + return Objects.hash(rootedPath, fileStateValue); + } + + @Override + public String toString() { + return rootedPath + ", " + fileStateValue; + } + } + + /** + * Base class for {@link FileValue}s for files whose fully resolved path is different than the + * requested path. For example, this is the case for the path "foo/bar/baz" if at least one of + * 'foo', 'foo/bar', or 'foo/bar/baz' is a symlink. + */ + private static class DifferentRealPathFileValue extends FileValue { + + protected final RootedPath realRootedPath; + protected final FileStateValue realFileStateValue; + + private DifferentRealPathFileValue(RootedPath realRootedPath, + FileStateValue realFileStateValue) { + this.realRootedPath = Preconditions.checkNotNull(realRootedPath); + this.realFileStateValue = Preconditions.checkNotNull(realFileStateValue); + } + + @Override + public RootedPath realRootedPath() { + return realRootedPath; + } + + @Override + FileStateValue realFileStateValue() { + return realFileStateValue; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != DifferentRealPathFileValue.class) { + return false; + } + DifferentRealPathFileValue other = (DifferentRealPathFileValue) obj; + return realRootedPath.equals(other.realRootedPath) + && realFileStateValue.equals(other.realFileStateValue); + } + + @Override + public int hashCode() { + return Objects.hash(realRootedPath, realFileStateValue); + } + + @Override + public String toString() { + return realRootedPath + ", " + realFileStateValue + " (symlink ancestor)"; + } + } + + /** Implementation of {@link FileValue} for files that are symlinks. */ + private static final class SymlinkFileValue extends DifferentRealPathFileValue { + private final PathFragment linkValue; + + private SymlinkFileValue(RootedPath realRootedPath, FileStateValue realFileState, + PathFragment linkTarget) { + super(realRootedPath, realFileState); + this.linkValue = linkTarget; + } + + @Override + public boolean isSymlink() { + return true; + } + + @Override + public PathFragment getUnresolvedLinkTarget() { + return linkValue; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != SymlinkFileValue.class) { + return false; + } + SymlinkFileValue other = (SymlinkFileValue) obj; + return realRootedPath.equals(other.realRootedPath) + && realFileStateValue.equals(other.realFileStateValue) + && linkValue.equals(other.linkValue); + } + + @Override + public int hashCode() { + return Objects.hash(realRootedPath, realFileStateValue, linkValue, Boolean.TRUE); + } + + @Override + public String toString() { + return String.format("symlink (real_path=%s, real_state=%s, link_value=%s)", + realRootedPath, realFileStateValue, linkValue); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunction.java new file mode 100644 index 0000000..a559206 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunction.java
@@ -0,0 +1,320 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.FilesetOutputSymlink; +import com.google.devtools.build.lib.actions.FilesetTraversalParams; +import com.google.devtools.build.lib.actions.FilesetTraversalParams.DirectTraversal; +import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalFunction.DanglingSymlinkException; +import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalFunction.RecursiveFilesystemTraversalException; +import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; + +/** SkyFunction for {@link FilesetEntryValue}. */ +public final class FilesetEntryFunction implements SkyFunction { + + private static final class MissingDepException extends Exception {} + + private static final class FilesetEntryFunctionException extends SkyFunctionException { + FilesetEntryFunctionException(RecursiveFilesystemTraversalException e) { + super(e, Transience.PERSISTENT); + } + } + + @Override + public SkyValue compute(SkyKey key, Environment env) throws FilesetEntryFunctionException { + FilesetTraversalParams t = (FilesetTraversalParams) key.argument(); + Preconditions.checkState( + t.getNestedTraversal().isPresent() != t.getDirectTraversal().isPresent(), + "Exactly one of the nested and direct traversals must be specified: %s", t); + + // Create the set of excluded files. Only top-level files can be excluded, i.e. ones that are + // directly under the root if the root is a directory. + Set<String> exclusions = createExclusionSet(t.getExcludedFiles()); + + // The map of output symlinks. Each key is the path of a output symlink that the Fileset must + // create, relative to the Fileset.out directory, and each value specifies extra information + // about the link (its target, associated metadata and again its name). + Map<PathFragment, FilesetOutputSymlink> outputSymlinks = new LinkedHashMap<>(); + + if (t.getNestedTraversal().isPresent()) { + // The "nested" traversal parameters are present if and only if FilesetEntry.srcdir specifies + // another Fileset (a "nested" one). + FilesetEntryValue nested = (FilesetEntryValue) env.getValue( + FilesetEntryValue.key(t.getNestedTraversal().get())); + if (env.valuesMissing()) { + return null; + } + + for (FilesetOutputSymlink s : nested.getSymlinks()) { + maybeStoreSymlink(s, t.getDestPath(), exclusions, outputSymlinks); + } + } else { + // The "nested" traversal params are absent if and only if the "direct" traversal params are + // present, which is the case when the FilesetEntry specifies a package's BUILD file, a + // directory or a list of files. + + // The root of the direct traversal is defined as follows. + // + // If FilesetEntry.files is specified, then a TraversalRequest is created for each entry, the + // root being the respective entry itself. These are all traversed for they may be + // directories or symlinks to directories, and we need to establish Skyframe dependencies on + // their contents for incremental correctness. If an entry is indeed a directory (but not when + // it's a symlink to one) then we have to create symlinks to each of their childen. + // (NB: there seems to be no good reason for this, it's just how legacy Fileset works. We may + // want to consider creating a symlink just for the directory and not for its child elements.) + // + // If FilesetEntry.files is not specified, then srcdir refers to either a BUILD file or a + // directory. For the former, the root will be the parent of the BUILD file. For the latter, + // the root will be srcdir itself. + DirectTraversal direct = t.getDirectTraversal().get(); + + RecursiveFilesystemTraversalValue rftv; + try { + // Traverse the filesystem to establish skyframe dependencies. + rftv = traverse(env, createErrorInfo(t), direct); + } catch (MissingDepException e) { + return null; + } + + // The root can only be absent for the EMPTY rftv instance. + if (!rftv.getResolvedRoot().isPresent()) { + return FilesetEntryValue.EMPTY; + } + + ResolvedFile resolvedRoot = rftv.getResolvedRoot().get(); + + // Handle dangling symlinks gracefully be returning empty results. + if (!resolvedRoot.type.exists()) { + return FilesetEntryValue.EMPTY; + } + + // The prefix to remove is the entire path of the root. This is OK: + // - when the root is a file, this removes the entire path, but the traversal's destination + // path is actually the name of the output symlink, so this works out correctly + // - when the root is a directory or a symlink to one then we need to strip off the + // directory's path from every result (this is how the output symlinks must be created) + // before making them relative to the destination path + PathFragment prefixToRemove = direct.getRoot().getRelativePart(); + + Iterable<ResolvedFile> results = null; + + if (direct.isRecursive() + || (resolvedRoot.type.isDirectory() && !resolvedRoot.type.isSymlink())) { + // The traversal is recursive (requested for an entire FilesetEntry.srcdir) or it was + // requested for a FilesetEntry.files entry which turned out to be a directory. We need to + // create an output symlink for every file in it and all of its subdirectories. Only + // exception is when the subdirectory is really a symlink to a directory -- no output + // shall be created for the contents of those. + // Now we create Dir objects to model the filesystem tree. The object employs a trick to + // find directory symlinks: directory symlinks have corresponding ResolvedFile entries and + // are added as files too, while their children, also added as files, contain the path of + // the parent. Finding and discarding the children is easy if we traverse the tree from + // root to leaf. + DirectoryTree root = new DirectoryTree(); + for (ResolvedFile f : rftv.getTransitiveFiles().toCollection()) { + PathFragment path = f.getNameInSymlinkTree().relativeTo(prefixToRemove); + if (path.segmentCount() > 0) { + path = t.getDestPath().getRelative(path); + DirectoryTree dir = root; + for (int i = 0; i < path.segmentCount() - 1; ++i) { + dir = dir.addOrGetSubdir(path.getSegment(i)); + } + dir.maybeAddFile(f); + } + } + // Here's where the magic happens. The returned iterable will yield all files in the + // directory that are not under symlinked directories, as well as all directory symlinks. + results = root.iterateFiles(); + } else { + // If we're on this branch then the traversal was done for just one entry in + // FilesetEntry.files (which was not a directory, so it was either a file, a symlink to one + // or a symlink to a directory), meaning we'll have only one output symlink. + results = ImmutableList.of(resolvedRoot); + } + + // Create one output symlink for each entry in the results. + for (ResolvedFile f : results) { + PathFragment targetName; + try { + targetName = f.getTargetInSymlinkTree(direct.isFollowingSymlinks()); + } catch (DanglingSymlinkException e) { + throw new FilesetEntryFunctionException(e); + } + + // Metadata field must be present. It can only be absent when stripped by tests. + String metadata = Integer.toHexString(f.metadata.get().hashCode()); + + // The linkName has to be under the traversal's root, which is also the prefix to remove. + PathFragment linkName = f.getNameInSymlinkTree().relativeTo(prefixToRemove); + maybeStoreSymlink(linkName, targetName, metadata, t.getDestPath(), exclusions, + outputSymlinks); + } + } + + return FilesetEntryValue.of(ImmutableSet.copyOf(outputSymlinks.values())); + } + + /** Stores an output symlink unless it's excluded or would overwrite an existing one. */ + private static void maybeStoreSymlink(FilesetOutputSymlink nestedLink, PathFragment destPath, + Set<String> exclusions, Map<PathFragment, FilesetOutputSymlink> result) { + maybeStoreSymlink(nestedLink.name, nestedLink.target, nestedLink.metadata, destPath, + exclusions, result); + } + + /** Stores an output symlink unless it's excluded or would overwrite an existing one. */ + private static void maybeStoreSymlink(PathFragment linkName, PathFragment linkTarget, + String metadata, PathFragment destPath, Set<String> exclusions, + Map<PathFragment, FilesetOutputSymlink> result) { + if (!exclusions.contains(linkName.getPathString())) { + linkName = destPath.getRelative(linkName); + if (!result.containsKey(linkName)) { + result.put(linkName, new FilesetOutputSymlink(linkName, linkTarget, metadata)); + } + } + } + + private static Set<String> createExclusionSet(Set<String> input) { + return Sets.filter(input, new Predicate<String>() { + @Override + public boolean apply(String e) { + // Keep the top-level exclusions only. Do not look for "/" but count the path segments + // instead, in anticipation of future Windows support. + return new PathFragment(e).segmentCount() == 1; + } + }); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + private RecursiveFilesystemTraversalValue traverse(Environment env, String errorInfo, + DirectTraversal traversal) throws MissingDepException { + SkyKey depKey = RecursiveFilesystemTraversalValue.key( + new RecursiveFilesystemTraversalValue.TraversalRequest(traversal.getRoot().asRootedPath(), + traversal.isGenerated(), traversal.getCrossPackageBoundary(), traversal.isPackage(), + errorInfo)); + RecursiveFilesystemTraversalValue v = (RecursiveFilesystemTraversalValue) env.getValue(depKey); + if (env.valuesMissing()) { + throw new MissingDepException(); + } + return v; + } + + private static String createErrorInfo(FilesetTraversalParams t) { + if (t.getDirectTraversal().isPresent()) { + DirectTraversal direct = t.getDirectTraversal().get(); + return String.format("Fileset '%s' traversing %s %s", t.getOwnerLabel(), + direct.isPackage() ? "package" : "file (or directory)", + direct.getRoot().getRelativePart().getPathString()); + } else { + return String.format("Fileset '%s' traversing another Fileset", t.getOwnerLabel()); + } + } + + /** + * Models a FilesetEntryFunction's portion of the symlink output tree created by a Fileset rule. + * + * <p>A Fileset rule's output is computed by zero or more {@link FilesetEntryFunction}s, resulting + * in one {@link FilesetEntryValue} for each. Each of those represents a portion of the grand + * output tree of the Fileset. These portions are later merged and written to the fileset manifest + * file, which is then consumed by a tool that ultimately creates the symlinks in the filesystem. + * + * <p>Because the Fileset doesn't process the lists in the FilesetEntryValues any further than + * merging them, they have to adhere to the conventions of the manifest file. One of these is that + * files are alphabetically ordered (enables the consumer of the manifest to work faster than + * otherwise) and another is that the contents of regular directories are listed, but contents + * of directory symlinks are not, only the symlinks are. (Other details of the manifest file are + * not relevant here.) + * + * <p>See {@link DirectoryTree#iterateFiles()} for more details. + */ + private static final class DirectoryTree { + // Use TreeMaps for the benefit of alphabetically ordered iteration. + public final Map<String, ResolvedFile> files = new TreeMap<>(); + public final Map<String, DirectoryTree> dirs = new TreeMap<>(); + + DirectoryTree addOrGetSubdir(String name) { + DirectoryTree result = dirs.get(name); + if (result == null) { + result = new DirectoryTree(); + dirs.put(name, result); + } + return result; + } + + void maybeAddFile(ResolvedFile r) { + String name = r.getNameInSymlinkTree().getBaseName(); + if (!files.containsKey(name)) { + files.put(name, r); + } + } + + /** + * Lazily yields all files in this directory and all of its subdirectories. + * + * <p>The function first yields all the files directly under this directory, in alphabetical + * order. Then come the contents of subdirectories, processed recursively in the same fashion + * as this directory, and also in alphabetical order. + * + * <p>If a directory symlink is encountered its contents are not listed, only the symlink is. + */ + Iterable<ResolvedFile> iterateFiles() { + // 1. Filter directory symlinks. If the symlink target contains files, those were added + // as normal files so their parent directory (the symlink) would show up in the dirs map + // (as a directory) as well as in the files map (as a symlink to a directory). + final Set<String> fileNames = files.keySet(); + Iterable<Map.Entry<String, DirectoryTree>> noDirSymlinkes = Iterables.filter(dirs.entrySet(), + new Predicate<Map.Entry<String, DirectoryTree>>() { + @Override + public boolean apply(Map.Entry<String, DirectoryTree> input) { + return !fileNames.contains(input.getKey()); + } + }); + + // 2. Extract the iterables of the true subdirectories. + Iterable<Iterable<ResolvedFile>> subdirIters = Iterables.transform(noDirSymlinkes, + new Function<Map.Entry<String, DirectoryTree>, Iterable<ResolvedFile>>() { + @Override + public Iterable<ResolvedFile> apply(Entry<String, DirectoryTree> input) { + return input.getValue().iterateFiles(); + } + }); + + // 3. Just concat all subdirectory iterations for one, seamless iteration. + Iterable<ResolvedFile> dirsIter = Iterables.concat(subdirIters); + + return Iterables.concat(files.values(), dirsIter); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryValue.java new file mode 100644 index 0000000..e7b6580 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesetEntryValue.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.FilesetOutputSymlink; +import com.google.devtools.build.lib.actions.FilesetTraversalParams; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** Output symlinks produced by a whole FilesetEntry or by a single file in FilesetEntry.files. */ +public final class FilesetEntryValue implements SkyValue { + static final FilesetEntryValue EMPTY = + new FilesetEntryValue(ImmutableSet.<FilesetOutputSymlink>of()); + + private final ImmutableSet<FilesetOutputSymlink> symlinks; + + private FilesetEntryValue(ImmutableSet<FilesetOutputSymlink> symlinks) { + this.symlinks = symlinks; + } + + static FilesetEntryValue of(ImmutableSet<FilesetOutputSymlink> symlinks) { + if (symlinks.isEmpty()) { + return EMPTY; + } else { + return new FilesetEntryValue(symlinks); + } + } + + /** Returns the list of output symlinks. */ + public ImmutableSet<FilesetOutputSymlink> getSymlinks() { + return symlinks; + } + + public static SkyKey key(FilesetTraversalParams params) { + return new SkyKey(SkyFunctions.FILESET_ENTRY, params); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java new file mode 100644 index 0000000..be4f4e8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java
@@ -0,0 +1,398 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ExecutorShutdownUtil; +import com.google.devtools.build.lib.concurrent.Sharder; +import com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.BatchStat; +import com.google.devtools.build.lib.vfs.FileStatusWithDigest; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.Differencer; +import com.google.devtools.build.skyframe.MemoizingEvaluator; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +/** + * A helper class to find dirty values by accessing the filesystem directly (contrast with + * {@link DiffAwareness}). + */ +class FilesystemValueChecker { + + private static final int DIRTINESS_CHECK_THREADS = 50; + private static final Logger LOG = Logger.getLogger(FilesystemValueChecker.class.getName()); + + private static final Predicate<SkyKey> FILE_STATE_AND_DIRECTORY_LISTING_STATE_FILTER = + SkyFunctionName.functionIsIn(ImmutableSet.of(SkyFunctions.FILE_STATE, + SkyFunctions.DIRECTORY_LISTING_STATE)); + private static final Predicate<SkyKey> ACTION_FILTER = + SkyFunctionName.functionIs(SkyFunctions.ACTION_EXECUTION); + + private final TimestampGranularityMonitor tsgm; + private final Supplier<Map<SkyKey, SkyValue>> valuesSupplier; + private AtomicInteger modifiedOutputFilesCounter = new AtomicInteger(0); + + FilesystemValueChecker(final MemoizingEvaluator evaluator, TimestampGranularityMonitor tsgm) { + this.tsgm = tsgm; + + // Construct the full map view of the entire graph at most once ("memoized"), lazily. If + // getDirtyFilesystemValues(Iterable<SkyKey>) is called on an empty Iterable, we avoid having + // to create the Map of value keys to values. This is useful in the case where the graph + // getValues() method could be slow. + this.valuesSupplier = Suppliers.memoize(new Supplier<Map<SkyKey, SkyValue>>() { + @Override + public Map<SkyKey, SkyValue> get() { + return evaluator.getValues(); + } + }); + } + + Iterable<SkyKey> getFilesystemSkyKeys() { + return Iterables.filter(valuesSupplier.get().keySet(), + FILE_STATE_AND_DIRECTORY_LISTING_STATE_FILTER); + } + + Differencer.Diff getDirtyFilesystemSkyKeys() throws InterruptedException { + return getDirtyFilesystemValues(getFilesystemSkyKeys()); + } + + /** + * Check the given file and directory values for modifications. {@code values} is assumed to only + * have {@link FileValue}s and {@link DirectoryListingStateValue}s. + */ + Differencer.Diff getDirtyFilesystemValues(Iterable<SkyKey> values) + throws InterruptedException { + return getDirtyValues(values, FILE_STATE_AND_DIRECTORY_LISTING_STATE_FILTER, + new DirtyChecker() { + @Override + public DirtyResult check(SkyKey key, SkyValue oldValue, TimestampGranularityMonitor tsgm) { + if (key.functionName() == SkyFunctions.FILE_STATE) { + return checkFileStateValue((RootedPath) key.argument(), (FileStateValue) oldValue, + tsgm); + } else if (key.functionName() == SkyFunctions.DIRECTORY_LISTING_STATE) { + return checkDirectoryListingStateValue((RootedPath) key.argument(), + (DirectoryListingStateValue) oldValue); + } else { + throw new IllegalStateException("Unexpected key type " + key); + } + } + }); + } + + /** + * Return a collection of action values which have output files that are not in-sync with + * the on-disk file value (were modified externally). + */ + public Collection<SkyKey> getDirtyActionValues(@Nullable final BatchStat batchStatter) + throws InterruptedException { + // CPU-bound (usually) stat() calls, plus a fudge factor. + LOG.info("Accumulating dirty actions"); + final int numOutputJobs = Runtime.getRuntime().availableProcessors() * 4; + final Set<SkyKey> actionSkyKeys = + Sets.filter(valuesSupplier.get().keySet(), ACTION_FILTER); + final Sharder<Pair<SkyKey, ActionExecutionValue>> outputShards = + new Sharder<>(numOutputJobs, actionSkyKeys.size()); + + for (SkyKey key : actionSkyKeys) { + outputShards.add(Pair.of(key, (ActionExecutionValue) valuesSupplier.get().get(key))); + } + LOG.info("Sharded action values for batching"); + + ExecutorService executor = Executors.newFixedThreadPool( + numOutputJobs, + new ThreadFactoryBuilder().setNameFormat("FileSystem Output File Invalidator %d").build()); + + Collection<SkyKey> dirtyKeys = Sets.newConcurrentHashSet(); + ThrowableRecordingRunnableWrapper wrapper = + new ThrowableRecordingRunnableWrapper("FileSystemValueChecker#getDirtyActionValues"); + + modifiedOutputFilesCounter.set(0); + for (List<Pair<SkyKey, ActionExecutionValue>> shard : outputShards) { + Runnable job = (batchStatter == null) + ? outputStatJob(dirtyKeys, shard) + : batchStatJob(dirtyKeys, shard, batchStatter); + executor.submit(wrapper.wrap(job)); + } + + boolean interrupted = ExecutorShutdownUtil.interruptibleShutdown(executor); + Throwables.propagateIfPossible(wrapper.getFirstThrownError()); + LOG.info("Completed output file stat checks"); + if (interrupted) { + throw new InterruptedException(); + } + return dirtyKeys; + } + + private Runnable batchStatJob(final Collection<SkyKey> dirtyKeys, + final List<Pair<SkyKey, ActionExecutionValue>> shard, + final BatchStat batchStatter) { + return new Runnable() { + @Override + public void run() { + Map<Artifact, Pair<SkyKey, ActionExecutionValue>> artifactToKeyAndValue = new HashMap<>(); + for (Pair<SkyKey, ActionExecutionValue> keyAndValue : shard) { + ActionExecutionValue actionValue = keyAndValue.getSecond(); + if (actionValue == null) { + dirtyKeys.add(keyAndValue.getFirst()); + } else { + for (Artifact artifact : actionValue.getAllOutputArtifactData().keySet()) { + artifactToKeyAndValue.put(artifact, keyAndValue); + } + } + } + + List<Artifact> artifacts = ImmutableList.copyOf(artifactToKeyAndValue.keySet()); + List<FileStatusWithDigest> stats; + try { + stats = batchStatter.batchStat(/*includeDigest=*/true, /*includeLinks=*/true, + Artifact.asPathFragments(artifacts)); + } catch (IOException e) { + // Batch stat did not work. Log an exception and fall back on system calls. + LoggingUtil.logToRemote(Level.WARNING, "Unable to process batch stat", e); + outputStatJob(dirtyKeys, shard).run(); + return; + } catch (InterruptedException e) { + // We handle interrupt in the main thread. + return; + } + + Preconditions.checkState(artifacts.size() == stats.size(), + "artifacts.size() == %s stats.size() == %s", artifacts.size(), stats.size()); + for (int i = 0; i < artifacts.size(); i++) { + Artifact artifact = artifacts.get(i); + FileStatusWithDigest stat = stats.get(i); + Pair<SkyKey, ActionExecutionValue> keyAndValue = artifactToKeyAndValue.get(artifact); + ActionExecutionValue actionValue = keyAndValue.getSecond(); + SkyKey key = keyAndValue.getFirst(); + FileValue lastKnownData = actionValue.getAllOutputArtifactData().get(artifact); + try { + FileValue newData = FileAndMetadataCache.fileValueFromArtifact(artifact, stat, tsgm); + if (!newData.equals(lastKnownData)) { + modifiedOutputFilesCounter.getAndIncrement(); + dirtyKeys.add(key); + } + } catch (IOException e) { + // This is an unexpected failure getting a digest or symlink target. + modifiedOutputFilesCounter.getAndIncrement(); + dirtyKeys.add(key); + } + } + } + }; + } + + private Runnable outputStatJob(final Collection<SkyKey> dirtyKeys, + final List<Pair<SkyKey, ActionExecutionValue>> shard) { + return new Runnable() { + @Override + public void run() { + for (Pair<SkyKey, ActionExecutionValue> keyAndValue : shard) { + ActionExecutionValue value = keyAndValue.getSecond(); + if (value == null || actionValueIsDirtyWithDirectSystemCalls(value)) { + dirtyKeys.add(keyAndValue.getFirst()); + } + } + } + }; + } + + /** + * Returns number of modified output files inside of dirty actions. + */ + int getNumberOfModifiedOutputFiles() { + return modifiedOutputFilesCounter.get(); + } + + private boolean actionValueIsDirtyWithDirectSystemCalls(ActionExecutionValue actionValue) { + boolean isDirty = false; + for (Map.Entry<Artifact, FileValue> entry : + actionValue.getAllOutputArtifactData().entrySet()) { + Artifact artifact = entry.getKey(); + FileValue lastKnownData = entry.getValue(); + try { + if (!FileAndMetadataCache.fileValueFromArtifact(artifact, null, tsgm).equals( + lastKnownData)) { + modifiedOutputFilesCounter.getAndIncrement(); + isDirty = true; + } + } catch (IOException e) { + // This is an unexpected failure getting a digest or symlink target. + modifiedOutputFilesCounter.getAndIncrement(); + isDirty = true; + } + } + return isDirty; + } + + private BatchDirtyResult getDirtyValues(Iterable<SkyKey> values, + Predicate<SkyKey> keyFilter, + final DirtyChecker checker) throws InterruptedException { + ExecutorService executor = Executors.newFixedThreadPool(DIRTINESS_CHECK_THREADS, + new ThreadFactoryBuilder().setNameFormat("FileSystem Value Invalidator %d").build()); + + final BatchDirtyResult batchResult = new BatchDirtyResult(); + ThrowableRecordingRunnableWrapper wrapper = + new ThrowableRecordingRunnableWrapper("FilesystemValueChecker#getDirtyValues"); + for (final SkyKey key : values) { + Preconditions.checkState(keyFilter.apply(key), key); + final SkyValue value = valuesSupplier.get().get(key); + executor.execute(wrapper.wrap(new Runnable() { + @Override + public void run() { + if (value == null) { + // value will be null if the value is in error or part of a cycle. + // TODO(bazel-team): This is overly conservative. + batchResult.add(key, /*newValue=*/null); + return; + } + DirtyResult result = checker.check(key, value, tsgm); + if (result.isDirty()) { + batchResult.add(key, result.getNewValue()); + } + } + })); + } + + boolean interrupted = ExecutorShutdownUtil.interruptibleShutdown(executor); + Throwables.propagateIfPossible(wrapper.getFirstThrownError()); + if (interrupted) { + throw new InterruptedException(); + } + return batchResult; + } + + private static DirtyResult checkFileStateValue(RootedPath rootedPath, + FileStateValue fileStateValue, TimestampGranularityMonitor tsgm) { + try { + FileStateValue newValue = FileStateValue.create(rootedPath, tsgm); + return newValue.equals(fileStateValue) + ? DirtyResult.NOT_DIRTY : DirtyResult.dirtyWithNewValue(newValue); + } catch (InconsistentFilesystemException | IOException e) { + // TODO(bazel-team): An IOException indicates a failure to get a file digest or a symlink + // target, not a missing file. Such a failure really shouldn't happen, so failing early + // may be better here. + return DirtyResult.DIRTY; + } + } + + private static DirtyResult checkDirectoryListingStateValue(RootedPath dirRootedPath, + DirectoryListingStateValue directoryListingStateValue) { + try { + DirectoryListingStateValue newValue = DirectoryListingStateValue.create(dirRootedPath); + return newValue.equals(directoryListingStateValue) + ? DirtyResult.NOT_DIRTY : DirtyResult.dirtyWithNewValue(newValue); + } catch (IOException e) { + return DirtyResult.DIRTY; + } + } + + /** + * Result of a batch call to {@link DirtyChecker#check}. Partitions the dirty values based on + * whether we have a new value available for them or not. + */ + private static class BatchDirtyResult implements Differencer.Diff { + + private final Set<SkyKey> concurrentDirtyKeysWithoutNewValues = + Collections.newSetFromMap(new ConcurrentHashMap<SkyKey, Boolean>()); + private final ConcurrentHashMap<SkyKey, SkyValue> concurrentDirtyKeysWithNewValues = + new ConcurrentHashMap<>(); + + private void add(SkyKey key, @Nullable SkyValue newValue) { + if (newValue == null) { + concurrentDirtyKeysWithoutNewValues.add(key); + } else { + concurrentDirtyKeysWithNewValues.put(key, newValue); + } + } + + @Override + public Iterable<SkyKey> changedKeysWithoutNewValues() { + return concurrentDirtyKeysWithoutNewValues; + } + + @Override + public Map<SkyKey, ? extends SkyValue> changedKeysWithNewValues() { + return concurrentDirtyKeysWithNewValues; + } + } + + private static class DirtyResult { + + static final DirtyResult NOT_DIRTY = new DirtyResult(false, null); + static final DirtyResult DIRTY = new DirtyResult(true, null); + + private final boolean isDirty; + @Nullable private final SkyValue newValue; + + private DirtyResult(boolean isDirty, @Nullable SkyValue newValue) { + this.isDirty = isDirty; + this.newValue = newValue; + } + + boolean isDirty() { + return isDirty; + } + + /** + * If {@code isDirty()}, then either returns the new value for the value or {@code null} if + * the new value wasn't computed. In the case where the value is dirty and a new value is + * available, then the new value can be injected into the skyframe graph. Otherwise, the value + * should simply be invalidated. + */ + @Nullable + SkyValue getNewValue() { + Preconditions.checkState(isDirty()); + return newValue; + } + + static DirtyResult dirtyWithNewValue(SkyValue newValue) { + return new DirtyResult(true, newValue); + } + } + + private static interface DirtyChecker { + DirtyResult check(SkyKey key, SkyValue oldValue, TimestampGranularityMonitor tsgm); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/GlobDescriptor.java b/src/main/java/com/google/devtools/build/lib/skyframe/GlobDescriptor.java new file mode 100644 index 0000000..5baeae8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/GlobDescriptor.java
@@ -0,0 +1,113 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.util.StringCanonicalizer; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.Serializable; +import java.util.Objects; + +/** + * A descriptor for a glob request. + * + * <p>{@code subdir} must be empty or point to an existing directory.</p> + * + * <p>{@code pattern} must be valid, as indicated by {@code UnixGlob#checkPatternForError}. + */ +@ThreadSafe +public final class GlobDescriptor implements Serializable { + final PackageIdentifier packageId; + final PathFragment subdir; + final String pattern; + final boolean excludeDirs; + + /** + * Constructs a GlobDescriptor. + * + * @param packageId the name of the owner package (must be an existing package) + * @param subdir the subdirectory being looked at (must exist and must be a directory. It's + * assumed that there are no other packages between {@code packageName} and + * {@code subdir}. + * @param pattern a valid glob pattern + * @param excludeDirs true if directories should be excluded from results + */ + GlobDescriptor(PackageIdentifier packageId, PathFragment subdir, String pattern, + boolean excludeDirs) { + this.packageId = Preconditions.checkNotNull(packageId); + this.subdir = Preconditions.checkNotNull(subdir); + this.pattern = Preconditions.checkNotNull(StringCanonicalizer.intern(pattern)); + this.excludeDirs = excludeDirs; + } + + @Override + public String toString() { + return String.format("<GlobDescriptor packageName=%s subdir=%s pattern=%s excludeDirs=%s>", + packageId, subdir, pattern, excludeDirs); + } + + /** + * Returns the package that "owns" this glob. + * + * <p>The glob evaluation code ensures that the boundaries of this package are not crossed. + */ + public PackageIdentifier getPackageId() { + return packageId; + } + + /** + * Returns the subdirectory of the package under consideration. + */ + PathFragment getSubdir() { + return subdir; + } + + /** + * Returns the glob pattern under consideration. May contain wildcards. + * + * <p>As the glob evaluator traverses deeper into the file tree, components are added at the + * beginning of {@code subdir} and removed from the beginning of {@code pattern}. + */ + String getPattern() { + return pattern; + } + + /** + * Returns true if directories should be excluded from results. + */ + boolean excludeDirs() { + return excludeDirs; + } + + @Override + public int hashCode() { + return Objects.hash(packageId, subdir, pattern, excludeDirs); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof GlobDescriptor)) { + return false; + } + GlobDescriptor other = (GlobDescriptor) obj; + return packageId.equals(other.packageId) && subdir.equals(other.subdir) + && pattern.equals(other.pattern) && excludeDirs == other.excludeDirs; + } +} \ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/GlobFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/GlobFunction.java new file mode 100644 index 0000000..a1fdcb2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/GlobFunction.java
@@ -0,0 +1,251 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.vfs.Dirent; +import com.google.devtools.build.lib.vfs.Dirent.Type; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.lib.vfs.UnixGlob; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +/** + * A {@link SkyFunction} for {@link GlobValue}s. + * + * <p>This code drives the glob matching process. + */ +final class GlobFunction implements SkyFunction { + + private final Cache<String, Pattern> regexPatternCache = + CacheBuilder.newBuilder().concurrencyLevel(4).build(); + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws GlobFunctionException { + GlobDescriptor glob = (GlobDescriptor) skyKey.argument(); + + PackageLookupValue globPkgLookupValue = (PackageLookupValue) + env.getValue(PackageLookupValue.key(glob.getPackageId())); + if (globPkgLookupValue == null) { + return null; + } + Preconditions.checkState(globPkgLookupValue.packageExists(), "%s isn't an existing package", + glob.getPackageId()); + // Note that this implies that the package's BUILD file exists which implies that the + // package's directory exists. + + PathFragment globSubdir = glob.getSubdir(); + if (!globSubdir.equals(PathFragment.EMPTY_FRAGMENT)) { + PackageLookupValue globSubdirPkgLookupValue = (PackageLookupValue) env.getValue( + PackageLookupValue.key(glob.getPackageId().getPackageFragment() + .getRelative(globSubdir))); + if (globSubdirPkgLookupValue == null) { + return null; + } + if (globSubdirPkgLookupValue.packageExists()) { + // We crossed the package boundary, that is, pkg/subdir contains a BUILD file and thus + // defines another package, so glob expansion should not descend into that subdir. + return GlobValue.EMPTY; + } + } + + String pattern = glob.getPattern(); + // Split off the first path component of the pattern. + int slashPos = pattern.indexOf("/"); + String patternHead; + String patternTail; + if (slashPos == -1) { + patternHead = pattern; + patternTail = null; + } else { + // Substrings will share the backing array of the original glob string. That should be fine. + patternHead = pattern.substring(0, slashPos); + patternTail = pattern.substring(slashPos + 1); + } + + NestedSetBuilder<PathFragment> matches = NestedSetBuilder.stableOrder(); + + // "**" also matches an empty segment, so try the case where it is not present. + if ("**".equals(patternHead)) { + if (patternTail == null) { + if (!glob.excludeDirs()) { + matches.add(globSubdir); + } + } else { + SkyKey globKey = GlobValue.internalKey( + glob.getPackageId(), globSubdir, patternTail, glob.excludeDirs()); + GlobValue globValue = (GlobValue) env.getValue(globKey); + if (globValue == null) { + return null; + } + matches.addTransitive(globValue.getMatches()); + } + } + + PathFragment dirPathFragment = glob.getPackageId().getPackageFragment().getRelative(globSubdir); + RootedPath dirRootedPath = RootedPath.toRootedPath(globPkgLookupValue.getRoot(), + dirPathFragment); + if (containsGlobs(patternHead)) { + // Pattern contains globs, so a directory listing is required. + // + // Note that we have good reason to believe the directory exists: if this is the + // top-level directory of the package, the package's existence implies the directory's + // existence; if this is a lower-level directory in the package, then we got here from + // previous directory listings. Filesystem operations concurrent with build could mean the + // directory no longer exists, but DirectoryListingFunction handles that gracefully. + DirectoryListingValue listingValue = (DirectoryListingValue) + env.getValue(DirectoryListingValue.key(dirRootedPath)); + if (listingValue == null) { + return null; + } + + for (Dirent dirent : listingValue.getDirents()) { + Type direntType = dirent.getType(); + String fileName = dirent.getName(); + + boolean isDirectory = (direntType == Dirent.Type.DIRECTORY); + + if (!UnixGlob.matches(patternHead, fileName, regexPatternCache)) { + continue; + } + + if (direntType == Dirent.Type.SYMLINK) { + // TODO(bazel-team): Consider extracting the symlink resolution logic. + // For symlinks, look up the corresponding FileValue. This ensures that if the symlink + // changes and "switches types" (say, from a file to a directory), this value will be + // invalidated. + RootedPath symlinkRootedPath = RootedPath.toRootedPath(globPkgLookupValue.getRoot(), + dirPathFragment.getRelative(fileName)); + FileValue symlinkFileValue = (FileValue) env.getValue(FileValue.key(symlinkRootedPath)); + if (symlinkFileValue == null) { + continue; + } + if (!symlinkFileValue.isSymlink()) { + throw new GlobFunctionException(new InconsistentFilesystemException( + "readdir and stat disagree about whether " + symlinkRootedPath.asPath() + + " is a symlink."), Transience.TRANSIENT); + } + isDirectory = symlinkFileValue.isDirectory(); + } + + String subdirPattern = "**".equals(patternHead) ? glob.getPattern() : patternTail; + addFile(fileName, glob, subdirPattern, patternTail == null, isDirectory, + matches, env); + } + } else { + // Pattern does not contain globs, so a direct stat is enough. + String fileName = patternHead; + RootedPath fileRootedPath = RootedPath.toRootedPath(globPkgLookupValue.getRoot(), + dirPathFragment.getRelative(fileName)); + FileValue fileValue = (FileValue) env.getValue(FileValue.key(fileRootedPath)); + if (fileValue == null) { + return null; + } + if (fileValue.exists()) { + addFile(fileName, glob, patternTail, patternTail == null, + fileValue.isDirectory(), matches, env); + } + } + + if (env.valuesMissing()) { + return null; + } + + NestedSet<PathFragment> matchesBuilt = matches.build(); + // Use the same value to represent that we did not match anything. + if (matchesBuilt.isEmpty()) { + return GlobValue.EMPTY; + } + return new GlobValue(matchesBuilt); + } + + /** + * Returns true if the given pattern contains globs. + */ + private boolean containsGlobs(String pattern) { + return pattern.contains("*") || pattern.contains("?"); + } + + /** + * Includes the given file/directory in the glob. + * + * <p>{@code fileName} must exist. + * + * <p>{@code isDirectory} must be true iff the file is a directory. + * + * <p>{@code directResult} must be set if the file should be included in the result set + * directly rather than recursed into if it is a directory. + */ + private void addFile(String fileName, GlobDescriptor glob, String subdirPattern, + boolean directResult, boolean isDirectory, NestedSetBuilder<PathFragment> matches, + Environment env) { + if (isDirectory && subdirPattern != null) { + // This is a directory, and the pattern covers files under that directory. + SkyKey subdirGlobKey = GlobValue.internalKey(glob.getPackageId(), + glob.getSubdir().getRelative(fileName), subdirPattern, glob.excludeDirs()); + GlobValue subdirGlob = (GlobValue) env.getValue(subdirGlobKey); + if (subdirGlob == null) { + return; + } + matches.addTransitive(subdirGlob.getMatches()); + } + + if (directResult && !(isDirectory && glob.excludeDirs())) { + if (isDirectory) { + // TODO(bazel-team): Refactor. This is basically inlined code from the next recursion level. + // Ensure that subdirectories that contain other packages are not picked up. + PathFragment directory = glob.getPackageId().getPackageFragment() + .getRelative(glob.getSubdir()).getRelative(fileName); + PackageLookupValue pkgLookupValue = (PackageLookupValue) + env.getValue(PackageLookupValue.key(directory)); + if (pkgLookupValue == null) { + return; + } + if (pkgLookupValue.packageExists()) { + // The file is a directory and contains another package. + return; + } + } + matches.add(glob.getSubdir().getRelative(fileName)); + } + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link GlobFunction#compute}. + */ + private static final class GlobFunctionException extends SkyFunctionException { + public GlobFunctionException(InconsistentFilesystemException e, Transience transience) { + super(e, transience); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/GlobValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/GlobValue.java new file mode 100644 index 0000000..6de0fbd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/GlobValue.java
@@ -0,0 +1,132 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.UnixGlob; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * A value corresponding to a glob. + */ +@Immutable +@ThreadSafe +final class GlobValue implements SkyValue { + + static final GlobValue EMPTY = new GlobValue( + NestedSetBuilder.<PathFragment>emptySet(Order.STABLE_ORDER)); + + private final NestedSet<PathFragment> matches; + + GlobValue(NestedSet<PathFragment> matches) { + this.matches = Preconditions.checkNotNull(matches); + } + + /** + * Returns glob matches. + */ + NestedSet<PathFragment> getMatches() { + return matches; + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + if (!(other instanceof GlobValue)) { + return false; + } + // shallowEquals() may fail to detect that two equivalent (according to toString()) + // NestedSets are equal, but will always detect when two NestedSets are different. + // This makes this implementation of equals() overly strict, but we only call this + // method when doing change pruning, which can accept false negatives. + return getMatches().shallowEquals(((GlobValue) other).getMatches()); + } + + @Override + public int hashCode() { + return matches.shallowHashCode(); + } + + /** + * Constructs a {@link SkyKey} for a glob lookup. {@code packageName} is assumed to be an + * existing package. Trying to glob into a non-package is undefined behavior. + * + * @throws InvalidGlobPatternException if the pattern is not valid. + */ + @ThreadSafe + static SkyKey key(PackageIdentifier packageId, String pattern, boolean excludeDirs) + throws InvalidGlobPatternException { + if (pattern.indexOf('?') != -1) { + throw new InvalidGlobPatternException(pattern, "wildcard ? forbidden"); + } + + String error = UnixGlob.checkPatternForError(pattern); + if (error != null) { + throw new InvalidGlobPatternException(pattern, error); + } + + return internalKey(packageId, PathFragment.EMPTY_FRAGMENT, pattern, excludeDirs); + } + + /** + * Constructs a {@link SkyKey} for a glob lookup. + * + * <p>Do not use outside {@code GlobFunction}. + */ + @ThreadSafe + static SkyKey internalKey(PackageIdentifier packageId, PathFragment subdir, String pattern, + boolean excludeDirs) { + return new SkyKey(SkyFunctions.GLOB, + new GlobDescriptor(packageId, subdir, pattern, excludeDirs)); + } + + /** + * Constructs a {@link SkyKey} for a glob lookup. + * + * <p>Do not use outside {@code GlobFunction}. + */ + @ThreadSafe + static SkyKey internalKey(GlobDescriptor glob, String subdirName) { + return internalKey(glob.packageId, glob.subdir.getRelative(subdirName), + glob.pattern, glob.excludeDirs); + } + + /** + * An exception that indicates that a glob pattern is syntactically invalid. + */ + @ThreadSafe + static final class InvalidGlobPatternException extends Exception { + private final String pattern; + + InvalidGlobPatternException(String pattern, String error) { + super(error); + this.pattern = pattern; + } + + @Override + public String toString() { + return String.format("invalid glob pattern '%s': %s", pattern, getMessage()); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/IncompatibleViewException.java b/src/main/java/com/google/devtools/build/lib/skyframe/IncompatibleViewException.java new file mode 100644 index 0000000..9d6f550 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/IncompatibleViewException.java
@@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; + +/** + * Thrown on {@link DiffAwareness#getDiff} to indicate that the given {@link DiffAwareness.View}s + * are incompatible with the {@link DiffAwareness} instance. + */ +public class IncompatibleViewException extends Exception { + public IncompatibleViewException(String msg) { + super(Preconditions.checkNotNull(msg)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/InconsistentFilesystemException.java b/src/main/java/com/google/devtools/build/lib/skyframe/InconsistentFilesystemException.java new file mode 100644 index 0000000..26cb02f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/InconsistentFilesystemException.java
@@ -0,0 +1,27 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +/** + * Used to indicate a filesystem inconsistency, e.g. file 'a/b' exists but directory 'a' doesn't + * exist. This generally means the result of the build is undefined but we shouldn't crash hard. + */ +public class InconsistentFilesystemException extends Exception { + public InconsistentFilesystemException(String inconsistencyMessage) { + super("Inconsistent filesystem operations. " + inconsistencyMessage + " The results of the " + + "build are not guaranteed to be correct. You should probably run 'blaze clean' and " + + "investigate the filesystem inconsistency (likely due to filesytem updates concurrent " + + "with the build)"); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/LocalDiffAwareness.java b/src/main/java/com/google/devtools/build/lib/skyframe/LocalDiffAwareness.java new file mode 100644 index 0000000..861f89ac --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/LocalDiffAwareness.java
@@ -0,0 +1,329 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.vfs.ModifiedFileSet; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.nio.file.ClosedWatchServiceException; +import java.nio.file.FileSystems; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardWatchEventKinds; +import java.nio.file.WatchEvent; +import java.nio.file.WatchEvent.Kind; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashSet; +import java.util.Set; + +/** + * File system watcher for local filesystems. It's able to provide a list of changed + * files between two consecutive calls. Uses the standard Java WatchService, which uses + * 'inotify' on Linux. + */ +public class LocalDiffAwareness implements DiffAwareness { + + /** Factory for creating {@link LocalDiffAwareness} instances. */ + public static class Factory implements DiffAwareness.Factory { + @Override + public DiffAwareness maybeCreate(com.google.devtools.build.lib.vfs.Path pathEntry) { + com.google.devtools.build.lib.vfs.Path resolvedPathEntry; + try { + resolvedPathEntry = pathEntry.resolveSymbolicLinks(); + } catch (IOException e) { + return null; + } + PathFragment resolvedPathEntryFragment = resolvedPathEntry.asFragment(); + // There's no good way to automatically detect network file systems. We rely on a blacklist + // for now (and maybe add a command-line option in the future?). + for (String prefix : Constants.WATCHFS_BLACKLIST) { + if (resolvedPathEntryFragment.startsWith(new PathFragment(prefix))) { + return null; + } + } + + WatchService watchService; + try { + watchService = FileSystems.getDefault().newWatchService(); + } catch (IOException e) { + return null; + } + return new LocalDiffAwareness(resolvedPathEntryFragment.toString(), + watchService); + } + } + + private int numGetCurrentViewCalls = 0; + + /** + * Bijection from WatchKey to the (absolute) Path being watched. WatchKeys don't have this + * functionality built-in so we do it ourselves. + */ + private final HashBiMap<WatchKey, Path> watchKeyToDirBiMap = HashBiMap.create(); + + /** Root directory to watch. This is an absolute path. */ + private final Path watchRootPath; + + /** Every directory is registered under this watch service. */ + private WatchService watchService; + + private LocalDiffAwareness(String watchRoot, WatchService watchService) { + this.watchRootPath = FileSystems.getDefault().getPath(watchRoot); + this.watchService = watchService; + } + + /** + * The WatchService is inherently sequential and side-effectful, so we enforce this by only + * supporting {@link #getDiff} calls that happen to be sequential. + */ + private static class SequentialView implements DiffAwareness.View { + private final LocalDiffAwareness owner; + private final int position; + private final Set<Path> modifiedAbsolutePaths; + + public SequentialView(LocalDiffAwareness owner, int position, Set<Path> modifiedAbsolutePaths) { + this.owner = owner; + this.position = position; + this.modifiedAbsolutePaths = modifiedAbsolutePaths; + } + + public static boolean areInSequence(SequentialView oldView, SequentialView newView) { + return oldView.owner == newView.owner && (oldView.position + 1) == newView.position; + } + } + + @Override + public SequentialView getCurrentView() throws BrokenDiffAwarenessException { + Set<Path> modifiedAbsolutePaths; + if (numGetCurrentViewCalls++ == 0) { + try { + registerSubDirectoriesAndReturnContents(watchRootPath); + } catch (IOException e) { + close(); + throw new BrokenDiffAwarenessException( + "Error encountered with local file system watcher " + e); + } + modifiedAbsolutePaths = ImmutableSet.of(); + } else { + try { + modifiedAbsolutePaths = collectChanges(); + } catch (BrokenDiffAwarenessException e) { + close(); + throw e; + } catch (IOException e) { + close(); + throw new BrokenDiffAwarenessException( + "Error encountered with local file system watcher " + e); + } catch (ClosedWatchServiceException e) { + throw new BrokenDiffAwarenessException( + "Internal error with the local file system watcher " + e); + } + } + return new SequentialView(this, numGetCurrentViewCalls, modifiedAbsolutePaths); + } + + @Override + public ModifiedFileSet getDiff(View oldView, View newView) + throws IncompatibleViewException, BrokenDiffAwarenessException { + SequentialView oldSequentialView; + SequentialView newSequentialView; + try { + oldSequentialView = (SequentialView) oldView; + newSequentialView = (SequentialView) newView; + } catch (ClassCastException e) { + throw new IncompatibleViewException("Given views are not from LocalDiffAwareness"); + } + if (!SequentialView.areInSequence(oldSequentialView, newSequentialView)) { + return ModifiedFileSet.EVERYTHING_MODIFIED; + } + return ModifiedFileSet.builder() + .modifyAll(Iterables.transform(newSequentialView.modifiedAbsolutePaths, + nioAbsolutePathToPathFragment)) + .build(); + } + + @Override + public void close() { + try { + watchService.close(); + } catch (IOException ignored) { + // Nothing we can do here. + } + } + + /** Converts java.nio.file.Path objects to vfs.PathFragment. */ + private final Function<Path, PathFragment> nioAbsolutePathToPathFragment = + new Function<Path, PathFragment>() { + @Override + public PathFragment apply(Path input) { + Preconditions.checkArgument(input.startsWith(watchRootPath), "%s %s", input, + watchRootPath); + return new PathFragment(watchRootPath.relativize(input).toString()); + } + }; + + /** Returns the changed files caught by the watch service. */ + private Set<Path> collectChanges() throws BrokenDiffAwarenessException, IOException { + Set<Path> createdFilesAndDirectories = new HashSet<Path>(); + Set<Path> deletedOrModifiedFilesAndDirectories = new HashSet<Path>(); + Set<Path> deletedTrackedDirectories = new HashSet<Path>(); + + WatchKey watchKey; + while ((watchKey = watchService.poll()) != null) { + Path dir = watchKeyToDirBiMap.get(watchKey); + Preconditions.checkArgument(dir != null); + + // We replay all the events for this watched directory in chronological order and + // construct the diff of this directory since the last #collectChanges call. + for (WatchEvent<?> event : watchKey.pollEvents()) { + Kind<?> kind = event.kind(); + if (kind == StandardWatchEventKinds.OVERFLOW) { + // TODO(bazel-team): find out when an overflow might happen, and maybe handle it more + // gently. + throw new BrokenDiffAwarenessException("Overflow when watching local filesystem for " + + "changes"); + } + if (event.context() == null) { + // The WatchService documentation mentions that WatchEvent#context may return null, but + // doesn't explain how/why it would do so. Looking at the implementation, it only + // happens on an overflow event. But we make no assumptions about that implementation + // detail here. + throw new BrokenDiffAwarenessException("Insufficient information from local file system " + + "watcher"); + } + // For the events we've registered, the context given is a relative path. + Path relativePath = (Path) event.context(); + Path path = dir.resolve(relativePath); + Preconditions.checkState(path.isAbsolute(), path); + if (kind == StandardWatchEventKinds.ENTRY_CREATE) { + createdFilesAndDirectories.add(path); + deletedOrModifiedFilesAndDirectories.remove(path); + } else if (kind == StandardWatchEventKinds.ENTRY_DELETE) { + createdFilesAndDirectories.remove(path); + deletedOrModifiedFilesAndDirectories.add(path); + if (watchKeyToDirBiMap.containsValue(path)) { + // If the deleted directory has children, then there will also be events for the + // WatchKey of the directory itself. WatchService#poll doesn't specify the order in + // which WatchKeys are returned, so the key for the directory itself may be processed + // *after* the current key (the parent of the deleted directory), and so we don't want + // to remove the deleted directory from our bimap just yet. + // + // For example, suppose we have the file '/root/a/foo.txt' and are watching the + // directories '/root' and '/root/a'. If the directory '/root/a' gets deleted then the + // following is a valid sequence of events by key. + // + // WatchKey '/root/' + // WatchEvent EVENT_MODIFY 'a' + // WatchEvent EVENT_DELETE 'a' + // WatchKey '/root/a' + // WatchEvent EVENT_DELETE 'foo.txt' + deletedTrackedDirectories.add(path); + } + } else if (kind == StandardWatchEventKinds.ENTRY_MODIFY) { + // If a file was created and then modified, then the net diff is that it was + // created. + if (!createdFilesAndDirectories.contains(path)) { + deletedOrModifiedFilesAndDirectories.add(path); + } + } + } + + if (!watchKey.reset()) { + // Watcher got deleted, directory no longer valid. + watchKeyToDirBiMap.remove(watchKey); + } + } + + for (Path path : deletedTrackedDirectories) { + WatchKey staleKey = watchKeyToDirBiMap.inverse().get(path); + watchKeyToDirBiMap.remove(staleKey); + } + if (watchKeyToDirBiMap.isEmpty()) { + // No more directories to watch, something happened the root directory being watched. + throw new IOException("Root directory " + watchRootPath + " became inaccessible."); + } + + Set<Path> changedPaths = new HashSet<Path>(); + for (Path path : createdFilesAndDirectories) { + if (Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { + // This is a new directory, so changes to it since its creation have not been watched. + // We manually traverse the directory tree to register all the new subdirectories and find + // all the new subdirectories and files. + changedPaths.addAll(registerSubDirectoriesAndReturnContents(path)); + } else { + changedPaths.add(path); + } + } + changedPaths.addAll(deletedOrModifiedFilesAndDirectories); + return changedPaths; + } + + /** + * Traverses directory tree to register subdirectories. Returns all paths traversed (as absolute + * paths). + */ + private Set<Path> registerSubDirectoriesAndReturnContents(Path rootDir) throws IOException { + Set<Path> visitedAbsolutePaths = new HashSet<Path>(); + // Note that this does not follow symlinks. + Files.walkFileTree(rootDir, new WatcherFileVisitor(visitedAbsolutePaths)); + return visitedAbsolutePaths; + } + + /** File visitor used by Files.walkFileTree() upon traversing subdirectories. */ + private class WatcherFileVisitor extends SimpleFileVisitor<Path> { + + private final Set<Path> visitedAbsolutePaths; + + private WatcherFileVisitor(Set<Path> visitedPaths) { + this.visitedAbsolutePaths = visitedPaths; + } + + @Override + public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) { + Preconditions.checkState(path.isAbsolute(), path); + visitedAbsolutePaths.add(path); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs) + throws IOException { + // It's important that we register the directory before we visit its children. This way we + // are guaranteed to see new files/directories either on this #getDiff or the next one. + // Otherwise, e.g., an intra-build creation of a child directory will be forever missed if it + // happens before the directory is listed as part of the visitation. + WatchKey key = path.register(watchService, + StandardWatchEventKinds.ENTRY_CREATE, + StandardWatchEventKinds.ENTRY_MODIFY, + StandardWatchEventKinds.ENTRY_DELETE); + Preconditions.checkState(path.isAbsolute(), path); + visitedAbsolutePaths.add(path); + watchKeyToDirBiMap.put(key, path); + return FileVisitResult.CONTINUE; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/MutableSupplier.java b/src/main/java/com/google/devtools/build/lib/skyframe/MutableSupplier.java new file mode 100644 index 0000000..86de11d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/MutableSupplier.java
@@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Objects; +import com.google.common.base.Supplier; + +/** + * Supplier whose value can be changed by clients who have a reference to it as a MutableSupplier. + * Unlike an {@code AtomicReference}, clients who are passed a MutableSupplier as a Supplier cannot + * change its value without a reckless cast. + */ +public class MutableSupplier<T> implements Supplier<T> { + private T val; + + @Override + public T get() { + return val; + } + + /** + * Sets the value of the object supplied. Do not cast a Supplier to a MutableSupplier in order to + * call this method! + */ + public void set(T newVal) { + val = newVal; + } + + @SuppressWarnings("deprecation") // MoreObjects.toStringHelper() is not in Guava + @Override + public String toString() { + return Objects.toStringHelper(getClass()) + .add("val", val).toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java new file mode 100644 index 0000000..2404b99 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java
@@ -0,0 +1,809 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException; +import com.google.devtools.build.lib.packages.BuildFileNotFoundException; +import com.google.devtools.build.lib.packages.CachingPackageLocator; +import com.google.devtools.build.lib.packages.InvalidPackageNameException; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.PackageFactory.Globber; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; +import com.google.devtools.build.lib.packages.PackageLoadedEvent; +import com.google.devtools.build.lib.packages.Preprocessor; +import com.google.devtools.build.lib.packages.RuleVisibility; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.skyframe.ASTFileLookupValue.ASTLookupInputException; +import com.google.devtools.build.lib.skyframe.GlobValue.InvalidGlobPatternException; +import com.google.devtools.build.lib.skyframe.SkylarkImportLookupFunction.SkylarkImportFailedException; +import com.google.devtools.build.lib.syntax.BuildFileAST; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.ParserInputSource; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.syntax.Statement; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.JavaClock; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.skyframe.ValueOrException3; +import com.google.devtools.build.skyframe.ValueOrException4; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.Nullable; + +/** + * A SkyFunction for {@link PackageValue}s. + */ +public class PackageFunction implements SkyFunction { + + private final EventHandler reporter; + private final PackageFactory packageFactory; + private final CachingPackageLocator packageLocator; + private final ConcurrentMap<PackageIdentifier, Package.LegacyBuilder> packageFunctionCache; + private final AtomicBoolean showLoadingProgress; + private final AtomicReference<EventBus> eventBus; + private final AtomicInteger numPackagesLoaded; + private final Profiler profiler = Profiler.instance(); + + private static final PathFragment PRELUDE_FILE_FRAGMENT = + new PathFragment(Constants.PRELUDE_FILE_DEPOT_RELATIVE_PATH); + + static final String DEFAULTS_PACKAGE_NAME = "tools/defaults"; + public static final String EXTERNAL_PACKAGE_NAME = "external"; + + static { + Preconditions.checkArgument(!PRELUDE_FILE_FRAGMENT.isAbsolute()); + } + + public PackageFunction(Reporter reporter, PackageFactory packageFactory, + CachingPackageLocator pkgLocator, AtomicBoolean showLoadingProgress, + ConcurrentMap<PackageIdentifier, Package.LegacyBuilder> packageFunctionCache, + AtomicReference<EventBus> eventBus, AtomicInteger numPackagesLoaded) { + this.reporter = reporter; + + this.packageFactory = packageFactory; + this.packageLocator = pkgLocator; + this.showLoadingProgress = showLoadingProgress; + this.packageFunctionCache = packageFunctionCache; + this.eventBus = eventBus; + this.numPackagesLoaded = numPackagesLoaded; + } + + private static void maybeThrowFilesystemInconsistency(String packageName, + Exception skyframeException, boolean packageWasInError) + throws InternalInconsistentFilesystemException { + if (!packageWasInError) { + throw new InternalInconsistentFilesystemException(packageName, "Encountered error '" + + skyframeException.getMessage() + "' but didn't encounter it when doing the same thing " + + "earlier in the build"); + } + } + + /** + * Marks the given dependencies, and returns those already present. Ignores any exception + * thrown while building the dependency, except for filesystem inconsistencies. + * + * <p>We need to mark dependencies implicitly used by the legacy package loading code, but we + * don't care about any skyframe errors since the package knows whether it's in error or not. + */ + private static Pair<? extends Map<PathFragment, PackageLookupValue>, Boolean> + getPackageLookupDepsAndPropagateInconsistentFilesystemExceptions(String packageName, + Iterable<SkyKey> depKeys, Environment env, boolean packageWasInError) + throws InternalInconsistentFilesystemException { + Preconditions.checkState( + Iterables.all(depKeys, SkyFunctions.isSkyFunction(SkyFunctions.PACKAGE_LOOKUP)), depKeys); + boolean packageShouldBeInError = packageWasInError; + ImmutableMap.Builder<PathFragment, PackageLookupValue> builder = ImmutableMap.builder(); + for (Map.Entry<SkyKey, ValueOrException3<BuildFileNotFoundException, + InconsistentFilesystemException, FileSymlinkCycleException>> entry : + env.getValuesOrThrow(depKeys, BuildFileNotFoundException.class, + InconsistentFilesystemException.class, + FileSymlinkCycleException.class).entrySet()) { + PathFragment pkgName = ((PackageIdentifier) entry.getKey().argument()).getPackageFragment(); + try { + PackageLookupValue value = (PackageLookupValue) entry.getValue().get(); + if (value != null) { + builder.put(pkgName, value); + } + } catch (BuildFileNotFoundException e) { + maybeThrowFilesystemInconsistency(packageName, e, packageWasInError); + } catch (InconsistentFilesystemException e) { + throw new InternalInconsistentFilesystemException(packageName, e); + } catch (FileSymlinkCycleException e) { + // Legacy doesn't detect symlink cycles. + packageShouldBeInError = true; + } + } + return Pair.of(builder.build(), packageShouldBeInError); + } + + private static boolean markFileDepsAndPropagateInconsistentFilesystemExceptions( + String packageName, Iterable<SkyKey> depKeys, Environment env, boolean packageWasInError) + throws InternalInconsistentFilesystemException { + Preconditions.checkState( + Iterables.all(depKeys, SkyFunctions.isSkyFunction(SkyFunctions.FILE)), depKeys); + boolean packageShouldBeInError = packageWasInError; + for (Map.Entry<SkyKey, ValueOrException3<IOException, FileSymlinkCycleException, + InconsistentFilesystemException>> entry : env.getValuesOrThrow(depKeys, IOException.class, + FileSymlinkCycleException.class, InconsistentFilesystemException.class).entrySet()) { + try { + entry.getValue().get(); + } catch (IOException e) { + maybeThrowFilesystemInconsistency(packageName, e, packageWasInError); + } catch (FileSymlinkCycleException e) { + // Legacy doesn't detect symlink cycles. + packageShouldBeInError = true; + } catch (InconsistentFilesystemException e) { + throw new InternalInconsistentFilesystemException(packageName, e); + } + } + return packageShouldBeInError; + } + + private static boolean markGlobDepsAndPropagateInconsistentFilesystemExceptions( + String packageName, Iterable<SkyKey> depKeys, Environment env, boolean packageWasInError) + throws InternalInconsistentFilesystemException { + Preconditions.checkState( + Iterables.all(depKeys, SkyFunctions.isSkyFunction(SkyFunctions.GLOB)), depKeys); + boolean packageShouldBeInError = packageWasInError; + for (Map.Entry<SkyKey, ValueOrException4<IOException, BuildFileNotFoundException, + FileSymlinkCycleException, InconsistentFilesystemException>> entry : + env.getValuesOrThrow(depKeys, IOException.class, BuildFileNotFoundException.class, + FileSymlinkCycleException.class, InconsistentFilesystemException.class).entrySet()) { + try { + entry.getValue().get(); + } catch (IOException | BuildFileNotFoundException e) { + maybeThrowFilesystemInconsistency(packageName, e, packageWasInError); + } catch (FileSymlinkCycleException e) { + // Legacy doesn't detect symlink cycles. + packageShouldBeInError = true; + } catch (InconsistentFilesystemException e) { + throw new InternalInconsistentFilesystemException(packageName, e); + } + } + return packageShouldBeInError; + } + + /** + * Marks dependencies implicitly used by legacy package loading code, after the fact. Note that + * the given package might already be in error. + * + * <p>Any skyframe exceptions encountered here are ignored, as similar errors should have + * already been encountered by legacy package loading (if not, then the filesystem is + * inconsistent). + */ + private static boolean markDependenciesAndPropagateInconsistentFilesystemExceptions( + Package pkg, Environment env, Collection<Pair<String, Boolean>> globPatterns, + Map<Label, Path> subincludes) throws InternalInconsistentFilesystemException { + boolean packageShouldBeInError = pkg.containsErrors(); + + // TODO(bazel-team): This means that many packages will have to be preprocessed twice. Ouch! + // We need a better continuation mechanism to avoid repeating work. [skyframe-loading] + + // TODO(bazel-team): It would be preferable to perform I/O from the package preprocessor via + // Skyframe rather than add (potentially incomplete) dependencies after the fact. + // [skyframe-loading] + + Set<SkyKey> subincludePackageLookupDepKeys = Sets.newHashSet(); + for (Label label : pkg.getSubincludeLabels()) { + // Declare a dependency on the package lookup for the package giving access to the label. + subincludePackageLookupDepKeys.add(PackageLookupValue.key(label.getPackageFragment())); + } + Pair<? extends Map<PathFragment, PackageLookupValue>, Boolean> subincludePackageLookupResult = + getPackageLookupDepsAndPropagateInconsistentFilesystemExceptions(pkg.getName(), + subincludePackageLookupDepKeys, env, pkg.containsErrors()); + Map<PathFragment, PackageLookupValue> subincludePackageLookupDeps = + subincludePackageLookupResult.getFirst(); + packageShouldBeInError = subincludePackageLookupResult.getSecond(); + List<SkyKey> subincludeFileDepKeys = Lists.newArrayList(); + for (Entry<Label, Path> subincludeEntry : subincludes.entrySet()) { + // Ideally, we would have a direct dependency on the target with the given label, but then + // subincluding a file from the same package will cause a dependency cycle, since targets + // depend on their containing packages. + Label label = subincludeEntry.getKey(); + PackageLookupValue subincludePackageLookupValue = + subincludePackageLookupDeps.get(label.getPackageFragment()); + if (subincludePackageLookupValue != null) { + // Declare a dependency on the actual file that was subincluded. + Path subincludeFilePath = subincludeEntry.getValue(); + if (subincludeFilePath != null) { + if (!subincludePackageLookupValue.packageExists()) { + // Legacy blaze puts a non-null path when only when the package does indeed exist. + throw new InternalInconsistentFilesystemException(pkg.getName(), String.format( + "Unexpected package in %s. Was it modified during the build?", subincludeFilePath)); + } + // Sanity check for consistency of Skyframe and legacy blaze. + Path subincludeFilePathSkyframe = + subincludePackageLookupValue.getRoot().getRelative(label.toPathFragment()); + if (!subincludeFilePathSkyframe.equals(subincludeFilePath)) { + throw new InternalInconsistentFilesystemException(pkg.getName(), String.format( + "Inconsistent package location for %s: '%s' vs '%s'. " + + "Was the source tree modified during the build?", + label.getPackageFragment(), subincludeFilePathSkyframe, subincludeFilePath)); + } + // The actual file may be under a different package root than the package being + // constructed. + SkyKey subincludeSkyKey = + FileValue.key(RootedPath.toRootedPath(subincludePackageLookupValue.getRoot(), + subincludeFilePath)); + subincludeFileDepKeys.add(subincludeSkyKey); + } + } + } + packageShouldBeInError = markFileDepsAndPropagateInconsistentFilesystemExceptions( + pkg.getName(), subincludeFileDepKeys, env, pkg.containsErrors()); + // Another concern is a subpackage cutting off the subinclude label, but this is already + // handled by the legacy package loading code which calls into our SkyframePackageLocator. + + // TODO(bazel-team): In the long term, we want to actually resolve the glob patterns within + // Skyframe. For now, just logging the glob requests provides correct incrementality and + // adequate performance. + PackageIdentifier packageId = pkg.getPackageIdentifier(); + List<SkyKey> globDepKeys = Lists.newArrayList(); + for (Pair<String, Boolean> globPattern : globPatterns) { + String pattern = globPattern.getFirst(); + boolean excludeDirs = globPattern.getSecond(); + SkyKey globSkyKey; + try { + globSkyKey = GlobValue.key(packageId, pattern, excludeDirs); + } catch (InvalidGlobPatternException e) { + // Globs that make it to pkg.getGlobPatterns() should already be filtered for errors. + throw new IllegalStateException(e); + } + globDepKeys.add(globSkyKey); + } + packageShouldBeInError = markGlobDepsAndPropagateInconsistentFilesystemExceptions( + pkg.getName(), globDepKeys, env, pkg.containsErrors()); + return packageShouldBeInError; + } + + /** + * Adds a dependency on the WORKSPACE file, representing it as a special type of package. + * @throws PackageFunctionException if there is an error computing the workspace file or adding + * its rules to the //external package. + */ + private SkyValue getExternalPackage(Environment env, Path packageLookupPath) + throws PackageFunctionException { + RootedPath workspacePath = RootedPath.toRootedPath( + packageLookupPath, new PathFragment("WORKSPACE")); + SkyKey workspaceKey = WorkspaceFileValue.key(workspacePath); + WorkspaceFileValue workspace = null; + try { + workspace = (WorkspaceFileValue) env.getValueOrThrow(workspaceKey, IOException.class, + FileSymlinkCycleException.class, InconsistentFilesystemException.class, + EvalException.class); + } catch (IOException | FileSymlinkCycleException | InconsistentFilesystemException + | EvalException e) { + throw new PackageFunctionException(new BadWorkspaceFileException(e.getMessage()), + Transience.PERSISTENT); + } + if (workspace == null) { + return null; + } + + Package pkg = workspace.getPackage(); + Event.replayEventsOn(env.getListener(), pkg.getEvents()); + if (pkg.containsErrors()) { + throw new PackageFunctionException(new BuildFileContainsErrorsException("external", + "Package 'external' contains errors"), + pkg.containsTemporaryErrors() ? Transience.TRANSIENT : Transience.PERSISTENT); + } + + return new PackageValue(pkg); + } + + @Override + public SkyValue compute(SkyKey key, Environment env) throws PackageFunctionException, + InterruptedException { + PackageIdentifier packageId = (PackageIdentifier) key.argument(); + PathFragment packageNameFragment = packageId.getPackageFragment(); + String packageName = packageNameFragment.getPathString(); + + SkyKey packageLookupKey = PackageLookupValue.key(packageId); + PackageLookupValue packageLookupValue; + try { + packageLookupValue = (PackageLookupValue) + env.getValueOrThrow(packageLookupKey, BuildFileNotFoundException.class, + InconsistentFilesystemException.class); + } catch (BuildFileNotFoundException e) { + throw new PackageFunctionException(e, Transience.PERSISTENT); + } catch (InconsistentFilesystemException e) { + // This error is not transient from the perspective of the PackageFunction. + throw new PackageFunctionException( + new InternalInconsistentFilesystemException(packageName, e), Transience.PERSISTENT); + } + if (packageLookupValue == null) { + return null; + } + + if (!packageLookupValue.packageExists()) { + switch (packageLookupValue.getErrorReason()) { + case NO_BUILD_FILE: + case DELETED_PACKAGE: + case NO_EXTERNAL_PACKAGE: + throw new PackageFunctionException(new BuildFileNotFoundException(packageName, + packageLookupValue.getErrorMsg()), Transience.PERSISTENT); + case INVALID_PACKAGE_NAME: + throw new PackageFunctionException(new InvalidPackageNameException(packageName, + packageLookupValue.getErrorMsg()), Transience.PERSISTENT); + default: + // We should never get here. + Preconditions.checkState(false); + } + } + + if (packageName.equals(EXTERNAL_PACKAGE_NAME)) { + return getExternalPackage(env, packageLookupValue.getRoot()); + } + + RootedPath buildFileRootedPath = RootedPath.toRootedPath(packageLookupValue.getRoot(), + packageNameFragment.getChild("BUILD")); + FileValue buildFileValue; + try { + buildFileValue = (FileValue) env.getValueOrThrow(FileValue.key(buildFileRootedPath), + IOException.class, FileSymlinkCycleException.class, + InconsistentFilesystemException.class); + } catch (IOException | FileSymlinkCycleException | InconsistentFilesystemException e) { + throw new IllegalStateException("Package lookup succeeded but encountered error when " + + "getting FileValue for BUILD file directly.", e); + } + if (buildFileValue == null) { + return null; + } + Preconditions.checkState(buildFileValue.exists(), + "Package lookup succeeded but BUILD file doesn't exist"); + Path buildFilePath = buildFileRootedPath.asPath(); + + String replacementContents = null; + if (packageName.equals(DEFAULTS_PACKAGE_NAME)) { + replacementContents = PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.get(env); + if (replacementContents == null) { + return null; + } + } + + RuleVisibility defaultVisibility = PrecomputedValue.DEFAULT_VISIBILITY.get(env); + if (defaultVisibility == null) { + return null; + } + + ASTFileLookupValue astLookupValue = null; + SkyKey astLookupKey = null; + try { + astLookupKey = ASTFileLookupValue.key(PRELUDE_FILE_FRAGMENT); + } catch (ASTLookupInputException e) { + // There's a static check ensuring that PRELUDE_FILE_FRAGMENT is relative. + throw new IllegalStateException(e); + } + try { + astLookupValue = (ASTFileLookupValue) env.getValueOrThrow(astLookupKey, + ErrorReadingSkylarkExtensionException.class, InconsistentFilesystemException.class); + } catch (ErrorReadingSkylarkExtensionException | InconsistentFilesystemException e) { + throw new PackageFunctionException(new BadPreludeFileException(packageName, e.getMessage()), + Transience.PERSISTENT); + } + if (astLookupValue == null) { + return null; + } + List<Statement> preludeStatements = astLookupValue == ASTFileLookupValue.NO_FILE + ? ImmutableList.<Statement>of() : astLookupValue.getAST().getStatements(); + + // Load the BUILD file AST and handle Skylark dependencies. This way BUILD files are + // only loaded twice if there are unavailable Skylark or package dependencies or an + // IOException occurs. Note that the BUILD files are still parsed two times. + ParserInputSource inputSource; + try { + if (showLoadingProgress.get() && !packageFunctionCache.containsKey(packageId)) { + // TODO(bazel-team): don't duplicate the loading message if there are unavailable + // Skylark dependencies. + reporter.handle(Event.progress("Loading package: " + packageName)); + } + inputSource = ParserInputSource.create(buildFilePath); + } catch (IOException e) { + env.getListener().handle(Event.error(Location.fromFile(buildFilePath), e.getMessage())); + // Note that we did this work, so we should conservatively report this error as transient. + throw new PackageFunctionException(new BuildFileContainsErrorsException( + packageName, e.getMessage()), Transience.TRANSIENT); + } + SkylarkImportResult importResult = fetchImportsFromBuildFile( + buildFilePath, packageId.getRepository(), preludeStatements, inputSource, packageName, env); + if (importResult == null) { + return null; + } + + Package.LegacyBuilder legacyPkgBuilder = loadPackage(inputSource, replacementContents, + packageId, buildFilePath, defaultVisibility, preludeStatements, importResult); + legacyPkgBuilder.buildPartial(); + try { + handleLabelsCrossingSubpackagesAndPropagateInconsistentFilesystemExceptions( + packageLookupValue.getRoot(), packageId, legacyPkgBuilder, env); + } catch (InternalInconsistentFilesystemException e) { + packageFunctionCache.remove(packageId); + throw new PackageFunctionException(e, + e.isTransient() ? Transience.TRANSIENT : Transience.PERSISTENT); + } + if (env.valuesMissing()) { + // The package we just loaded will be in the {@code packageFunctionCache} next when this + // SkyFunction is called again. + return null; + } + Collection<Pair<String, Boolean>> globPatterns = legacyPkgBuilder.getGlobPatterns(); + Map<Label, Path> subincludes = legacyPkgBuilder.getSubincludes(); + Package pkg = legacyPkgBuilder.finishBuild(); + Event.replayEventsOn(env.getListener(), pkg.getEvents()); + boolean packageShouldBeConsideredInError = pkg.containsErrors(); + try { + packageShouldBeConsideredInError = + markDependenciesAndPropagateInconsistentFilesystemExceptions(pkg, env, + globPatterns, subincludes); + } catch (InternalInconsistentFilesystemException e) { + packageFunctionCache.remove(packageId); + throw new PackageFunctionException(e, + e.isTransient() ? Transience.TRANSIENT : Transience.PERSISTENT); + } + + if (env.valuesMissing()) { + return null; + } + // We know this SkyFunction will not be called again, so we can remove the cache entry. + packageFunctionCache.remove(packageId); + + if (packageShouldBeConsideredInError) { + throw new PackageFunctionException(new BuildFileContainsErrorsException(pkg, + "Package '" + packageName + "' contains errors"), + pkg.containsTemporaryErrors() ? Transience.TRANSIENT : Transience.PERSISTENT); + } + return new PackageValue(pkg); + } + + private SkylarkImportResult fetchImportsFromBuildFile(Path buildFilePath, RepositoryName repo, + List<Statement> preludeStatements, ParserInputSource inputSource, + String packageName, Environment env) throws PackageFunctionException { + StoredEventHandler eventHandler = new StoredEventHandler(); + BuildFileAST buildFileAST = BuildFileAST.parseBuildFile( + inputSource, preludeStatements, eventHandler, null, true); + + if (eventHandler.hasErrors()) { + // In case of Python preprocessing, errors have already been reported (see checkSyntax). + // In other cases, errors will be reported later. + // TODO(bazel-team): maybe we could get rid of checkSyntax and always report errors here? + return new SkylarkImportResult( + ImmutableMap.<PathFragment, SkylarkEnvironment>of(), + ImmutableList.<Label>of()); + } + + ImmutableCollection<PathFragment> imports = buildFileAST.getImports(); + Map<PathFragment, SkylarkEnvironment> importMap = new HashMap<>(); + ImmutableList.Builder<SkylarkFileDependency> fileDependencies = ImmutableList.builder(); + try { + for (PathFragment importFile : imports) { + SkyKey importsLookupKey = SkylarkImportLookupValue.key(repo, importFile); + SkylarkImportLookupValue importLookupValue = (SkylarkImportLookupValue) + env.getValueOrThrow(importsLookupKey, SkylarkImportFailedException.class, + InconsistentFilesystemException.class, ASTLookupInputException.class, + BuildFileNotFoundException.class); + if (importLookupValue != null) { + importMap.put(importFile, importLookupValue.getImportedEnvironment()); + fileDependencies.add(importLookupValue.getDependency()); + } + } + } catch (SkylarkImportFailedException e) { + env.getListener().handle(Event.error(Location.fromFile(buildFilePath), e.getMessage())); + throw new PackageFunctionException(new BuildFileContainsErrorsException(packageName, + e.getMessage()), Transience.PERSISTENT); + } catch (InconsistentFilesystemException e) { + throw new PackageFunctionException(new InternalInconsistentFilesystemException(packageName, + e), Transience.PERSISTENT); + } catch (ASTLookupInputException e) { + // The load syntax is bad in the BUILD file so BuildFileContainsErrorsException is OK. + throw new PackageFunctionException(new BuildFileContainsErrorsException(packageName, + e.getMessage()), Transience.PERSISTENT); + } catch (BuildFileNotFoundException e) { + throw new PackageFunctionException(e, Transience.PERSISTENT); + } + if (env.valuesMissing()) { + // There are unavailable Skylark dependencies. + return null; + } + return new SkylarkImportResult(importMap, transitiveClosureOfLabels(fileDependencies.build())); + } + + private ImmutableList<Label> transitiveClosureOfLabels( + ImmutableList<SkylarkFileDependency> immediateDeps) { + Set<Label> transitiveClosure = Sets.newHashSet(); + transitiveClosureOfLabels(immediateDeps, transitiveClosure); + return ImmutableList.copyOf(transitiveClosure); + } + + private void transitiveClosureOfLabels( + ImmutableList<SkylarkFileDependency> immediateDeps, Set<Label> transitiveClosure) { + for (SkylarkFileDependency dep : immediateDeps) { + if (!transitiveClosure.contains(dep.getLabel())) { + transitiveClosure.add(dep.getLabel()); + transitiveClosureOfLabels(dep.getDependencies(), transitiveClosure); + } + } + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + private static void handleLabelsCrossingSubpackagesAndPropagateInconsistentFilesystemExceptions( + Path pkgRoot, PackageIdentifier pkgId, Package.LegacyBuilder pkgBuilder, Environment env) + throws InternalInconsistentFilesystemException { + Set<SkyKey> containingPkgLookupKeys = Sets.newHashSet(); + Map<Target, SkyKey> targetToKey = new HashMap<>(); + for (Target target : pkgBuilder.getTargets()) { + PathFragment dir = target.getLabel().toPathFragment().getParentDirectory(); + PackageIdentifier dirId = new PackageIdentifier(pkgId.getRepository(), dir); + if (dir.equals(pkgId.getPackageFragment())) { + continue; + } + SkyKey key = ContainingPackageLookupValue.key(dirId); + targetToKey.put(target, key); + containingPkgLookupKeys.add(key); + } + Map<Label, SkyKey> subincludeToKey = new HashMap<>(); + for (Label subincludeLabel : pkgBuilder.getSubincludeLabels()) { + PathFragment dir = subincludeLabel.toPathFragment().getParentDirectory(); + PackageIdentifier dirId = new PackageIdentifier(pkgId.getRepository(), dir); + if (dir.equals(pkgId.getPackageFragment())) { + continue; + } + SkyKey key = ContainingPackageLookupValue.key(dirId); + subincludeToKey.put(subincludeLabel, key); + containingPkgLookupKeys.add(ContainingPackageLookupValue.key(dirId)); + } + Map<SkyKey, ValueOrException3<BuildFileNotFoundException, InconsistentFilesystemException, + FileSymlinkCycleException>> containingPkgLookupValues = env.getValuesOrThrow( + containingPkgLookupKeys, BuildFileNotFoundException.class, + InconsistentFilesystemException.class, FileSymlinkCycleException.class); + if (env.valuesMissing()) { + return; + } + for (Target target : ImmutableSet.copyOf(pkgBuilder.getTargets())) { + SkyKey key = targetToKey.get(target); + if (!containingPkgLookupValues.containsKey(key)) { + continue; + } + ContainingPackageLookupValue containingPackageLookupValue = + getContainingPkgLookupValueAndPropagateInconsistentFilesystemExceptions( + pkgId.getPackageFragment().getPathString(), containingPkgLookupValues.get(key), env); + if (maybeAddEventAboutLabelCrossingSubpackage(pkgBuilder, pkgRoot, target.getLabel(), + target.getLocation(), containingPackageLookupValue)) { + pkgBuilder.removeTarget(target); + pkgBuilder.setContainsErrors(); + } + } + for (Label subincludeLabel : pkgBuilder.getSubincludeLabels()) { + SkyKey key = subincludeToKey.get(subincludeLabel); + if (!containingPkgLookupValues.containsKey(key)) { + continue; + } + ContainingPackageLookupValue containingPackageLookupValue = + getContainingPkgLookupValueAndPropagateInconsistentFilesystemExceptions( + pkgId.getPackageFragment().getPathString(), containingPkgLookupValues.get(key), env); + if (maybeAddEventAboutLabelCrossingSubpackage(pkgBuilder, pkgRoot, subincludeLabel, + /*location=*/null, containingPackageLookupValue)) { + pkgBuilder.setContainsErrors(); + } + } + } + + @Nullable + private static ContainingPackageLookupValue + getContainingPkgLookupValueAndPropagateInconsistentFilesystemExceptions(String packageName, + ValueOrException3<BuildFileNotFoundException, InconsistentFilesystemException, + FileSymlinkCycleException> containingPkgLookupValueOrException, Environment env) + throws InternalInconsistentFilesystemException { + try { + return (ContainingPackageLookupValue) containingPkgLookupValueOrException.get(); + } catch (BuildFileNotFoundException | FileSymlinkCycleException e) { + env.getListener().handle(Event.error(null, e.getMessage())); + return null; + } catch (InconsistentFilesystemException e) { + throw new InternalInconsistentFilesystemException(packageName, e); + } + } + + private static boolean maybeAddEventAboutLabelCrossingSubpackage( + Package.LegacyBuilder pkgBuilder, Path pkgRoot, Label label, @Nullable Location location, + @Nullable ContainingPackageLookupValue containingPkgLookupValue) { + if (containingPkgLookupValue == null) { + return true; + } + if (!containingPkgLookupValue.hasContainingPackage()) { + // The missing package here is a problem, but it's not an error from the perspective of + // PackageFunction. + return false; + } + PackageIdentifier containingPkg = containingPkgLookupValue.getContainingPackageName(); + if (containingPkg.equals(label.getPackageIdentifier())) { + // The label does not cross a subpackage boundary. + return false; + } + if (!containingPkg.getPackageFragment().startsWith(label.getPackageFragment())) { + // This label is referencing an imaginary package, because the containing package should + // extend the label's package: if the label is //a/b:c/d, the containing package could be + // //a/b/c or //a/b, but should never be //a. Usually such errors will be caught earlier, but + // in some exceptional cases (such as a Python-aware BUILD file catching its own io + // exceptions), it reaches here, and we tolerate it. + return false; + } + PathFragment labelNameFragment = new PathFragment(label.getName()); + String message = String.format("Label '%s' crosses boundary of subpackage '%s'", + label, containingPkg); + Path containingRoot = containingPkgLookupValue.getContainingPackageRoot(); + if (pkgRoot.equals(containingRoot)) { + PathFragment labelNameInContainingPackage = labelNameFragment.subFragment( + containingPkg.getPackageFragment().segmentCount() + - label.getPackageFragment().segmentCount(), + labelNameFragment.segmentCount()); + message += " (perhaps you meant to put the colon here: " + + "'//" + containingPkg + ":" + labelNameInContainingPackage + "'?)"; + } else { + message += " (have you deleted " + containingPkg + "/BUILD? " + + "If so, use the --deleted_packages=" + containingPkg + " option)"; + } + pkgBuilder.addEvent(Event.error(location, message)); + return true; + } + + /** + * Constructs a {@link Package} object for the given package using legacy package loading. + * Note that the returned package may be in error. + */ + private Package.LegacyBuilder loadPackage(ParserInputSource inputSource, + @Nullable String replacementContents, + PackageIdentifier packageId, Path buildFilePath, RuleVisibility defaultVisibility, + List<Statement> preludeStatements, SkylarkImportResult importResult) + throws InterruptedException { + ParserInputSource replacementSource = replacementContents == null ? null + : ParserInputSource.create(replacementContents, buildFilePath); + Package.LegacyBuilder pkgBuilder = packageFunctionCache.get(packageId); + if (pkgBuilder == null) { + Clock clock = new JavaClock(); + long startTime = clock.nanoTime(); + profiler.startTask(ProfilerTask.CREATE_PACKAGE, packageId.toString()); + try { + Globber globber = packageFactory.createLegacyGlobber(buildFilePath.getParentDirectory(), + packageId, packageLocator); + StoredEventHandler localReporter = new StoredEventHandler(); + Preprocessor.Result preprocessingResult = replacementSource == null + ? packageFactory.preprocess(packageId, buildFilePath, inputSource, globber, + localReporter) + : Preprocessor.Result.noPreprocessing(replacementSource); + pkgBuilder = packageFactory.createPackageFromPreprocessingResult(packageId, buildFilePath, + preprocessingResult, localReporter.getEvents(), preludeStatements, + importResult.importMap, importResult.fileDependencies, packageLocator, + defaultVisibility, globber); + if (eventBus.get() != null) { + eventBus.get().post(new PackageLoadedEvent(packageId.toString(), + (clock.nanoTime() - startTime) / (1000 * 1000), + // It's impossible to tell if the package was loaded before, so we always pass false. + /*reloading=*/false, + // This isn't completely correct since we may encounter errors later (e.g. filesystem + // inconsistencies) + !pkgBuilder.containsErrors())); + } + numPackagesLoaded.incrementAndGet(); + packageFunctionCache.put(packageId, pkgBuilder); + } finally { + profiler.completeTask(ProfilerTask.CREATE_PACKAGE); + } + } + return pkgBuilder; + } + + private static class InternalInconsistentFilesystemException extends NoSuchPackageException { + private boolean isTransient; + + /** + * Used to represent a filesystem inconsistency discovered outside the + * {@link PackageFunction}. + */ + public InternalInconsistentFilesystemException(String packageName, + InconsistentFilesystemException e) { + super(packageName, e.getMessage(), e); + // This is not a transient error from the perspective of the PackageFunction. + this.isTransient = false; + } + + /** Used to represent a filesystem inconsistency discovered by the {@link PackageFunction}. */ + public InternalInconsistentFilesystemException(String packageName, + String inconsistencyMessage) { + this(packageName, new InconsistentFilesystemException(inconsistencyMessage)); + this.isTransient = true; + } + + public boolean isTransient() { + return isTransient; + } + } + + private static class BadWorkspaceFileException extends NoSuchPackageException { + private BadWorkspaceFileException(String message) { + super("external", "Error encountered while dealing with the WORKSPACE file: " + message); + } + } + + private static class BadPreludeFileException extends NoSuchPackageException { + private BadPreludeFileException(String packageName, String message) { + super(packageName, "Error encountered while reading the prelude file: " + message); + } + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link PackageFunction#compute}. + */ + private static class PackageFunctionException extends SkyFunctionException { + public PackageFunctionException(NoSuchPackageException e, Transience transience) { + super(e, transience); + } + } + + /** A simple value class to store the result of the Skylark imports.*/ + private static final class SkylarkImportResult { + private final Map<PathFragment, SkylarkEnvironment> importMap; + private final ImmutableList<Label> fileDependencies; + private SkylarkImportResult(Map<PathFragment, SkylarkEnvironment> importMap, + ImmutableList<Label> fileDependencies) { + this.importMap = importMap; + this.fileDependencies = fileDependencies; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupFunction.java new file mode 100644 index 0000000..ae4ee55 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupFunction.java
@@ -0,0 +1,180 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.cmdline.LabelValidator; +import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException; +import com.google.devtools.build.lib.packages.BuildFileNotFoundException; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.Nullable; + +/** + * SkyFunction for {@link PackageLookupValue}s. + */ +class PackageLookupFunction implements SkyFunction { + + private final AtomicReference<ImmutableSet<String>> deletedPackages; + + PackageLookupFunction(AtomicReference<ImmutableSet<String>> deletedPackages) { + this.deletedPackages = deletedPackages; + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws PackageLookupFunctionException { + PathPackageLocator pkgLocator = PrecomputedValue.PATH_PACKAGE_LOCATOR.get(env); + PackageIdentifier packageKey = (PackageIdentifier) skyKey.argument(); + if (!packageKey.getRepository().isDefault()) { + return computeExternalPackageLookupValue(skyKey, env); + } + PathFragment pkg = packageKey.getPackageFragment(); + + // This represents a package lookup at the package root. + if (pkg.equals(PathFragment.EMPTY_FRAGMENT)) { + return PackageLookupValue.invalidPackageName("The empty package name is invalid"); + } + + String pkgName = pkg.getPathString(); + String packageNameErrorMsg = LabelValidator.validatePackageName(pkgName); + if (packageNameErrorMsg != null) { + return PackageLookupValue.invalidPackageName("Invalid package name '" + pkgName + "': " + + packageNameErrorMsg); + } + + if (deletedPackages.get().contains(pkg.getPathString())) { + return PackageLookupValue.deletedPackage(); + } + + // TODO(bazel-team): The following is O(n^2) on the number of elements on the package path due + // to having restart the SkyFunction after every new dependency. However, if we try to batch + // the missing value keys, more dependencies than necessary will be declared. This wart can be + // fixed once we have nicer continuation support [skyframe-loading] + for (Path packagePathEntry : pkgLocator.getPathEntries()) { + PackageLookupValue value = getPackageLookupValue(env, packagePathEntry, pkg); + if (value == null || value.packageExists()) { + return value; + } + } + return PackageLookupValue.noBuildFile(); + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + private PackageLookupValue getPackageLookupValue(Environment env, Path packagePathEntry, + PathFragment pkgFragment) throws PackageLookupFunctionException { + PathFragment buildFileFragment; + if (pkgFragment.getPathString().equals(PackageFunction.EXTERNAL_PACKAGE_NAME)) { + buildFileFragment = new PathFragment("WORKSPACE"); + } else { + buildFileFragment = pkgFragment.getChild("BUILD"); + } + RootedPath buildFileRootedPath = RootedPath.toRootedPath(packagePathEntry, + buildFileFragment); + String basename = buildFileRootedPath.asPath().getBaseName(); + SkyKey fileSkyKey = FileValue.key(buildFileRootedPath); + FileValue fileValue = null; + try { + fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class, + FileSymlinkCycleException.class, InconsistentFilesystemException.class); + } catch (IOException e) { + String pkgName = pkgFragment.getPathString(); + // TODO(bazel-team): throw an IOException here and let PackageFunction wrap that into a + // BuildFileNotFoundException. + throw new PackageLookupFunctionException(new BuildFileNotFoundException(pkgName, + "IO errors while looking for " + basename + " file reading " + + buildFileRootedPath.asPath() + ": " + e.getMessage(), e), + Transience.PERSISTENT); + } catch (FileSymlinkCycleException e) { + String pkgName = buildFileRootedPath.asPath().getPathString(); + throw new PackageLookupFunctionException(new BuildFileNotFoundException(pkgName, + "Symlink cycle detected while trying to find " + basename + " file " + + buildFileRootedPath.asPath()), + Transience.PERSISTENT); + } catch (InconsistentFilesystemException e) { + // This error is not transient from the perspective of the PackageLookupFunction. + throw new PackageLookupFunctionException(e, Transience.PERSISTENT); + } + if (fileValue == null) { + return null; + } + if (fileValue.isFile()) { + return PackageLookupValue.success(buildFileRootedPath.getRoot()); + } + return PackageLookupValue.noBuildFile(); + } + + /** + * Gets a PackageLookupValue from a different Bazel repository. + * + * To do this, it looks up the "external" package and finds a path mapping for the repository + * name. + */ + private PackageLookupValue computeExternalPackageLookupValue( + SkyKey skyKey, Environment env) throws PackageLookupFunctionException { + PackageIdentifier id = (PackageIdentifier) skyKey.argument(); + SkyKey repositoryKey = RepositoryValue.key(id.getRepository()); + RepositoryValue repositoryValue = null; + try { + repositoryValue = (RepositoryValue) env.getValueOrThrow( + repositoryKey, NoSuchPackageException.class, IOException.class, EvalException.class); + if (repositoryValue == null) { + return null; + } + } catch (NoSuchPackageException e) { + throw new PackageLookupFunctionException(e, Transience.PERSISTENT); + } catch (IOException e) { + throw new PackageLookupFunctionException(new BuildFileContainsErrorsException( + PackageFunction.EXTERNAL_PACKAGE_NAME, e.getMessage()), Transience.PERSISTENT); + } catch (EvalException e) { + throw new PackageLookupFunctionException(new BuildFileContainsErrorsException( + PackageFunction.EXTERNAL_PACKAGE_NAME, e.getMessage()), Transience.PERSISTENT); + } + + return getPackageLookupValue(env, repositoryValue.getPath(), id.getPackageFragment()); + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link PackageLookupFunction#compute}. + */ + private static final class PackageLookupFunctionException extends SkyFunctionException { + public PackageLookupFunctionException(NoSuchPackageException e, Transience transience) { + super(e, transience); + } + + public PackageLookupFunctionException(InconsistentFilesystemException e, + Transience transience) { + super(e, transience); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupValue.java new file mode 100644 index 0000000..c877d38 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageLookupValue.java
@@ -0,0 +1,249 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * A value that represents a package lookup result. + * + * <p>Package lookups will always produce a value. On success, the {@code #getRoot} returns the + * package path root under which the package resides and the package's BUILD file is guaranteed to + * exist; on failure, {@code #getErrorReason} and {@code #getErrorMsg} describe why the package + * doesn't exist. + * + * <p>Implementation detail: we use inheritance here to optimize for memory usage. + */ +abstract class PackageLookupValue implements SkyValue { + + enum ErrorReason { + // There is no BUILD file. + NO_BUILD_FILE, + + // The package name is invalid. + INVALID_PACKAGE_NAME, + + // The package is considered deleted because of --deleted_packages. + DELETED_PACKAGE, + + // The //external package could not be loaded, either because the WORKSPACE file could not be + // parsed or the packages it references cannot be loaded. + NO_EXTERNAL_PACKAGE + } + + protected PackageLookupValue() { + } + + public static PackageLookupValue success(Path root) { + return new SuccessfulPackageLookupValue(root); + } + + public static PackageLookupValue noBuildFile() { + return NoBuildFilePackageLookupValue.INSTANCE; + } + + public static PackageLookupValue noExternalPackage() { + return NoExternalPackageLookupValue.INSTANCE; + } + + public static PackageLookupValue invalidPackageName(String errorMsg) { + return new InvalidNamePackageLookupValue(errorMsg); + } + + public static PackageLookupValue deletedPackage() { + return DeletedPackageLookupValue.INSTANCE; + } + + /** + * For a successful package lookup, returns the root (package path entry) that the package + * resides in. + */ + public abstract Path getRoot(); + + /** + * Returns whether the package lookup was successful. + */ + public abstract boolean packageExists(); + + /** + * For an unsuccessful package lookup, gets the reason why {@link #packageExists} returns + * {@code false}. + */ + abstract ErrorReason getErrorReason(); + + /** + * For an unsuccessful package lookup, gets a detailed error message for {@link #getErrorReason} + * that is suitable for reporting to a user. + */ + abstract String getErrorMsg(); + + static SkyKey key(PathFragment directory) { + Preconditions.checkArgument(!directory.isAbsolute(), directory); + return key(PackageIdentifier.createInDefaultRepo(directory)); + } + + static SkyKey key(PackageIdentifier pkgIdentifier) { + return new SkyKey(SkyFunctions.PACKAGE_LOOKUP, pkgIdentifier); + } + + private static class SuccessfulPackageLookupValue extends PackageLookupValue { + + private final Path root; + + private SuccessfulPackageLookupValue(Path root) { + this.root = root; + } + + @Override + public boolean packageExists() { + return true; + } + + @Override + public Path getRoot() { + return root; + } + + @Override + ErrorReason getErrorReason() { + throw new IllegalStateException(); + } + + @Override + String getErrorMsg() { + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof SuccessfulPackageLookupValue)) { + return false; + } + SuccessfulPackageLookupValue other = (SuccessfulPackageLookupValue) obj; + return root.equals(other.root); + } + + @Override + public int hashCode() { + return root.hashCode(); + } + } + + private abstract static class UnsuccessfulPackageLookupValue extends PackageLookupValue { + + @Override + public boolean packageExists() { + return false; + } + + @Override + public Path getRoot() { + throw new IllegalStateException(); + } + } + + private static class NoBuildFilePackageLookupValue extends UnsuccessfulPackageLookupValue { + + public static final NoBuildFilePackageLookupValue INSTANCE = + new NoBuildFilePackageLookupValue(); + + private NoBuildFilePackageLookupValue() { + } + + @Override + ErrorReason getErrorReason() { + return ErrorReason.NO_BUILD_FILE; + } + + @Override + String getErrorMsg() { + return "BUILD file not found on package path"; + } + } + + private static class NoExternalPackageLookupValue extends UnsuccessfulPackageLookupValue { + + public static final NoExternalPackageLookupValue INSTANCE = + new NoExternalPackageLookupValue(); + + private NoExternalPackageLookupValue() { + } + + @Override + ErrorReason getErrorReason() { + return ErrorReason.NO_EXTERNAL_PACKAGE; + } + + @Override + String getErrorMsg() { + return "Error loading the //external package"; + } + } + + private static class InvalidNamePackageLookupValue extends UnsuccessfulPackageLookupValue { + + private final String errorMsg; + + private InvalidNamePackageLookupValue(String errorMsg) { + this.errorMsg = errorMsg; + } + + @Override + ErrorReason getErrorReason() { + return ErrorReason.INVALID_PACKAGE_NAME; + } + + @Override + String getErrorMsg() { + return errorMsg; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof InvalidNamePackageLookupValue)) { + return false; + } + InvalidNamePackageLookupValue other = (InvalidNamePackageLookupValue) obj; + return errorMsg.equals(other.errorMsg); + } + + @Override + public int hashCode() { + return errorMsg.hashCode(); + } + } + + private static class DeletedPackageLookupValue extends UnsuccessfulPackageLookupValue { + + public static final DeletedPackageLookupValue INSTANCE = new DeletedPackageLookupValue(); + + private DeletedPackageLookupValue() { + } + + @Override + ErrorReason getErrorReason() { + return ErrorReason.DELETED_PACKAGE; + } + + @Override + String getErrorMsg() { + return "Package is considered deleted due to --deleted_packages"; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageValue.java new file mode 100644 index 0000000..65fd2af --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageValue.java
@@ -0,0 +1,55 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * A Skyframe value representing a package. + */ +@Immutable +@ThreadSafe +public class PackageValue implements SkyValue { + + private final Package pkg; + + PackageValue(Package pkg) { + this.pkg = Preconditions.checkNotNull(pkg); + } + + public Package getPackage() { + return pkg; + } + + @Override + public String toString() { + return "<PackageValue name=" + pkg.getName() + ">"; + } + + @ThreadSafe + public static SkyKey key(PathFragment pkgName) { + return key(PackageIdentifier.createInDefaultRepo(pkgName)); + } + + public static SkyKey key(PackageIdentifier pkgIdentifier) { + return new SkyKey(SkyFunctions.PACKAGE, pkgIdentifier); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java b/src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java new file mode 100644 index 0000000..5116a6f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java
@@ -0,0 +1,131 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.Dirent; +import com.google.devtools.build.lib.vfs.FileStatus; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.lib.vfs.UnixGlob; + +import java.io.IOException; +import java.util.Collection; + +/** + * A per-build cache of filesystem operations for Skyframe invocations of legacy package loading. + */ +class PerBuildSyscallCache implements UnixGlob.FilesystemCalls { + + private final LoadingCache<Pair<Path, Symlinks>, FileStatus> statCache = + newStatMap(); + private final LoadingCache<Pair<Path, Symlinks>, Pair<Collection<Dirent>, IOException>> + readdirCache = newReaddirMap(); + + private static final FileStatus NO_STATUS = new FakeFileStatus(); + + @Override + public Collection<Dirent> readdir(Path path, Symlinks symlinks) throws IOException { + Pair<Collection<Dirent>, IOException> result = + readdirCache.getUnchecked(Pair.of(path, symlinks)); + Collection<Dirent> entries = result.getFirst(); + if (entries != null) { + return entries; + } + throw result.getSecond(); + } + + @Override + public FileStatus statNullable(Path path, Symlinks symlinks) { + FileStatus status = statCache.getUnchecked(Pair.of(path, symlinks)); + return (status == NO_STATUS) ? null : status; + } + + // This is used because the cache implementations don't allow null. + private static final class FakeFileStatus implements FileStatus { + @Override + public long getLastChangeTime() { + throw new UnsupportedOperationException(); + } + + @Override + public long getNodeId() { + throw new UnsupportedOperationException(); + } + + @Override + public long getLastModifiedTime() { + throw new UnsupportedOperationException(); + } + + @Override + public long getSize() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isDirectory() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isFile() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isSymbolicLink() { + throw new UnsupportedOperationException(); + } + } + + /** + * A cache of stat calls. + * Input: (path, following_symlinks) + * Output: FileStatus + */ + private static LoadingCache<Pair<Path, Symlinks>, FileStatus> newStatMap() { + return CacheBuilder.newBuilder().build( + new CacheLoader<Pair<Path, Symlinks>, FileStatus>() { + @Override + public FileStatus load(Pair<Path, Symlinks> p) { + FileStatus f = p.first.statNullable(p.second); + return (f == null) ? NO_STATUS : f; + } + }); + } + + /** + * A cache of readdir calls. + * Input: (path, following_symlinks) + * Output: A union of (Dirents, IOException). + */ + private static + LoadingCache<Pair<Path, Symlinks>, Pair<Collection<Dirent>, IOException>> newReaddirMap() { + return CacheBuilder.newBuilder().build( + new CacheLoader<Pair<Path, Symlinks>, Pair<Collection<Dirent>, IOException>>() { + @Override + public Pair<Collection<Dirent>, IOException> load(Pair<Path, Symlinks> p) { + try { + return Pair.of(p.first.readdir(p.second), null); + } catch (IOException e) { + return Pair.of(null, e); + } + } + }); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetFunction.java new file mode 100644 index 0000000..03920b7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetFunction.java
@@ -0,0 +1,145 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency; +import com.google.devtools.build.lib.analysis.LabelAndConfiguration; +import com.google.devtools.build.lib.analysis.TargetAndConfiguration; +import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.RawAttributeMapper; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ConflictException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Build a post-processed ConfiguredTarget, vetting it for action conflict issues. + */ +public class PostConfiguredTargetFunction implements SkyFunction { + private static final Function<Dependency, SkyKey> TO_KEYS = + new Function<Dependency, SkyKey>() { + @Override + public SkyKey apply(Dependency input) { + return PostConfiguredTargetValue.key( + new ConfiguredTargetKey(input.getLabel(), input.getConfiguration())); + } + }; + + private final SkyframeExecutor.BuildViewProvider buildViewProvider; + + public PostConfiguredTargetFunction( + SkyframeExecutor.BuildViewProvider buildViewProvider) { + this.buildViewProvider = Preconditions.checkNotNull(buildViewProvider); + } + + @Nullable + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + ImmutableMap<Action, ConflictException> badActions = PrecomputedValue.BAD_ACTIONS.get(env); + ConfiguredTargetValue ctValue = (ConfiguredTargetValue) + env.getValue(ConfiguredTargetValue.key((ConfiguredTargetKey) skyKey.argument())); + SkyframeDependencyResolver resolver = + buildViewProvider.getSkyframeBuildView().createDependencyResolver(env); + if (env.valuesMissing()) { + return null; + } + + for (Action action : ctValue.getActions()) { + if (badActions.containsKey(action)) { + throw new ActionConflictFunctionException(badActions.get(action)); + } + } + + ConfiguredTarget ct = ctValue.getConfiguredTarget(); + TargetAndConfiguration ctgValue = + new TargetAndConfiguration(ct.getTarget(), ct.getConfiguration()); + + Set<ConfigMatchingProvider> configConditions = + getConfigurableAttributeConditions(ctgValue, env); + if (configConditions == null) { + return null; + } + + Collection<Dependency> deps = resolver.dependentNodes(ctgValue, configConditions); + env.getValues(Iterables.transform(deps, TO_KEYS)); + if (env.valuesMissing()) { + return null; + } + + return new PostConfiguredTargetValue(ct); + } + + /** + * Returns the configurable attribute conditions necessary to evaluate the given configured + * target, or null if not all dependencies have yet been SkyFrame-evaluated. + */ + @Nullable + private Set<ConfigMatchingProvider> getConfigurableAttributeConditions( + TargetAndConfiguration ctg, Environment env) { + if (!(ctg.getTarget() instanceof Rule)) { + return ImmutableSet.of(); + } + Rule rule = (Rule) ctg.getTarget(); + RawAttributeMapper mapper = RawAttributeMapper.of(rule); + Set<SkyKey> depKeys = new LinkedHashSet<>(); + for (Attribute attribute : rule.getAttributes()) { + for (Label label : mapper.getConfigurabilityKeys(attribute.getName(), attribute.getType())) { + if (!Type.Selector.isReservedLabel(label)) { + depKeys.add(ConfiguredTargetValue.key(label, ctg.getConfiguration())); + } + } + } + Map<SkyKey, SkyValue> cts = env.getValues(depKeys); + if (env.valuesMissing()) { + return null; + } + ImmutableSet.Builder<ConfigMatchingProvider> conditions = ImmutableSet.builder(); + for (SkyValue ctValue : cts.values()) { + ConfiguredTarget ct = ((ConfiguredTargetValue) ctValue).getConfiguredTarget(); + conditions.add(Preconditions.checkNotNull(ct.getProvider(ConfigMatchingProvider.class))); + } + return conditions.build(); + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return Label.print(((LabelAndConfiguration) skyKey.argument()).getLabel()); + } + + private static class ActionConflictFunctionException extends SkyFunctionException { + public ActionConflictFunctionException(ConflictException e) { + super(e, Transience.PERSISTENT); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetValue.java new file mode 100644 index 0000000..42d2b38 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/PostConfiguredTargetValue.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * A post-processed ConfiguredTarget which is known to be transitively error-free from action + * conflict issues. + */ +class PostConfiguredTargetValue implements SkyValue { + + private final ConfiguredTarget ct; + + public PostConfiguredTargetValue(ConfiguredTarget ct) { + this.ct = Preconditions.checkNotNull(ct); + } + + public static ImmutableList<SkyKey> keys(Iterable<ConfiguredTargetKey> lacs) { + ImmutableList.Builder<SkyKey> keys = ImmutableList.builder(); + for (ConfiguredTargetKey lac : lacs) { + keys.add(key(lac)); + } + return keys.build(); + } + + public static SkyKey key(ConfiguredTargetKey lac) { + return new SkyKey(SkyFunctions.POST_CONFIGURED_TARGET, lac); + } + + public ConfiguredTarget getCt() { + return ct; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedFunction.java new file mode 100644 index 0000000..8254987 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedFunction.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * Builder for {@link PrecomputedValue}s. + * + * <p>Always throws an error, because the values aren't computed inside the skyframe framework. + */ +public class PrecomputedFunction implements SkyFunction { + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, + InterruptedException { + throw new IllegalStateException(skyKey + " not set"); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java new file mode 100644 index 0000000..bb2656d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java
@@ -0,0 +1,182 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.TopLevelArtifactContext; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey; +import com.google.devtools.build.lib.packages.RuleVisibility; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ConflictException; +import com.google.devtools.build.skyframe.Injectable; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.Map; +import java.util.UUID; + +import javax.annotation.Nullable; + +/** + * A value that represents something computed outside of the skyframe framework. These values are + * "precomputed" from skyframe's perspective and so the graph needs to be prepopulated with them + * (e.g. via injection). + */ +public class PrecomputedValue implements SkyValue { + /** + * An externally-injected precomputed value. Exists so that modules can inject precomputed values + * into Skyframe's graph. + * + * <p>{@see com.google.devtools.build.lib.blaze.BlazeModule#getPrecomputedValues}. + */ + public static final class Injected { + private final Precomputed<?> precomputed; + private final Supplier<? extends Object> supplier; + + private Injected(Precomputed<?> precomputed, Supplier<? extends Object> supplier) { + this.precomputed = precomputed; + this.supplier = supplier; + } + + void inject(Injectable injectable) { + injectable.inject(ImmutableMap.of(precomputed.key, new PrecomputedValue(supplier.get()))); + } + } + + public static <T> Injected injected(Precomputed<T> precomputed, Supplier<T> value) { + return new Injected(precomputed, value); + } + + public static <T> Injected injected(Precomputed<T> precomputed, T value) { + return new Injected(precomputed, Suppliers.ofInstance(value)); + } + + static final Precomputed<String> DEFAULTS_PACKAGE_CONTENTS = + new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "default_pkg")); + + static final Precomputed<RuleVisibility> DEFAULT_VISIBILITY = + new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "default_visibility")); + + static final Precomputed<UUID> BUILD_ID = + new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "build_id")); + + static final Precomputed<WorkspaceStatusAction> WORKSPACE_STATUS_KEY = + new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "workspace_status_action")); + + static final Precomputed<Action> COVERAGE_REPORT_KEY = + new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "coverage_report_action")); + + static final Precomputed<TopLevelArtifactContext> TOP_LEVEL_CONTEXT = + new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "top_level_context")); + + static final Precomputed<Map<BuildInfoKey, BuildInfoFactory>> BUILD_INFO_FACTORIES = + new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "build_info_factories")); + + static final Precomputed<Map<String, String>> TEST_ENVIRONMENT_VARIABLES = + new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "test_environment")); + + static final Precomputed<BlazeDirectories> BLAZE_DIRECTORIES = + new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "blaze_directories")); + + static final Precomputed<ImmutableMap<Action, ConflictException>> BAD_ACTIONS = + new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "bad_actions")); + + public static final Precomputed<PathPackageLocator> PATH_PACKAGE_LOCATOR = + new Precomputed<>(new SkyKey(SkyFunctions.PRECOMPUTED, "path_package_locator")); + + private final Object value; + + public PrecomputedValue(Object value) { + this.value = Preconditions.checkNotNull(value); + } + + /** + * Returns the value of the variable. + */ + public Object get() { + return value; + } + + @Override + public int hashCode() { + return value.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof PrecomputedValue)) { + return false; + } + PrecomputedValue other = (PrecomputedValue) obj; + return value.equals(other.value); + } + + @Override + public String toString() { + return "<BuildVariable " + value + ">"; + } + + public static final void dependOnBuildId(SkyFunction.Environment env) { + BUILD_ID.get(env); + } + + /** + * A helper object corresponding to a variable in Skyframe. + * + * <p>Instances do not have internal state. + */ + public static final class Precomputed<T> { + private final SkyKey key; + + public Precomputed(SkyKey key) { + this.key = key; + } + + @VisibleForTesting + SkyKey getKeyForTesting() { + return key; + } + + /** + * Retrieves the value of this variable from Skyframe. + * + * <p>If the value was not set, an exception will be raised. + */ + @Nullable + @SuppressWarnings("unchecked") + public T get(SkyFunction.Environment env) { + PrecomputedValue value = (PrecomputedValue) env.getValue(key); + if (value == null) { + return null; + } + return (T) value.get(); + } + + /** + * Injects a new variable value. + */ + void set(Injectable injectable, T value) { + injectable.inject(ImmutableMap.of(key, new PrecomputedValue(value))); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java new file mode 100644 index 0000000..d54e98f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunction.java
@@ -0,0 +1,454 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Verify; +import com.google.common.collect.Collections2; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile; +import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.TraversalRequest; +import com.google.devtools.build.lib.vfs.Dirent; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** A {@link SkyFunction} to build {@link RecursiveFilesystemTraversalValue}s. */ +public final class RecursiveFilesystemTraversalFunction implements SkyFunction { + + private static final class MissingDepException extends Exception {} + + /** Base class for exceptions that {@link RecursiveFilesystemTraversalFunctionException} wraps. */ + public abstract static class RecursiveFilesystemTraversalException extends Exception { + protected RecursiveFilesystemTraversalException(String message) { + super(message); + } + } + + /** Thrown when a generated directory's root-relative path conflicts with a package's path. */ + public static final class GeneratedPathConflictException extends + RecursiveFilesystemTraversalException { + GeneratedPathConflictException(TraversalRequest traversal) { + super(String.format( + "Generated directory %s conflicts with package under the same path. Additional info: %s", + traversal.path.getRelativePath().getPathString(), + traversal.errorInfo != null ? traversal.errorInfo : traversal.toString())); + } + } + + /** + * Thrown when the traversal encounters a subdirectory with a BUILD file but is not allowed to + * recurse into it. + */ + public static final class CannotCrossPackageBoundaryException extends + RecursiveFilesystemTraversalException { + CannotCrossPackageBoundaryException(String message) { + super(message); + } + } + + /** + * Thrown when a dangling symlink is attempted to be dereferenced. + * + * <p>Note: this class is not identical to the one in com.google.devtools.build.lib.view.fileset + * and it's not easy to merge the two because of the dependency structure. The other one will + * probably be removed along with the rest of the legacy Fileset code. + */ + public static final class DanglingSymlinkException extends RecursiveFilesystemTraversalException { + public final String path; + public final String unresolvedLink; + + public DanglingSymlinkException(String path, String unresolvedLink) { + super("Found dangling symlink: " + path + ", unresolved path: "); + Preconditions.checkArgument(path != null && !path.isEmpty()); + Preconditions.checkArgument(unresolvedLink != null && !unresolvedLink.isEmpty()); + this.path = path; + this.unresolvedLink = unresolvedLink; + } + + public String getPath() { + return path; + } + } + + /** Exception type thrown by {@link RecursiveFilesystemTraversalFunction#compute}. */ + private static final class RecursiveFilesystemTraversalFunctionException extends + SkyFunctionException { + RecursiveFilesystemTraversalFunctionException(RecursiveFilesystemTraversalException e) { + super(e, Transience.PERSISTENT); + } + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) + throws RecursiveFilesystemTraversalFunctionException { + TraversalRequest traversal = (TraversalRequest) skyKey.argument(); + try { + // Stat the traversal root. + FileInfo rootInfo = lookUpFileInfo(env, traversal); + + if (!rootInfo.type.exists()) { + // May be a dangling symlink or a non-existent file. Handle gracefully. + if (rootInfo.type.isSymlink()) { + return resultForDanglingSymlink(traversal.path, rootInfo); + } else { + return RecursiveFilesystemTraversalValue.EMPTY; + } + } + + if (rootInfo.type.isFile()) { + // The root is a file or a symlink to one. + return resultForFileRoot(traversal.path, rootInfo); + } + + // Otherwise the root is a directory or a symlink to one. + PkgLookupResult pkgLookupResult = checkIfPackage(env, traversal, rootInfo); + traversal = pkgLookupResult.traversal; + + if (pkgLookupResult.isConflicting()) { + // The traversal was requested for an output directory whose root-relative path conflicts + // with a source package. We can't handle that, bail out. + throw new RecursiveFilesystemTraversalFunctionException( + new GeneratedPathConflictException(traversal)); + } else if (pkgLookupResult.isPackage() && !traversal.skipTestingForSubpackage) { + // The traversal was requested for a directory that defines a package. + if (traversal.crossPkgBoundaries) { + // We are free to traverse the subpackage but we need to display a warning. + String msg = traversal.errorInfo + " crosses package boundary into package rooted at " + + traversal.path.getRelativePath().getPathString(); + env.getListener().handle(new Event(EventKind.WARNING, null, msg)); + } else { + // We cannot traverse the subpackage and should skip it silently. Return empty results. + return RecursiveFilesystemTraversalValue.EMPTY; + } + } + + // We are free to traverse this directory. + Collection<SkyKey> dependentKeys = createRecursiveTraversalKeys(env, traversal); + return resultForDirectory(traversal, rootInfo, traverseChildren(env, dependentKeys)); + } catch (MissingDepException e) { + return null; + } + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + private static final class FileInfo { + final FileType type; + final FileStateValue metadata; + @Nullable final RootedPath realPath; + @Nullable final PathFragment unresolvedSymlinkTarget; + + FileInfo(FileType type, FileStateValue metadata, @Nullable RootedPath realPath, + @Nullable PathFragment unresolvedSymlinkTarget) { + this.type = Preconditions.checkNotNull(type); + this.metadata = Preconditions.checkNotNull(metadata); + this.realPath = realPath; + this.unresolvedSymlinkTarget = unresolvedSymlinkTarget; + } + + @Override + public String toString() { + if (type.isSymlink()) { + return String.format("(%s: link_value=%s, real_path=%s)", type, + unresolvedSymlinkTarget.getPathString(), realPath); + } else { + return String.format("(%s: real_path=%s)", type, realPath); + } + } + } + + private static FileInfo lookUpFileInfo(Environment env, TraversalRequest traversal) + throws MissingDepException { + // Stat the file. + FileValue fileValue = (FileValue) getDependentSkyValue(env, FileValue.key(traversal.path)); + if (fileValue.exists()) { + // If it exists, it may either be a symlink or a file/directory. + PathFragment unresolvedLinkTarget = null; + FileType type = null; + if (fileValue.isSymlink()) { + unresolvedLinkTarget = fileValue.getUnresolvedLinkTarget(); + type = fileValue.isDirectory() ? FileType.SYMLINK_TO_DIRECTORY : FileType.SYMLINK_TO_FILE; + } else { + type = fileValue.isDirectory() ? FileType.DIRECTORY : FileType.FILE; + } + return new FileInfo(type, fileValue.realFileStateValue(), + fileValue.realRootedPath(), unresolvedLinkTarget); + } else { + // If it doesn't exist, or it's a dangling symlink, we still want to handle that gracefully. + return new FileInfo( + fileValue.isSymlink() ? FileType.DANGLING_SYMLINK : FileType.NONEXISTENT, + fileValue.realFileStateValue(), null, + fileValue.isSymlink() ? fileValue.getUnresolvedLinkTarget() : null); + } + } + + private static final class PkgLookupResult { + private enum Type { + CONFLICT, DIRECTORY, PKG + } + + private final Type type; + final TraversalRequest traversal; + final FileInfo rootInfo; + + /** Result for a generated directory that conflicts with a source package. */ + static PkgLookupResult conflict(TraversalRequest traversal, FileInfo rootInfo) { + return new PkgLookupResult(Type.CONFLICT, traversal, rootInfo); + } + + /** Result for a source or generated directory (not a package). */ + static PkgLookupResult directory(TraversalRequest traversal, FileInfo rootInfo) { + return new PkgLookupResult(Type.DIRECTORY, traversal, rootInfo); + } + + /** Result for a package, i.e. a directory with a BUILD file. */ + static PkgLookupResult pkg(TraversalRequest traversal, FileInfo rootInfo) { + return new PkgLookupResult(Type.PKG, traversal, rootInfo); + } + + private PkgLookupResult(Type type, TraversalRequest traversal, FileInfo rootInfo) { + this.type = Preconditions.checkNotNull(type); + this.traversal = Preconditions.checkNotNull(traversal); + this.rootInfo = Preconditions.checkNotNull(rootInfo); + } + + boolean isPackage() { + return type == Type.PKG; + } + + boolean isConflicting() { + return type == Type.CONFLICT; + } + + @Override + public String toString() { + return String.format("(%s: info=%s, traversal=%s)", type, rootInfo, traversal); + } + } + + /** + * Checks whether the {@code traversal}'s path refers to a package directory. + * + * @return the result of the lookup; it contains potentially new {@link TraversalRequest} and + * {@link FileInfo} so the caller should use these instead of the old ones (this happens when + * a package is found, but under a different root than expected) + */ + private static PkgLookupResult checkIfPackage(Environment env, TraversalRequest traversal, + FileInfo rootInfo) throws MissingDepException { + Preconditions.checkArgument(rootInfo.type.exists() && !rootInfo.type.isFile(), + "{%s} {%s}", traversal, rootInfo); + PackageLookupValue pkgLookup = (PackageLookupValue) getDependentSkyValue(env, + PackageLookupValue.key(traversal.path.getRelativePath())); + + if (pkgLookup.packageExists()) { + if (traversal.isGenerated) { + // The traversal's root was a generated directory, but its root-relative path conflicts with + // an existing package. + return PkgLookupResult.conflict(traversal, rootInfo); + } else { + // The traversal's root was a source directory and it defines a package. + Path pkgRoot = pkgLookup.getRoot(); + if (!pkgRoot.equals(traversal.path.getRoot())) { + // However the root of this package is different from what we expected. stat() the real + // BUILD file of that package. + traversal = traversal.forChangedRootPath(pkgRoot); + rootInfo = lookUpFileInfo(env, traversal); + Verify.verify(rootInfo.type.exists(), "{%s} {%s}", traversal, rootInfo); + } + return PkgLookupResult.pkg(traversal, rootInfo); + } + } else { + // The traversal's root was a directory (source or generated one), no package exists under the + // same root-relative path. + return PkgLookupResult.directory(traversal, rootInfo); + } + } + + /** + * List the directory and create {@code SkyKey}s to request contents of its children recursively. + * + * <p>The returned keys are of type {@link SkyFunctions#RECURSIVE_FILESYSTEM_TRAVERSAL}. + */ + private static Collection<SkyKey> createRecursiveTraversalKeys(Environment env, + TraversalRequest traversal) throws MissingDepException { + // Use the traversal's path, even if it's a symlink. The contents of the directory, as listed + // in the result, must be relative to it. + DirectoryListingValue dirListing = (DirectoryListingValue) getDependentSkyValue(env, + DirectoryListingValue.key(traversal.path)); + + List<SkyKey> result = new ArrayList<>(); + for (Dirent dirent : dirListing.getDirents()) { + RootedPath childPath = RootedPath.toRootedPath(traversal.path.getRoot(), + traversal.path.getRelativePath().getRelative(dirent.getName())); + TraversalRequest childTraversal = traversal.forChildEntry(childPath); + result.add(RecursiveFilesystemTraversalValue.key(childTraversal)); + } + return result; + } + + /** + * Creates result for a dangling symlink. + * + * @param linkName path to the symbolic link + * @param info the {@link FileInfo} associated with the link file + */ + private static RecursiveFilesystemTraversalValue resultForDanglingSymlink(RootedPath linkName, + FileInfo info) { + Preconditions.checkState(info.type.isSymlink() && !info.type.exists(), "{%s} {%s}", linkName, + info.type); + return RecursiveFilesystemTraversalValue.of( + ResolvedFile.danglingSymlink(linkName, info.unresolvedSymlinkTarget, info.metadata)); + } + + /** + * Creates results for a file or for a symlink that points to one. + * + * <p>A symlink may be direct (points to a file) or transitive (points at a direct or transitive + * symlink). + */ + private static RecursiveFilesystemTraversalValue resultForFileRoot(RootedPath path, + FileInfo info) { + Preconditions.checkState(info.type.isFile() && info.type.exists(), "{%s} {%s}", path, + info.type); + if (info.type.isSymlink()) { + return RecursiveFilesystemTraversalValue.of(ResolvedFile.symlinkToFile(info.realPath, path, + info.unresolvedSymlinkTarget, info.metadata)); + } else { + return RecursiveFilesystemTraversalValue.of(ResolvedFile.regularFile(path, info.metadata)); + } + } + + private static RecursiveFilesystemTraversalValue resultForDirectory(TraversalRequest traversal, + FileInfo rootInfo, Collection<RecursiveFilesystemTraversalValue> subdirTraversals) { + // Collect transitive closure of files in subdirectories. + NestedSetBuilder<ResolvedFile> paths = NestedSetBuilder.stableOrder(); + for (RecursiveFilesystemTraversalValue child : subdirTraversals) { + paths.addTransitive(child.getTransitiveFiles()); + } + ResolvedFile root; + if (rootInfo.type.isSymlink()) { + root = ResolvedFile.symlinkToDirectory(rootInfo.realPath, traversal.path, + rootInfo.unresolvedSymlinkTarget, rootInfo.metadata); + paths.add(root); + } else { + root = ResolvedFile.directory(rootInfo.realPath); + } + return RecursiveFilesystemTraversalValue.of(root, paths.build()); + } + + private static SkyValue getDependentSkyValue(Environment env, SkyKey key) + throws MissingDepException { + SkyValue value = env.getValue(key); + if (env.valuesMissing()) { + throw new MissingDepException(); + } + return value; + } + + /** + * Requests Skyframe to compute the dependent values and returns them. + * + * <p>The keys must all be {@link SkyFunctions#RECURSIVE_FILESYSTEM_TRAVERSAL} keys. + */ + private static Collection<RecursiveFilesystemTraversalValue> traverseChildren( + Environment env, Iterable<SkyKey> keys) + throws MissingDepException { + Map<SkyKey, SkyValue> values = env.getValues(keys); + if (env.valuesMissing()) { + throw new MissingDepException(); + } + return Collections2.transform(values.values(), + new Function<SkyValue, RecursiveFilesystemTraversalValue>() { + @Override + public RecursiveFilesystemTraversalValue apply(SkyValue input) { + return (RecursiveFilesystemTraversalValue) input; + } + }); + } + + /** Type information about the filesystem entry residing at a path. */ + enum FileType { + /** A regular file. */ + FILE { + @Override boolean isFile() { return true; } + @Override boolean exists() { return true; } + @Override public String toString() { return "<f>"; } + }, + /** + * A symlink to a regular file. + * + * <p>The symlink may be direct (points to a non-symlink (here a file)) or it may be transitive + * (points to a direct or transitive symlink). + */ + SYMLINK_TO_FILE { + @Override boolean isFile() { return true; } + @Override boolean isSymlink() { return true; } + @Override boolean exists() { return true; } + @Override public String toString() { return "<lf>"; } + }, + /** A directory. */ + DIRECTORY { + @Override boolean isDirectory() { return true; } + @Override boolean exists() { return true; } + @Override public String toString() { return "<d>"; } + }, + /** + * A symlink to a directory. + * + * <p>The symlink may be direct (points to a non-symlink (here a directory)) or it may be + * transitive (points to a direct or transitive symlink). + */ + SYMLINK_TO_DIRECTORY { + @Override boolean isDirectory() { return true; } + @Override boolean isSymlink() { return true; } + @Override boolean exists() { return true; } + @Override public String toString() { return "<ld>"; } + }, + /** A dangling symlink, i.e. one whose target is known not to exist. */ + DANGLING_SYMLINK { + @Override boolean isFile() { throw new UnsupportedOperationException(); } + @Override boolean isDirectory() { throw new UnsupportedOperationException(); } + @Override boolean isSymlink() { return true; } + @Override public String toString() { return "<l?>"; } + }, + /** A path that does not exist or should be ignored. */ + NONEXISTENT { + @Override public String toString() { return "<?>"; } + }; + + boolean isFile() { return false; } + boolean isDirectory() { return false; } + boolean isSymlink() { return false; } + boolean exists() { return false; } + @Override public abstract String toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalValue.java new file mode 100644 index 0000000..023b1cf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalValue.java
@@ -0,0 +1,597 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Objects; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalFunction.DanglingSymlinkException; +import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalFunction.FileType; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import javax.annotation.Nullable; + +/** + * Collection of files found while recursively traversing a path. + * + * <p>The path may refer to files, symlinks or directories that may or may not exist. + * + * <p>Traversing a file or a symlink results in a single {@link ResolvedFile} corresponding to the + * file or symlink. + * + * <p>Traversing a directory results in a collection of {@link ResolvedFile}s for all files and + * symlinks under it, and in all of its subdirectories. The {@link TraversalRequest} can specify + * whether to traverse source subdirectories that are packages (have BUILD files in them). + * + * <p>Traversing a symlink that points to a directory is the same as traversing a normal directory. + * The paths in the result will not be resolved; the files will be listed under the symlink, as if + * it was the actual directory they reside in. + * + * <p>Editing a file that is part of this traversal, or adding or removing a file in a directory + * that is part of this traversal, will invalidate this {@link SkyValue}. This also applies to + * directories that are symlinked to. + */ +public final class RecursiveFilesystemTraversalValue implements SkyValue { + static final RecursiveFilesystemTraversalValue EMPTY = new RecursiveFilesystemTraversalValue( + Optional.<ResolvedFile>absent(), + NestedSetBuilder.<ResolvedFile>emptySet(Order.STABLE_ORDER)); + + /** The root of the traversal. May only be absent for the {@link #EMPTY} instance. */ + private final Optional<ResolvedFile> resolvedRoot; + + /** The transitive closure of {@link ResolvedFile}s. */ + private final NestedSet<ResolvedFile> resolvedPaths; + + private RecursiveFilesystemTraversalValue(Optional<ResolvedFile> resolvedRoot, + NestedSet<ResolvedFile> resolvedPaths) { + this.resolvedRoot = Preconditions.checkNotNull(resolvedRoot); + this.resolvedPaths = Preconditions.checkNotNull(resolvedPaths); + } + + static RecursiveFilesystemTraversalValue of(ResolvedFile resolvedRoot, + NestedSet<ResolvedFile> resolvedPaths) { + if (resolvedPaths.isEmpty()) { + return EMPTY; + } else { + return new RecursiveFilesystemTraversalValue(Optional.of(resolvedRoot), resolvedPaths); + } + } + + static RecursiveFilesystemTraversalValue of(ResolvedFile singleMember) { + return new RecursiveFilesystemTraversalValue(Optional.of(singleMember), + NestedSetBuilder.<ResolvedFile>create(Order.STABLE_ORDER, singleMember)); + } + + /** Returns the root of the traversal; absent only for the {@link #EMPTY} instance. */ + public Optional<ResolvedFile> getResolvedRoot() { + return resolvedRoot; + } + + /** + * Retrieves the set of {@link ResolvedFile}s that were found by this traversal. + * + * <p>The returned set may be empty if no files were found, or the ones found were to be + * considered non-existent. Unless it's empty, the returned set always includes the + * {@link #getResolvedRoot() resolved root}. + * + * <p>The returned set also includes symlinks. If a symlink points to a directory, its contents + * are also included in this set, and their path will start with the symlink's path, just like on + * a usual Unix file system. + */ + public NestedSet<ResolvedFile> getTransitiveFiles() { + return resolvedPaths; + } + + public static SkyKey key(TraversalRequest traversal) { + return new SkyKey(SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL, traversal); + } + + /** The parameters of a file or directory traversal. */ + public static final class TraversalRequest { + + /** The path to start the traversal from; may be a file, a directory or a symlink. */ + final RootedPath path; + + /** + * Whether the path is in the output tree. + * + * <p>Such paths and all their subdirectories are assumed not to define packages, so package + * lookup for them is skipped. + */ + final boolean isGenerated; + + /** Whether traversal should descend into directories that are roots of subpackages. */ + final boolean crossPkgBoundaries; + + /** + * Whether to skip checking if the root (if it's a directory) contains a BUILD file. + * + * <p>Such directories are not considered to be packages when this flag is true. This needs to + * be true in order to traverse directories of packages, but should be false for <i>their</i> + * subdirectories. + */ + final boolean skipTestingForSubpackage; + + /** Information to be attached to any error messages that may be reported. */ + @Nullable final String errorInfo; + + public TraversalRequest(RootedPath path, boolean isRootGenerated, + boolean crossPkgBoundaries, boolean skipTestingForSubpackage, + @Nullable String errorInfo) { + this.path = path; + this.isGenerated = isRootGenerated; + this.crossPkgBoundaries = crossPkgBoundaries; + this.skipTestingForSubpackage = skipTestingForSubpackage; + this.errorInfo = errorInfo; + } + + private TraversalRequest duplicate(RootedPath newRoot, boolean newSkipTestingForSubpackage) { + return new TraversalRequest(newRoot, isGenerated, crossPkgBoundaries, + newSkipTestingForSubpackage, errorInfo); + } + + /** Creates a new request to traverse a child element in the current directory (the root). */ + TraversalRequest forChildEntry(RootedPath newPath) { + return duplicate(newPath, false); + } + + /** + * Creates a new request for a changed root. + * + * <p>This method can be used when a package is found out to be under a different root path than + * originally assumed. + */ + TraversalRequest forChangedRootPath(Path newRoot) { + return duplicate(RootedPath.toRootedPath(newRoot, path.getRelativePath()), + skipTestingForSubpackage); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof TraversalRequest)) { + return false; + } + TraversalRequest o = (TraversalRequest) obj; + return path.equals(o.path) && isGenerated == o.isGenerated + && crossPkgBoundaries == o.crossPkgBoundaries + && skipTestingForSubpackage == o.skipTestingForSubpackage; + } + + @Override + public int hashCode() { + return Objects.hashCode(path, isGenerated, crossPkgBoundaries, skipTestingForSubpackage); + } + + @Override + public String toString() { + return String.format( + "TraversalParams(root=%s, is_generated=%d, skip_testing_for_subpkg=%d," + + " pkg_boundaries=%d)", path, isGenerated ? 1 : 0, skipTestingForSubpackage ? 1 : 0, + crossPkgBoundaries ? 1 : 0); + } + } + + /** + * Path and type information about a single file or symlink. + * + * <p>The object stores things such as the absolute path of the file or symlink, its exact type + * and, if it's a symlink, the resolved and unresolved link target paths. + */ + public abstract static class ResolvedFile { + private static final class Symlink { + private final RootedPath linkName; + private final PathFragment unresolvedLinkTarget; + // The resolved link target is stored in ResolvedFile.path + + private Symlink(RootedPath linkName, PathFragment unresolvedLinkTarget) { + this.linkName = Preconditions.checkNotNull(linkName); + this.unresolvedLinkTarget = Preconditions.checkNotNull(unresolvedLinkTarget); + } + + PathFragment getNameInSymlinkTree() { + return linkName.getRelativePath(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof Symlink)) { + return false; + } + Symlink o = (Symlink) obj; + return linkName.equals(o.linkName) && unresolvedLinkTarget.equals(o.unresolvedLinkTarget); + } + + @Override + public int hashCode() { + return Objects.hashCode(linkName, unresolvedLinkTarget); + } + + @Override + public String toString() { + return String.format("Symlink(link_name=%s, unresolved_target=%s)", + linkName, unresolvedLinkTarget); + } + } + + private static final class RegularFile extends ResolvedFile { + private RegularFile(RootedPath path) { + super(FileType.FILE, Optional.of(path), Optional.<FileStateValue>absent()); + } + + RegularFile(RootedPath path, FileStateValue metadata) { + super(FileType.FILE, Optional.of(path), Optional.of(metadata)); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof RegularFile)) { + return false; + } + return super.isEqualTo((RegularFile) obj); + } + + @Override + public String toString() { + return String.format("RegularFile(%s)", super.toString()); + } + + @Override + ResolvedFile stripMetadataForTesting() { + return new RegularFile(path.get()); + } + + @Override + public PathFragment getNameInSymlinkTree() { + return path.get().getRelativePath(); + } + + @Override + public PathFragment getTargetInSymlinkTree(boolean followSymlinks) { + return path.get().asPath().asFragment(); + } + } + + private static final class Directory extends ResolvedFile { + Directory(RootedPath path) { + super(FileType.DIRECTORY, Optional.of(path), Optional.of( + FileStateValue.DIRECTORY_FILE_STATE_NODE)); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof Directory)) { + return false; + } + return super.isEqualTo((Directory) obj); + } + + @Override + public String toString() { + return String.format("Directory(%s)", super.toString()); + } + + @Override + ResolvedFile stripMetadataForTesting() { + return this; + } + + @Override + public PathFragment getNameInSymlinkTree() { + return path.get().getRelativePath(); + } + + @Override + public PathFragment getTargetInSymlinkTree(boolean followSymlinks) { + return path.get().asPath().asFragment(); + } + } + + private static final class DanglingSymlink extends ResolvedFile { + private final Symlink symlink; + + private DanglingSymlink(Symlink symlink) { + super(FileType.DANGLING_SYMLINK, Optional.<RootedPath>absent(), + Optional.<FileStateValue>absent()); + this.symlink = symlink; + } + + DanglingSymlink(RootedPath linkNamePath, PathFragment linkTargetPath, + FileStateValue metadata) { + super(FileType.DANGLING_SYMLINK, Optional.<RootedPath>absent(), Optional.of(metadata)); + this.symlink = new Symlink(linkNamePath, linkTargetPath); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof DanglingSymlink)) { + return false; + } + DanglingSymlink o = (DanglingSymlink) obj; + return super.isEqualTo(o) && symlink.equals(o.symlink); + } + + @Override + public int hashCode() { + return Objects.hashCode(super.hashCode(), symlink); + } + + @Override + public String toString() { + return String.format("DanglingSymlink(%s, %s)", super.toString(), symlink); + } + + @Override + ResolvedFile stripMetadataForTesting() { + return new DanglingSymlink(symlink); + } + + @Override + public PathFragment getNameInSymlinkTree() { + return symlink.getNameInSymlinkTree(); + } + + @Override + public PathFragment getTargetInSymlinkTree(boolean followSymlinks) + throws DanglingSymlinkException { + if (followSymlinks) { + throw new DanglingSymlinkException(symlink.linkName.asPath().getPathString(), + symlink.unresolvedLinkTarget.getPathString()); + } else { + return symlink.unresolvedLinkTarget; + } + } + } + + private static final class SymlinkToFile extends ResolvedFile { + private final Symlink symlink; + + private SymlinkToFile(RootedPath targetPath, Symlink symlink) { + super(FileType.SYMLINK_TO_FILE, Optional.of(targetPath), Optional.<FileStateValue>absent()); + this.symlink = symlink; + } + + SymlinkToFile(RootedPath targetPath, RootedPath linkNamePath, + PathFragment linkTargetPath, FileStateValue metadata) { + super(FileType.SYMLINK_TO_FILE, Optional.of(targetPath), Optional.of(metadata)); + this.symlink = new Symlink(linkNamePath, linkTargetPath); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof SymlinkToFile)) { + return false; + } + SymlinkToFile o = (SymlinkToFile) obj; + return super.isEqualTo(o) && symlink.equals(o.symlink); + } + + @Override + public int hashCode() { + return Objects.hashCode(super.hashCode(), symlink); + } + + @Override + public String toString() { + return String.format("SymlinkToFile(%s, %s)", super.toString(), symlink); + } + + @Override + ResolvedFile stripMetadataForTesting() { + return new SymlinkToFile(path.get(), symlink); + } + + @Override + public PathFragment getNameInSymlinkTree() { + return symlink.getNameInSymlinkTree(); + } + + @Override + public PathFragment getTargetInSymlinkTree(boolean followSymlinks) { + return followSymlinks ? path.get().asPath().asFragment() : symlink.unresolvedLinkTarget; + } + } + + private static final class SymlinkToDirectory extends ResolvedFile { + private final Symlink symlink; + + private SymlinkToDirectory(RootedPath targetPath, Symlink symlink) { + super(FileType.SYMLINK_TO_DIRECTORY, Optional.of(targetPath), + Optional.<FileStateValue>absent()); + this.symlink = symlink; + } + + SymlinkToDirectory(RootedPath targetPath, RootedPath linkNamePath, + PathFragment linkValue, FileStateValue metadata) { + super(FileType.SYMLINK_TO_DIRECTORY, Optional.of(targetPath), Optional.of(metadata)); + this.symlink = new Symlink(linkNamePath, linkValue); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof SymlinkToDirectory)) { + return false; + } + SymlinkToDirectory o = (SymlinkToDirectory) obj; + return super.isEqualTo(o) && symlink.equals(o.symlink); + } + + @Override + public int hashCode() { + return Objects.hashCode(super.hashCode(), symlink); + } + + @Override + public String toString() { + return String.format("SymlinkToDirectory(%s, %s)", super.toString(), symlink); + } + + @Override + ResolvedFile stripMetadataForTesting() { + return new SymlinkToDirectory(path.get(), symlink); + } + + @Override + public PathFragment getNameInSymlinkTree() { + return symlink.getNameInSymlinkTree(); + } + + @Override + public PathFragment getTargetInSymlinkTree(boolean followSymlinks) { + return followSymlinks ? path.get().asPath().asFragment() : symlink.unresolvedLinkTarget; + } + } + + /** Type of the entity under {@link #path}. */ + final FileType type; + + /** + * Path of the file, directory or resolved target of the symlink. + * + * <p>May only be absent for dangling symlinks. + */ + protected final Optional<RootedPath> path; + + /** + * Associated metadata. + * + * <p>This field must be stored so that this {@link ResolvedFile} is (also) the function of the + * stat() of the file, but otherwise it is likely not something the consumer of the + * {@link ResolvedFile} is directly interested in. + * + * <p>May only be absent if stripped for tests. + */ + final Optional<FileStateValue> metadata; + + private ResolvedFile(FileType type, Optional<RootedPath> path, + Optional<FileStateValue> metadata) { + this.type = Preconditions.checkNotNull(type); + this.path = Preconditions.checkNotNull(path); + this.metadata = Preconditions.checkNotNull(metadata); + } + + static ResolvedFile regularFile(RootedPath path, FileStateValue metadata) { + return new RegularFile(path, metadata); + } + + static ResolvedFile directory(RootedPath path) { + return new Directory(path); + } + + static ResolvedFile symlinkToFile(RootedPath targetPath, RootedPath linkNamePath, + PathFragment linkTargetPath, FileStateValue metadata) { + return new SymlinkToFile(targetPath, linkNamePath, linkTargetPath, metadata); + } + + static ResolvedFile symlinkToDirectory(RootedPath targetPath, + RootedPath linkNamePath, PathFragment linkValue, FileStateValue metadata) { + return new SymlinkToDirectory(targetPath, linkNamePath, linkValue, metadata); + } + + static ResolvedFile danglingSymlink(RootedPath linkNamePath, PathFragment linkValue, + FileStateValue metadata) { + return new DanglingSymlink(linkNamePath, linkValue, metadata); + } + + private boolean isEqualTo(ResolvedFile o) { + return type.equals(o.type) && path.equals(o.path) && metadata.equals(o.metadata); + } + + @Override + public abstract boolean equals(Object obj); + + @Override + public int hashCode() { + return Objects.hashCode(type, path, metadata); + } + + @Override + public String toString() { + return String.format("type=%s, path=%s, metadata=%s", type, path, + metadata.isPresent() ? Integer.toHexString(metadata.get().hashCode()) : "(stripped)"); + } + + /** + * Returns the path of the Fileset-output symlink relative to the output directory. + * + * <p>The path should contain the FilesetEntry-specific destination directory (if any) and + * should have necessary prefixes stripped (if any). + */ + public abstract PathFragment getNameInSymlinkTree(); + + /** + * Returns the path of the symlink target. + * + * @throws DanglingSymlinkException if the target cannot be resolved because the symlink is + * dangling + */ + public abstract PathFragment getTargetInSymlinkTree(boolean followSymlinks) + throws DanglingSymlinkException; + + /** + * Returns a copy of this object with the metadata stripped away. + * + * <p>This method should only be used by tests that wish to assert that this + * {@link ResolvedFile} refers to the expected absolute path and has the expected type, without + * asserting its actual contents (which the metadata is a function of). + */ + @VisibleForTesting + abstract ResolvedFile stripMetadataForTesting(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof RecursiveFilesystemTraversalValue)) { + return false; + } + RecursiveFilesystemTraversalValue o = (RecursiveFilesystemTraversalValue) obj; + return resolvedRoot.equals(o.resolvedRoot) && resolvedPaths.equals(o.resolvedPaths); + } + + @Override + public int hashCode() { + return Objects.hashCode(resolvedRoot, resolvedPaths); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgFunction.java new file mode 100644 index 0000000..11ed3be --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgFunction.java
@@ -0,0 +1,151 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.vfs.Dirent; +import com.google.devtools.build.lib.vfs.Dirent.Type; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * RecursivePkgFunction builds up the set of packages underneath a given directory + * transitively. + * + * <p>Example: foo/BUILD, foo/sub/x, foo/subpkg/BUILD would yield transitive packages "foo" and + * "foo/subpkg". + */ +public class RecursivePkgFunction implements SkyFunction { + + private static final Order ORDER = Order.STABLE_ORDER; + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + RootedPath rootedPath = (RootedPath) skyKey.argument(); + Path root = rootedPath.getRoot(); + PathFragment rootRelativePath = rootedPath.getRelativePath(); + + SkyKey fileKey = FileValue.key(rootedPath); + FileValue fileValue = (FileValue) env.getValue(fileKey); + if (fileValue == null) { + return null; + } + + if (!fileValue.isDirectory()) { + return new RecursivePkgValue(NestedSetBuilder.<String>emptySet(ORDER)); + } + + if (fileValue.isSymlink()) { + // We do not follow directory symlinks when we look recursively for packages. It also + // prevents symlink loops. + return new RecursivePkgValue(NestedSetBuilder.<String>emptySet(ORDER)); + } + + PackageIdentifier packageId = PackageIdentifier.createInDefaultRepo( + rootRelativePath.getPathString()); + PackageLookupValue pkgLookupValue = + (PackageLookupValue) env.getValue(PackageLookupValue.key(packageId)); + if (pkgLookupValue == null) { + return null; + } + + NestedSetBuilder<String> packages = new NestedSetBuilder<>(ORDER); + + if (pkgLookupValue.packageExists()) { + if (pkgLookupValue.getRoot().equals(root)) { + try { + PackageValue pkgValue = (PackageValue) + env.getValueOrThrow(PackageValue.key(packageId), + NoSuchPackageException.class); + if (pkgValue == null) { + return null; + } + packages.add(pkgValue.getPackage().getName()); + } catch (NoSuchPackageException e) { + // The package had errors, but don't fail-fast as there might subpackages below the + // current directory. + env.getListener().handle(Event.error( + "package contains errors: " + rootRelativePath.getPathString())); + if (e.getPackage() != null) { + packages.add(e.getPackage().getName()); + } + } + } + // The package lookup succeeded, but was under a different root. We still, however, need to + // recursively consider subdirectories. For example: + // + // Pretend --package_path=rootA/workspace:rootB/workspace and these are the only files: + // rootA/workspace/foo/ + // rootA/workspace/foo/bar/BUILD + // rootB/workspace/foo/BUILD + // If we're doing a recursive package lookup under 'rootA/workspace' starting at 'foo', note + // that even though the package 'foo' is under 'rootB/workspace', there is still a package + // 'foo/bar' under 'rootA/workspace'. + } + + DirectoryListingValue dirValue = (DirectoryListingValue) + env.getValue(DirectoryListingValue.key(rootedPath)); + if (dirValue == null) { + return null; + } + + List<SkyKey> childDeps = Lists.newArrayList(); + for (Dirent dirent : dirValue.getDirents()) { + if (dirent.getType() != Type.DIRECTORY) { + // Non-directories can never host packages, and we do not follow symlinks (see above). + continue; + } + String basename = dirent.getName(); + if (rootRelativePath.equals(PathFragment.EMPTY_FRAGMENT) + && PathPackageLocator.DEFAULT_TOP_LEVEL_EXCLUDES.contains(basename)) { + continue; + } + SkyKey req = RecursivePkgValue.key(RootedPath.toRootedPath(root, + rootRelativePath.getRelative(basename))); + childDeps.add(req); + } + Map<SkyKey, SkyValue> childValueMap = env.getValues(childDeps); + if (env.valuesMissing()) { + return null; + } + // Aggregate the transitive subpackages. + for (SkyValue childValue : childValueMap.values()) { + if (childValue != null) { + packages.addTransitive(((RecursivePkgValue) childValue).getPackages()); + } + } + return new RecursivePkgValue(packages.build()); + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgValue.java new file mode 100644 index 0000000..4013b90 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePkgValue.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * This value represents the result of looking up all the packages under a given package path root, + * starting at a given directory. + */ +@Immutable +@ThreadSafe +public class RecursivePkgValue implements SkyValue { + + private final NestedSet<String> packages; + + public RecursivePkgValue(NestedSet<String> packages) { + this.packages = packages; + } + + /** + * Create a transitive package lookup request. + */ + @ThreadSafe + public static SkyKey key(RootedPath rootedPath) { + return new SkyKey(SkyFunctions.RECURSIVE_PKG, rootedPath); + } + + public NestedSet<String> getPackages() { + return packages; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java new file mode 100644 index 0000000..3183953 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryValue.java
@@ -0,0 +1,75 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Objects; +import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * A local view of an external repository. + */ +public class RepositoryValue implements SkyValue { + private final Path path; + + /** + * If path is a symlink, this will keep track of what the symlink actually points to (for + * checking equality). + */ + private final FileValue details; + + public RepositoryValue(Path path, FileValue repositoryDirectory) { + this.path = path; + this.details = repositoryDirectory; + } + + /** + * Returns the path to the directory containing the repository's contents. This directory is + * guaranteed to exist. It may contain a full Bazel repository (with a WORKSPACE file, + * directories, and BUILD files) or simply contain a file (or set of files) for, say, a jar from + * Maven. + */ + public Path getPath() { + return path; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof RepositoryValue) { + RepositoryValue otherValue = (RepositoryValue) other; + return path.equals(otherValue.path) && details.equals(otherValue.details); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hashCode(path, details); + } + + /** + * Creates a key from the given repository name. + */ + public static SkyKey key(RepositoryName repository) { + return new SkyKey(SkyFunctions.REPOSITORY, repository); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java new file mode 100644 index 0000000..e547166 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
@@ -0,0 +1,580 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.BuildView; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Preprocessor; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.ResourceUsage; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.ModifiedFileSet; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.BuildDriver; +import com.google.devtools.build.skyframe.Differencer; +import com.google.devtools.build.skyframe.ImmutableDiff; +import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator; +import com.google.devtools.build.skyframe.Injectable; +import com.google.devtools.build.skyframe.MemoizingEvaluator.EvaluatorSupplier; +import com.google.devtools.build.skyframe.RecordingDifferencer; +import com.google.devtools.build.skyframe.SequentialBuildDriver; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.Callable; + +/** + * A SkyframeExecutor that implicitly assumes that builds can be done incrementally from the most + * recent build. In other words, builds are "sequenced". + */ +public final class SequencedSkyframeExecutor extends SkyframeExecutor { + /** Lower limit for number of loaded packages to consider clearing CT values. */ + private int valueCacheEvictionLimit = -1; + + /** Union of labels of loaded packages since the last eviction of CT values. */ + private Set<PackageIdentifier> allLoadedPackages = ImmutableSet.of(); + private boolean lastAnalysisDiscarded = false; + + // Can only be set once (to false) over the lifetime of this object. If false, the graph will not + // store edges, saving memory but making incremental builds impossible. + private boolean keepGraphEdges = true; + + private RecordingDifferencer recordingDiffer; + private final DiffAwarenessManager diffAwarenessManager; + + private SequencedSkyframeExecutor(Reporter reporter, EvaluatorSupplier evaluatorSupplier, + PackageFactory pkgFactory, TimestampGranularityMonitor tsgm, + BlazeDirectories directories, Factory workspaceStatusActionFactory, + ImmutableList<BuildInfoFactory> buildInfoFactories, + Set<Path> immutableDirectories, + Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories, + Predicate<PathFragment> allowedMissingInputs, + Preprocessor.Factory.Supplier preprocessorFactorySupplier, + ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions, + ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) { + super(reporter, evaluatorSupplier, pkgFactory, tsgm, directories, + workspaceStatusActionFactory, buildInfoFactories, immutableDirectories, + allowedMissingInputs, preprocessorFactorySupplier, + extraSkyFunctions, extraPrecomputedValues); + this.diffAwarenessManager = new DiffAwarenessManager(diffAwarenessFactories, reporter); + } + + private SequencedSkyframeExecutor(Reporter reporter, PackageFactory pkgFactory, + TimestampGranularityMonitor tsgm, BlazeDirectories directories, + Factory workspaceStatusActionFactory, + ImmutableList<BuildInfoFactory> buildInfoFactories, + Set<Path> immutableDirectories, + Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories, + Predicate<PathFragment> allowedMissingInputs, + Preprocessor.Factory.Supplier preprocessorFactorySupplier, + ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions, + ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) { + this(reporter, InMemoryMemoizingEvaluator.SUPPLIER, pkgFactory, tsgm, + directories, workspaceStatusActionFactory, buildInfoFactories, immutableDirectories, + diffAwarenessFactories, allowedMissingInputs, preprocessorFactorySupplier, + extraSkyFunctions, extraPrecomputedValues); + } + + private static SequencedSkyframeExecutor create(Reporter reporter, + EvaluatorSupplier evaluatorSupplier, PackageFactory pkgFactory, + TimestampGranularityMonitor tsgm, BlazeDirectories directories, + Factory workspaceStatusActionFactory, ImmutableList<BuildInfoFactory> buildInfoFactories, + Set<Path> immutableDirectories, + Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories, + Predicate<PathFragment> allowedMissingInputs, + Preprocessor.Factory.Supplier preprocessorFactorySupplier, + ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions, + ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) { + SequencedSkyframeExecutor skyframeExecutor = new SequencedSkyframeExecutor(reporter, + evaluatorSupplier, pkgFactory, tsgm, directories, workspaceStatusActionFactory, + buildInfoFactories, immutableDirectories, diffAwarenessFactories, allowedMissingInputs, + preprocessorFactorySupplier, + extraSkyFunctions, extraPrecomputedValues); + skyframeExecutor.init(); + return skyframeExecutor; + } + + public static SequencedSkyframeExecutor create(Reporter reporter, PackageFactory pkgFactory, + TimestampGranularityMonitor tsgm, BlazeDirectories directories, + Factory workspaceStatusActionFactory, + ImmutableList<BuildInfoFactory> buildInfoFactories, + Set<Path> immutableDirectories, + Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories, + Predicate<PathFragment> allowedMissingInputs, + Preprocessor.Factory.Supplier preprocessorFactorySupplier, + ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions, + ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) { + return create(reporter, InMemoryMemoizingEvaluator.SUPPLIER, pkgFactory, tsgm, + directories, workspaceStatusActionFactory, buildInfoFactories, immutableDirectories, + diffAwarenessFactories, allowedMissingInputs, preprocessorFactorySupplier, + extraSkyFunctions, extraPrecomputedValues); + } + + @VisibleForTesting + public static SequencedSkyframeExecutor create(Reporter reporter, PackageFactory pkgFactory, + TimestampGranularityMonitor tsgm, BlazeDirectories directories, + WorkspaceStatusAction.Factory workspaceStatusActionFactory, + ImmutableList<BuildInfoFactory> buildInfoFactories, + Set<Path> immutableDirectories, + Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories) { + return create(reporter, pkgFactory, tsgm, directories, workspaceStatusActionFactory, + buildInfoFactories, immutableDirectories, diffAwarenessFactories, + Predicates.<PathFragment>alwaysFalse(), + Preprocessor.Factory.Supplier.NullSupplier.INSTANCE, + ImmutableMap.<SkyFunctionName, SkyFunction>of(), + ImmutableList.<PrecomputedValue.Injected>of()); + } + + @Override + protected BuildDriver newBuildDriver() { + return new SequentialBuildDriver(memoizingEvaluator); + } + + @Override + protected void init() { + // Note that we need to set recordingDiffer first since SkyframeExecutor#init calls + // SkyframeExecutor#evaluatorDiffer. + recordingDiffer = new RecordingDifferencer(); + super.init(); + } + + @Override + public void resetEvaluator() { + super.resetEvaluator(); + diffAwarenessManager.reset(); + } + + @Override + protected Differencer evaluatorDiffer() { + return recordingDiffer; + } + + @Override + protected Injectable injectable() { + return recordingDiffer; + } + + @VisibleForTesting + public RecordingDifferencer getDifferencerForTesting() { + return recordingDiffer; + } + + @Override + public void sync(PackageCacheOptions packageCacheOptions, Path workingDirectory, + String defaultsPackageContents, UUID commandId) + throws InterruptedException, AbruptExitException { + this.valueCacheEvictionLimit = packageCacheOptions.minLoadedPkgCountForCtNodeEviction; + super.sync(packageCacheOptions, workingDirectory, defaultsPackageContents, commandId); + handleDiffs(); + } + + /** + * The value types whose builders have direct access to the package locator, rather than accessing + * it via an explicit Skyframe dependency. They need to be invalidated if the package locator + * changes. + */ + private static final Set<SkyFunctionName> PACKAGE_LOCATOR_DEPENDENT_VALUES = ImmutableSet.of( + SkyFunctions.FILE_STATE, + SkyFunctions.FILE, + SkyFunctions.DIRECTORY_LISTING_STATE, + SkyFunctions.TARGET_PATTERN, + SkyFunctions.WORKSPACE_FILE); + + @Override + protected void onNewPackageLocator(PathPackageLocator oldLocator, PathPackageLocator pkgLocator) { + invalidate(SkyFunctionName.functionIsIn(PACKAGE_LOCATOR_DEPENDENT_VALUES)); + } + + @Override + protected void invalidate(Predicate<SkyKey> pred) { + recordingDiffer.invalidate(Iterables.filter(memoizingEvaluator.getValues().keySet(), pred)); + } + + private void invalidateDeletedPackages(Iterable<String> deletedPackages) { + ArrayList<SkyKey> packagesToInvalidate = Lists.newArrayList(); + for (String deletedPackage : deletedPackages) { + PathFragment pathFragment = new PathFragment(deletedPackage); + packagesToInvalidate.add(PackageLookupValue.key(pathFragment)); + } + recordingDiffer.invalidate(packagesToInvalidate); + } + + /** + * Sets the packages that should be treated as deleted and ignored. + */ + @Override + @VisibleForTesting // productionVisibility = Visibility.PRIVATE + public void setDeletedPackages(Iterable<String> pkgs) { + // Invalidate the old deletedPackages as they may exist now. + invalidateDeletedPackages(deletedPackages.get()); + deletedPackages.set(ImmutableSet.copyOf(pkgs)); + // Invalidate the new deletedPackages as we need to pretend that they don't exist now. + invalidateDeletedPackages(deletedPackages.get()); + } + + /** + * Uses diff awareness on all the package paths to invalidate changed files. + */ + @VisibleForTesting + public void handleDiffs() throws InterruptedException { + if (lastAnalysisDiscarded) { + // Values were cleared last build, but they couldn't be deleted because they were needed for + // the execution phase. We can delete them now. + dropConfiguredTargetsNow(); + lastAnalysisDiscarded = false; + } + modifiedFiles = 0; + Map<Path, DiffAwarenessManager.ProcessableModifiedFileSet> modifiedFilesByPathEntry = + Maps.newHashMap(); + Set<Pair<Path, DiffAwarenessManager.ProcessableModifiedFileSet>> + pathEntriesWithoutDiffInformation = Sets.newHashSet(); + for (Path pathEntry : pkgLocator.get().getPathEntries()) { + DiffAwarenessManager.ProcessableModifiedFileSet modifiedFileSet = + diffAwarenessManager.getDiff(pathEntry); + if (modifiedFileSet.getModifiedFileSet().treatEverythingAsModified()) { + pathEntriesWithoutDiffInformation.add(Pair.of(pathEntry, modifiedFileSet)); + } else { + modifiedFilesByPathEntry.put(pathEntry, modifiedFileSet); + } + } + handleDiffsWithCompleteDiffInformation(modifiedFilesByPathEntry); + handleDiffsWithMissingDiffInformation(pathEntriesWithoutDiffInformation); + } + + /** + * Invalidates files under path entries whose corresponding {@link DiffAwareness} gave an exact + * diff. Removes entries from the given map as they are processed. All of the files need to be + * invalidated, so the map should be empty upon completion of this function. + */ + private void handleDiffsWithCompleteDiffInformation( + Map<Path, DiffAwarenessManager.ProcessableModifiedFileSet> modifiedFilesByPathEntry) { + // It's important that the below code be uninterruptible, since we already promised to + // invalidate these files. + for (Path pathEntry : ImmutableSet.copyOf(modifiedFilesByPathEntry.keySet())) { + DiffAwarenessManager.ProcessableModifiedFileSet processableModifiedFileSet = + modifiedFilesByPathEntry.get(pathEntry); + ModifiedFileSet modifiedFileSet = processableModifiedFileSet.getModifiedFileSet(); + Preconditions.checkState(!modifiedFileSet.treatEverythingAsModified(), pathEntry); + Iterable<SkyKey> dirtyValues = getSkyKeysPotentiallyAffected( + modifiedFileSet.modifiedSourceFiles(), pathEntry); + handleChangedFiles(new ImmutableDiff(dirtyValues, ImmutableMap.<SkyKey, SkyValue>of())); + processableModifiedFileSet.markProcessed(); + } + } + + /** + * Finds and invalidates changed files under path entries whose corresponding + * {@link DiffAwareness} said all files may have been modified. + */ + private void handleDiffsWithMissingDiffInformation( + Set<Pair<Path, DiffAwarenessManager.ProcessableModifiedFileSet>> + pathEntriesWithoutDiffInformation) throws InterruptedException { + if (pathEntriesWithoutDiffInformation.isEmpty()) { + return; + } + // Before running the FilesystemValueChecker, ensure that all values marked for invalidation + // have actually been invalidated (recall that invalidation happens at the beginning of the + // next evaluate() call), because checking those is a waste of time. + buildDriver.evaluate(ImmutableList.<SkyKey>of(), false, + DEFAULT_THREAD_COUNT, reporter); + FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm); + // We need to manually check for changes to known files. This entails finding all dirty file + // system values under package roots for which we don't have diff information. If at least + // one path entry doesn't have diff information, then we're going to have to iterate over + // the skyframe values at least once no matter what so we might as well do so now and avoid + // doing so more than once. + Iterable<SkyKey> filesystemSkyKeys = fsnc.getFilesystemSkyKeys(); + // Partition by package path entry. + Multimap<Path, SkyKey> skyKeysByPathEntry = partitionSkyKeysByPackagePathEntry( + ImmutableSet.copyOf(pkgLocator.get().getPathEntries()), filesystemSkyKeys); + // Contains all file system values that we need to check for dirtiness. + List<Iterable<SkyKey>> valuesToCheckManually = Lists.newArrayList(); + for (Pair<Path, DiffAwarenessManager.ProcessableModifiedFileSet> pair : + pathEntriesWithoutDiffInformation) { + Path pathEntry = pair.getFirst(); + valuesToCheckManually.add(skyKeysByPathEntry.get(pathEntry)); + } + Differencer.Diff diff = fsnc.getDirtyFilesystemValues(Iterables.concat(valuesToCheckManually)); + handleChangedFiles(diff); + for (Pair<Path, DiffAwarenessManager.ProcessableModifiedFileSet> pair : + pathEntriesWithoutDiffInformation) { + DiffAwarenessManager.ProcessableModifiedFileSet processableModifiedFileSet = pair.getSecond(); + processableModifiedFileSet.markProcessed(); + } + } + + /** + * Partitions the given filesystem values based on which package path root they are under. + * Returns a {@link Multimap} {@code m} such that {@code m.containsEntry(k, pe)} is true for + * each filesystem valuekey {@code k} under a package path root {@code pe}. Note that values not + * under a package path root are not present in the returned {@link Multimap}; these values are + * unconditionally checked for changes on each incremental build. + */ + private static Multimap<Path, SkyKey> partitionSkyKeysByPackagePathEntry( + Set<Path> pkgRoots, Iterable<SkyKey> filesystemSkyKeys) { + ImmutableSetMultimap.Builder<Path, SkyKey> multimapBuilder = + ImmutableSetMultimap.builder(); + for (SkyKey key : filesystemSkyKeys) { + Preconditions.checkState(key.functionName() == SkyFunctions.FILE_STATE + || key.functionName() == SkyFunctions.DIRECTORY_LISTING_STATE, key); + Path root = ((RootedPath) key.argument()).getRoot(); + if (pkgRoots.contains(root)) { + multimapBuilder.put(root, key); + } + // We don't need to worry about FileStateValues for external files because they have a + // dependency on the build_id and so they get invalidated each build. + } + return multimapBuilder.build(); + } + + private void handleChangedFiles(Differencer.Diff diff) { + recordingDiffer.invalidate(diff.changedKeysWithoutNewValues()); + recordingDiffer.inject(diff.changedKeysWithNewValues()); + modifiedFiles += getNumberOfModifiedFiles(diff.changedKeysWithoutNewValues()); + modifiedFiles += getNumberOfModifiedFiles(diff.changedKeysWithNewValues().keySet()); + incrementalBuildMonitor.accrue(diff.changedKeysWithoutNewValues()); + incrementalBuildMonitor.accrue(diff.changedKeysWithNewValues().keySet()); + } + + private static int getNumberOfModifiedFiles(Iterable<SkyKey> modifiedValues) { + // We are searching only for changed files, DirectoryListingValues don't depend on + // child values, that's why they are invalidated separately + return Iterables.size(Iterables.filter(modifiedValues, + SkyFunctionName.functionIs(SkyFunctions.FILE_STATE))); + } + + @Override + public void decideKeepIncrementalState(boolean batch, BuildView.Options viewOptions) { + Preconditions.checkState(!active); + if (viewOptions == null) { + // Some blaze commands don't include the view options. Don't bother with them. + return; + } + if (batch && viewOptions.keepGoing && viewOptions.discardAnalysisCache) { + Preconditions.checkState(keepGraphEdges, "May only be called once if successful"); + keepGraphEdges = false; + // Graph will be recreated on next sync. + } + } + + @Override + public boolean hasIncrementalState() { + // TODO(bazel-team): Combine this method with clearSkyframeRelevantCaches() once legacy + // execution is removed [skyframe-execution]. + return keepGraphEdges; + } + + @Override + public void invalidateFilesUnderPathForTesting(ModifiedFileSet modifiedFileSet, Path pathEntry) + throws InterruptedException { + if (lastAnalysisDiscarded) { + // Values were cleared last build, but they couldn't be deleted because they were needed for + // the execution phase. We can delete them now. + dropConfiguredTargetsNow(); + lastAnalysisDiscarded = false; + } + Iterable<SkyKey> keys; + if (modifiedFileSet.treatEverythingAsModified()) { + Differencer.Diff diff = + new FilesystemValueChecker(memoizingEvaluator, tsgm).getDirtyFilesystemSkyKeys(); + keys = diff.changedKeysWithoutNewValues(); + recordingDiffer.inject(diff.changedKeysWithNewValues()); + } else { + keys = getSkyKeysPotentiallyAffected(modifiedFileSet.modifiedSourceFiles(), pathEntry); + } + syscalls.set(new PerBuildSyscallCache()); + recordingDiffer.invalidate(keys); + // Blaze invalidates transient errors on every build. + invalidateTransientErrors(); + } + + @Override + public void invalidateTransientErrors() { + checkActive(); + recordingDiffer.invalidateTransientErrors(); + } + + @Override + protected void invalidateDirtyActions(Iterable<SkyKey> dirtyActionValues) { + recordingDiffer.invalidate(dirtyActionValues); + } + + /** + * Save memory by removing references to configured targets and actions in Skyframe. + * + * <p>These values must be recreated on subsequent builds. We do not clear the top-level target + * values, since their configured targets are needed for the target completion middleman values. + * + * <p>The values are not deleted during this method call, because they are needed for the + * execution phase. Instead, their data is cleared. The next build will delete the values (and + * recreate them if necessary). + */ + private void discardAnalysisCache(Collection<ConfiguredTarget> topLevelTargets) { + lastAnalysisDiscarded = true; + for (Map.Entry<SkyKey, SkyValue> entry : memoizingEvaluator.getValues().entrySet()) { + if (!entry.getKey().functionName().equals(SkyFunctions.CONFIGURED_TARGET)) { + continue; + } + ConfiguredTargetValue ctValue = (ConfiguredTargetValue) entry.getValue(); + // ctValue may be null if target was not successfully analyzed. + if (ctValue != null && !topLevelTargets.contains(ctValue.getConfiguredTarget())) { + ctValue.clear(); + } + } + } + + @Override + public void clearAnalysisCache(Collection<ConfiguredTarget> topLevelTargets) { + discardAnalysisCache(topLevelTargets); + } + + @Override + public void dropConfiguredTargets() { + if (skyframeBuildView != null) { + skyframeBuildView.clearInvalidatedConfiguredTargets(); + } + memoizingEvaluator.delete( + // We delete any value that can hold an action -- all subclasses of ActionLookupValue -- as + // well as ActionExecutionValues, since they do not depend on ActionLookupValues. + SkyFunctionName.functionIsIn(ImmutableSet.of( + SkyFunctions.CONFIGURED_TARGET, + SkyFunctions.ACTION_LOOKUP, + SkyFunctions.BUILD_INFO, + SkyFunctions.TARGET_COMPLETION, + SkyFunctions.BUILD_INFO_COLLECTION, + SkyFunctions.ACTION_EXECUTION)) + ); + } + + /** + * Deletes all ConfiguredTarget values from the Skyframe cache. + * + * <p>After the execution of this method all invalidated and marked for deletion values + * (and the values depending on them) will be deleted from the cache. + * + * <p>WARNING: Note that a call to this method leaves legacy data inconsistent with Skyframe. + * The next build should clear the legacy caches. + */ + private void dropConfiguredTargetsNow() { + dropConfiguredTargets(); + // Run the invalidator to actually delete the values. + try { + progressReceiver.ignoreInvalidations = true; + callUninterruptibly(new Callable<Void>() { + @Override + public Void call() throws InterruptedException { + buildDriver.evaluate(ImmutableList.<SkyKey>of(), false, + ResourceUsage.getAvailableProcessors(), reporter); + return null; + } + }); + } catch (Exception e) { + throw new IllegalStateException(e); + } finally { + progressReceiver.ignoreInvalidations = false; + } + } + + /** + * Returns true if the old set of Packages is a subset or superset of the new one. + * + * <p>Compares the names of packages instead of the Package objects themselves (Package doesn't + * yet override #equals). Since packages store their names as a String rather than a Label, it's + * easier to use strings here. + */ + @VisibleForTesting + static boolean isBuildSubsetOrSupersetOfPreviousBuild(Set<PackageIdentifier> oldPackages, + Set<PackageIdentifier> newPackages) { + if (newPackages.size() <= oldPackages.size()) { + return Sets.difference(newPackages, oldPackages).isEmpty(); + } else if (oldPackages.size() < newPackages.size()) { + // No need to check for <= here, since the first branch does that already. + // If size(A) = size(B), then then A\B = 0 iff B\A = 0 + return Sets.difference(oldPackages, newPackages).isEmpty(); + } else { + return false; + } + } + + @Override + public void updateLoadedPackageSet(Set<PackageIdentifier> loadedPackages) { + Preconditions.checkState(valueCacheEvictionLimit >= 0, + "should have called setMinLoadedPkgCountForCtValueEviction earlier"); + + // Make a copy to avoid nesting SetView objects. It also computes size(), which we need below. + Set<PackageIdentifier> union = ImmutableSet.copyOf( + Sets.union(allLoadedPackages, loadedPackages)); + + if (union.size() < valueCacheEvictionLimit + || isBuildSubsetOrSupersetOfPreviousBuild(allLoadedPackages, loadedPackages)) { + allLoadedPackages = union; + } else { + dropConfiguredTargets(); + allLoadedPackages = loadedPackages; + } + } + + @Override + public void deleteOldNodes(long versionWindowForDirtyGc) { + // TODO(bazel-team): perhaps we should come up with a separate GC class dedicated to maintaining + // value garbage. If we ever do so, this logic should be moved there. + memoizingEvaluator.deleteDirty(versionWindowForDirtyGc); + } + + @Override + public void dumpPackages(PrintStream out) { + Iterable<SkyKey> packageSkyKeys = Iterables.filter(memoizingEvaluator.getValues().keySet(), + SkyFunctions.isSkyFunction(SkyFunctions.PACKAGE)); + out.println(Iterables.size(packageSkyKeys) + " packages"); + for (SkyKey packageSkyKey : packageSkyKeys) { + Package pkg = ((PackageValue) memoizingEvaluator.getValues().get(packageSkyKey)).getPackage(); + pkg.dump(out); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorFactory.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorFactory.java new file mode 100644 index 0000000..7e277a7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutorFactory.java
@@ -0,0 +1,53 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.Preprocessor; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; + +import java.util.Set; + +/** + * A factory of SkyframeExecutors that returns SequencedSkyframeExecutor. + */ +public class SequencedSkyframeExecutorFactory implements SkyframeExecutorFactory { + + @Override + public SkyframeExecutor create(Reporter reporter, PackageFactory pkgFactory, + TimestampGranularityMonitor tsgm, BlazeDirectories directories, + Factory workspaceStatusActionFactory, ImmutableList<BuildInfoFactory> buildInfoFactories, + Set<Path> immutableDirectories, + Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories, + Predicate<PathFragment> allowedMissingInputs, + Preprocessor.Factory.Supplier preprocessorFactorySupplier, + ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions, + ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) { + return SequencedSkyframeExecutor.create(reporter, pkgFactory, tsgm, directories, + workspaceStatusActionFactory, buildInfoFactories, immutableDirectories, + diffAwarenessFactories, allowedMissingInputs, preprocessorFactorySupplier, + extraSkyFunctions, extraPrecomputedValues); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java new file mode 100644 index 0000000..316d27d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
@@ -0,0 +1,81 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Predicate; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; + +/** + * Value types in Skyframe. + */ +public final class SkyFunctions { + public static final SkyFunctionName PRECOMPUTED = new SkyFunctionName("PRECOMPUTED", false); + public static final SkyFunctionName FILE_STATE = new SkyFunctionName("FILE_STATE", false); + public static final SkyFunctionName DIRECTORY_LISTING_STATE = + new SkyFunctionName("DIRECTORY_LISTING_STATE", false); + public static final SkyFunctionName FILE_SYMLINK_CYCLE_UNIQUENESS = + SkyFunctionName.computed("FILE_SYMLINK_CYCLE_UNIQUENESS_NODE"); + public static final SkyFunctionName FILE = SkyFunctionName.computed("FILE"); + public static final SkyFunctionName DIRECTORY_LISTING = + SkyFunctionName.computed("DIRECTORY_LISTING"); + public static final SkyFunctionName PACKAGE_LOOKUP = SkyFunctionName.computed("PACKAGE_LOOKUP"); + public static final SkyFunctionName CONTAINING_PACKAGE_LOOKUP = + SkyFunctionName.computed("CONTAINING_PACKAGE_LOOKUP"); + public static final SkyFunctionName AST_FILE_LOOKUP = SkyFunctionName.computed("AST_FILE_LOOKUP"); + public static final SkyFunctionName SKYLARK_IMPORTS_LOOKUP = + SkyFunctionName.computed("SKYLARK_IMPORTS_LOOKUP"); + public static final SkyFunctionName GLOB = SkyFunctionName.computed("GLOB"); + public static final SkyFunctionName PACKAGE = SkyFunctionName.computed("PACKAGE"); + public static final SkyFunctionName TARGET_MARKER = SkyFunctionName.computed("TARGET_MARKER"); + public static final SkyFunctionName TARGET_PATTERN = SkyFunctionName.computed("TARGET_PATTERN"); + public static final SkyFunctionName RECURSIVE_PKG = SkyFunctionName.computed("RECURSIVE_PKG"); + public static final SkyFunctionName TRANSITIVE_TARGET = + SkyFunctionName.computed("TRANSITIVE_TARGET"); + public static final SkyFunctionName CONFIGURED_TARGET = + SkyFunctionName.computed("CONFIGURED_TARGET"); + public static final SkyFunctionName ASPECT = SkyFunctionName.computed("ASPECT"); + public static final SkyFunctionName POST_CONFIGURED_TARGET = + SkyFunctionName.computed("POST_CONFIGURED_TARGET"); + public static final SkyFunctionName TARGET_COMPLETION = + SkyFunctionName.computed("TARGET_COMPLETION"); + public static final SkyFunctionName TEST_COMPLETION = + SkyFunctionName.computed("TEST_COMPLETION"); + public static final SkyFunctionName CONFIGURATION_FRAGMENT = + SkyFunctionName.computed("CONFIGURATION_FRAGMENT"); + public static final SkyFunctionName CONFIGURATION_COLLECTION = + SkyFunctionName.computed("CONFIGURATION_COLLECTION"); + public static final SkyFunctionName ARTIFACT = SkyFunctionName.computed("ARTIFACT"); + public static final SkyFunctionName ACTION_EXECUTION = + SkyFunctionName.computed("ACTION_EXECUTION"); + public static final SkyFunctionName ACTION_LOOKUP = SkyFunctionName.computed("ACTION_LOOKUP"); + public static final SkyFunctionName RECURSIVE_FILESYSTEM_TRAVERSAL = + SkyFunctionName.computed("RECURSIVE_DIRECTORY_TRAVERSAL"); + public static final SkyFunctionName FILESET_ENTRY = SkyFunctionName.computed("FILESET_ENTRY"); + public static final SkyFunctionName BUILD_INFO_COLLECTION = + SkyFunctionName.computed("BUILD_INFO_COLLECTION"); + public static final SkyFunctionName BUILD_INFO = SkyFunctionName.computed("BUILD_INFO"); + public static final SkyFunctionName WORKSPACE_FILE = SkyFunctionName.computed("WORKSPACE_FILE"); + public static final SkyFunctionName COVERAGE_REPORT = SkyFunctionName.computed("COVERAGE_REPORT"); + public static final SkyFunctionName REPOSITORY = SkyFunctionName.computed("REPOSITORY"); + + public static Predicate<SkyKey> isSkyFunction(final SkyFunctionName functionName) { + return new Predicate<SkyKey>() { + @Override + public boolean apply(SkyKey key) { + return key.functionName() == functionName; + } + }; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java new file mode 100644 index 0000000..bd591e1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java
@@ -0,0 +1,1152 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import static com.google.devtools.build.lib.vfs.FileSystemUtils.createDirectoryAndParents; + +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.common.eventbus.EventBus; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionCacheChecker; +import com.google.devtools.build.lib.actions.ActionCacheChecker.Token; +import com.google.devtools.build.lib.actions.ActionCompletionEvent; +import com.google.devtools.build.lib.actions.ActionExecutedEvent; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.ActionLogBufferPathGenerator; +import com.google.devtools.build.lib.actions.ActionMiddlemanEvent; +import com.google.devtools.build.lib.actions.ActionStartedEvent; +import com.google.devtools.build.lib.actions.Actions; +import com.google.devtools.build.lib.actions.AlreadyReportedActionExecutionException; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Artifact.MiddlemanExpander; +import com.google.devtools.build.lib.actions.ArtifactPrefixConflictException; +import com.google.devtools.build.lib.actions.CachedActionEvent; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.MapBasedActionGraph; +import com.google.devtools.build.lib.actions.MutableActionGraph; +import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; +import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit; +import com.google.devtools.build.lib.actions.ResourceManager; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.TargetOutOfDateException; +import com.google.devtools.build.lib.actions.cache.Digest; +import com.google.devtools.build.lib.actions.cache.DigestUtils; +import com.google.devtools.build.lib.actions.cache.Metadata; +import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.concurrent.ExecutorShutdownUtil; +import com.google.devtools.build.lib.concurrent.Sharder; +import com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.FileStatus; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.protobuf.ByteString; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.Nullable; + +/** + * Action executor: takes care of preparing an action for execution, executing it, validating that + * all output artifacts were created, error reporting, etc. + */ +public final class SkyframeActionExecutor { + private final Reporter reporter; + private final AtomicReference<EventBus> eventBus; + private final ResourceManager resourceManager; + private Executor executorEngine; + private ActionLogBufferPathGenerator actionLogBufferPathGenerator; + private ActionCacheChecker actionCacheChecker; + private ConcurrentMap<Artifact, Metadata> undeclaredInputsMetadata = new ConcurrentHashMap<>(); + private final Profiler profiler = Profiler.instance(); + private boolean explain; + + // We keep track of actions already executed this build in order to avoid executing a shared + // action twice. Note that we may still unnecessarily re-execute the action on a subsequent + // build: say actions A and B are shared. If A is requested on the first build and then B is + // requested on the second build, we will execute B even though its output files are up to date. + // However, we will not re-execute A on a subsequent build. + // We do not allow the shared action to re-execute in the same build, even after the first + // action has finished execution, because a downstream action might be reading the output file + // at the same time as the shared action was writing to it. + // This map is also used for Actions that try to execute twice because they have discovered + // headers -- the SkyFunction tries to declare a dep on the missing headers and has to restart. + // We don't want to execute the action again on the second entry to the SkyFunction. + // In both cases, we store the already-computed ActionExecutionValue to avoid having to compute it + // again. + private ConcurrentMap<Artifact, Pair<Action, FutureTask<ActionExecutionValue>>> buildActionMap; + + // Errors found when examining all actions in the graph are stored here, so that they can be + // thrown when execution of the action is requested. This field is set during each call to + // findAndStoreArtifactConflicts, and is preserved across builds otherwise. + private ImmutableMap<Action, ConflictException> badActionMap = ImmutableMap.of(); + private boolean keepGoing; + private boolean hadExecutionError; + private ActionInputFileCache perBuildFileCache; + private ProgressSupplier progressSupplier; + private ActionCompletedReceiver completionReceiver; + private final AtomicReference<ActionExecutionStatusReporter> statusReporterRef; + + SkyframeActionExecutor(Reporter reporter, ResourceManager resourceManager, + AtomicReference<EventBus> eventBus, + AtomicReference<ActionExecutionStatusReporter> statusReporterRef) { + this.reporter = reporter; + this.resourceManager = resourceManager; + this.eventBus = eventBus; + this.statusReporterRef = statusReporterRef; + } + + /** + * A typed union of {@link ActionConflictException}, which indicates two actions that generate + * the same {@link Artifact}, and {@link ArtifactPrefixConflictException}, which indicates that + * the path of one {@link Artifact} is a prefix of another. + */ + public static class ConflictException extends Exception { + @Nullable private final ActionConflictException ace; + @Nullable private final ArtifactPrefixConflictException apce; + + public ConflictException(ActionConflictException e) { + super(e); + this.ace = e; + this.apce = null; + } + + public ConflictException(ArtifactPrefixConflictException e) { + super(e); + this.ace = null; + this.apce = e; + } + + void rethrowTyped() throws ActionConflictException, ArtifactPrefixConflictException { + if (ace == null) { + throw Preconditions.checkNotNull(apce); + } + if (apce == null) { + throw Preconditions.checkNotNull(ace); + } + throw new IllegalStateException(); + } + } + + /** + * Return the map of mostly recently executed bad actions to their corresponding exception. + * See {#findAndStoreArtifactConflicts()}. + */ + public ImmutableMap<Action, ConflictException> badActions() { + // TODO(bazel-team): Move badActions() and findAndStoreArtifactConflicts() to SkyframeBuildView + // now that it's done in the analysis phase. + return badActionMap; + } + + /** + * Basic implementation of {@link MetadataHandler} that delegates to Skyframe for metadata and + * caches missing source artifacts (which must be undeclared inputs: discovered headers) to avoid + * excessive filesystem access. The discovered-header cache is available across actions. + */ + // TODO(bazel-team): remove when include scanning is skyframe-native. + private static class UndeclaredInputHandler implements MetadataHandler { + private final ConcurrentMap<Artifact, Metadata> undeclaredInputsMetadata; + private final MetadataHandler perActionHandler; + + UndeclaredInputHandler(MetadataHandler perActionHandler, + ConcurrentMap<Artifact, Metadata> undeclaredInputsMetadata) { + // Shared across all UndeclaredInputHandlers in this build. + this.undeclaredInputsMetadata = undeclaredInputsMetadata; + this.perActionHandler = perActionHandler; + } + + @Override + public Metadata getMetadataMaybe(Artifact artifact) { + try { + return getMetadata(artifact); + } catch (IOException e) { + return null; + } + } + + @Override + public Metadata getMetadata(Artifact artifact) throws IOException { + Metadata metadata = perActionHandler.getMetadata(artifact); + if (metadata != null) { + return metadata; + } + // Skyframe stats all generated artifacts, because either they are outputs of the action being + // executed or they are generated files already present in the graph. + Preconditions.checkState(artifact.isSourceArtifact(), artifact); + metadata = undeclaredInputsMetadata.get(artifact); + if (metadata != null) { + return metadata; + } + FileStatus stat = artifact.getPath().stat(); + if (DigestUtils.useFileDigest(artifact, stat.isFile(), stat.getSize())) { + metadata = new Metadata(Preconditions.checkNotNull( + DigestUtils.getDigestOrFail(artifact.getPath(), stat.getSize()), artifact)); + } else { + metadata = new Metadata(stat.getLastModifiedTime()); + } + // Cache for other actions that may also include without declaring. + Metadata oldMetadata = undeclaredInputsMetadata.put(artifact, metadata); + FileAndMetadataCache.checkInconsistentData(artifact, oldMetadata, metadata); + return metadata; + } + + @Override + public void setDigestForVirtualArtifact(Artifact artifact, Digest digest) { + perActionHandler.setDigestForVirtualArtifact(artifact, digest); + } + + @Override + public void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest) { + perActionHandler.injectDigest(output, statNoFollow, digest); + } + + @Override + public boolean artifactExists(Artifact artifact) { + return perActionHandler.artifactExists(artifact); + } + + @Override + public boolean isRegularFile(Artifact artifact) { + return perActionHandler.isRegularFile(artifact); + } + + @Override + public boolean isInjected(Artifact artifact) throws IOException { + return perActionHandler.isInjected(artifact); + } + + @Override + public void discardMetadata(Collection<Artifact> artifactList) { + // This input handler only caches undeclared inputs, which never need to be discarded + // intra-build. + perActionHandler.discardMetadata(artifactList); + } + } + + /** + * Find conflicts between generated artifacts. There are two ways to have conflicts. First, if + * two (unshareable) actions generate the same output artifact, this will result in an {@link + * ActionConflictException}. Second, if one action generates an artifact whose path is a prefix of + * another artifact's path, those two artifacts cannot exist simultaneously in the output tree. + * This causes an {@link ArtifactPrefixConflictException}. The relevant exceptions are stored in + * the executor in {@code badActionMap}, and will be thrown immediately when that action is + * executed. Those exceptions persist, so that even if the action is not executed this build, the + * first time it is executed, the correct exception will be thrown. + * + * <p>This method must be called if a new action was added to the graph this build, so + * whenever a new configured target was analyzed this build. It is somewhat expensive (~1s + * range for a medium build as of 2014), so it should only be called when necessary. + * + * <p>Conflicts found may not be requested this build, and so we may overzealously throw an error. + * For instance, if actions A and B generate the same artifact foo, and the user first requests + * A' depending on A, and then in a subsequent build B' depending on B, we will fail the second + * build, even though it would have succeeded if it had been the only build. However, since + * Skyframe does not know the transitive dependencies of the request, we err on the conservative + * side. + * + * <p>If the user first runs one action on the first build, and on the second build adds a + * conflicting action, only the second action's error may be reported (because the first action + * will be cached), whereas if both actions were requested for the first time, both errors would + * be reported. However, the first time an action is added to the build, we are guaranteed to find + * any conflicts it has, since this method will compare it against all other actions. So there is + * no sequence of builds that can evade the error. + */ + void findAndStoreArtifactConflicts(Iterable<ActionLookupValue> actionLookupValues) + throws InterruptedException { + ConcurrentMap<Action, ConflictException> temporaryBadActionMap = new ConcurrentHashMap<>(); + Pair<ActionGraph, SortedMap<PathFragment, Artifact>> result; + result = constructActionGraphAndPathMap(actionLookupValues, temporaryBadActionMap); + ActionGraph actionGraph = result.first; + SortedMap<PathFragment, Artifact> artifactPathMap = result.second; + + // Report an error for every derived artifact which is a prefix of another. + // If x << y << z (where x << y means "y starts with x"), then we only report (x,y), (x,z), but + // not (y,z). + Iterator<PathFragment> iter = artifactPathMap.keySet().iterator(); + if (!iter.hasNext()) { + // No actions in graph -- currently happens only in tests. Special-cased because .next() call + // below is unconditional. + this.badActionMap = ImmutableMap.of(); + return; + } + for (PathFragment pathJ = iter.next(); iter.hasNext(); ) { + // For each comparison, we have a prefix candidate (pathI) and a suffix candidate (pathJ). + // At the beginning of the loop, we set pathI to the last suffix candidate, since it has not + // yet been tested as a prefix candidate, and then set pathJ to the paths coming after pathI, + // until we come to one that does not contain pathI as a prefix. pathI is then verified not to + // be the prefix of any path, so we start the next run of the loop. + PathFragment pathI = pathJ; + // Compare pathI to the paths coming after it. + while (iter.hasNext()) { + pathJ = iter.next(); + if (pathJ.startsWith(pathI)) { // prefix conflict. + Artifact artifactI = Preconditions.checkNotNull(artifactPathMap.get(pathI), pathI); + Artifact artifactJ = Preconditions.checkNotNull(artifactPathMap.get(pathJ), pathJ); + Action actionI = + Preconditions.checkNotNull(actionGraph.getGeneratingAction(artifactI), artifactI); + Action actionJ = + Preconditions.checkNotNull(actionGraph.getGeneratingAction(artifactJ), artifactJ); + if (actionI.shouldReportPathPrefixConflict(actionJ)) { + ArtifactPrefixConflictException exception = new ArtifactPrefixConflictException(pathI, + pathJ, actionI.getOwner().getLabel(), actionJ.getOwner().getLabel()); + temporaryBadActionMap.put(actionI, new ConflictException(exception)); + temporaryBadActionMap.put(actionJ, new ConflictException(exception)); + } + } else { // pathJ didn't have prefix pathI, so no conflict possible for pathI. + break; + } + } + } + this.badActionMap = ImmutableMap.copyOf(temporaryBadActionMap); + } + + /** + * Simultaneously construct an action graph for all the actions in Skyframe and a map from + * {@link PathFragment}s to their respective {@link Artifact}s. We do this in a threadpool to save + * around 1.5 seconds on a mid-sized build versus a single-threaded operation. + */ + private static Pair<ActionGraph, SortedMap<PathFragment, Artifact>> + constructActionGraphAndPathMap( + Iterable<ActionLookupValue> values, + ConcurrentMap<Action, ConflictException> badActionMap) throws InterruptedException { + MutableActionGraph actionGraph = new MapBasedActionGraph(); + ConcurrentNavigableMap<PathFragment, Artifact> artifactPathMap = new ConcurrentSkipListMap<>(); + // Action graph construction is CPU-bound. + int numJobs = Runtime.getRuntime().availableProcessors(); + // No great reason for expecting 5000 action lookup values, but not worth counting size of + // values. + Sharder<ActionLookupValue> actionShards = new Sharder<>(numJobs, 5000); + for (ActionLookupValue value : values) { + actionShards.add(value); + } + + ThrowableRecordingRunnableWrapper wrapper = new ThrowableRecordingRunnableWrapper( + "SkyframeActionExecutor#constructActionGraphAndPathMap"); + + ExecutorService executor = Executors.newFixedThreadPool( + numJobs, + new ThreadFactoryBuilder().setNameFormat("ActionLookupValue Processor %d").build()); + for (List<ActionLookupValue> shard : actionShards) { + executor.execute( + wrapper.wrap(actionRegistration(shard, actionGraph, artifactPathMap, badActionMap))); + } + boolean interrupted = ExecutorShutdownUtil.interruptibleShutdown(executor); + Throwables.propagateIfPossible(wrapper.getFirstThrownError()); + if (interrupted) { + throw new InterruptedException(); + } + return Pair.<ActionGraph, SortedMap<PathFragment, Artifact>>of(actionGraph, artifactPathMap); + } + + private static Runnable actionRegistration( + final List<ActionLookupValue> values, + final MutableActionGraph actionGraph, + final ConcurrentMap<PathFragment, Artifact> artifactPathMap, + final ConcurrentMap<Action, ConflictException> badActionMap) { + return new Runnable() { + @Override + public void run() { + for (ActionLookupValue value : values) { + Set<Action> registeredActions = new HashSet<>(); + for (Map.Entry<Artifact, Action> entry : value.getMapForConsistencyCheck().entrySet()) { + Action action = entry.getValue(); + // We have an entry for each <action, artifact> pair. Only try to register each action + // once. + if (registeredActions.add(action)) { + try { + actionGraph.registerAction(action); + } catch (ActionConflictException e) { + Exception oldException = badActionMap.put(action, new ConflictException(e)); + Preconditions.checkState(oldException == null, + "%s | %s | %s", action, e, oldException); + // We skip the rest of the loop, and do not add the path->artifact mapping for this + // artifact below -- we don't need to check it since this action is already in + // error. + continue; + } + } + artifactPathMap.put(entry.getKey().getExecPath(), entry.getKey()); + } + } + } + }; + } + + void prepareForExecution(Executor executor, boolean keepGoing, + boolean explain, ActionCacheChecker actionCacheChecker) { + this.executorEngine = Preconditions.checkNotNull(executor); + + // Start with a new map each build so there's no issue with internal resizing. + this.buildActionMap = Maps.newConcurrentMap(); + this.keepGoing = keepGoing; + this.hadExecutionError = false; + this.actionCacheChecker = Preconditions.checkNotNull(actionCacheChecker); + // Don't cache possibly stale data from the last build. + undeclaredInputsMetadata = new ConcurrentHashMap<>(); + this.explain = explain; + } + + public void setActionLogBufferPathGenerator( + ActionLogBufferPathGenerator actionLogBufferPathGenerator) { + this.actionLogBufferPathGenerator = actionLogBufferPathGenerator; + } + + void executionOver() { + // This transitively holds a bunch of heavy objects, so it's important to clear it at the + // end of a build. + this.executorEngine = null; + } + + File getExecRoot() { + return executorEngine.getExecRoot().getPathFile(); + } + + boolean probeActionExecution(Action action) { + return buildActionMap.containsKey(action.getPrimaryOutput()); + } + + /** + * Executes the provided action on the current thread. Returns the ActionExecutionValue with the + * result, either computed here or already computed on another thread. + * + * <p>For use from {@link ArtifactFunction} only. + */ + ActionExecutionValue executeAction(Action action, FileAndMetadataCache graphFileCache, + Token token, long actionStartTime, + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Exception exception = badActionMap.get(action); + if (exception != null) { + // If action had a conflict with some other action in the graph, report it now. + reportError(exception.getMessage(), exception, action, null); + } + Artifact primaryOutput = action.getPrimaryOutput(); + FutureTask<ActionExecutionValue> actionTask = + new FutureTask<>(new ActionRunner(action, graphFileCache, token, + actionStartTime, actionExecutionContext)); + // Check to see if another action is already executing/has executed this value. + Pair<Action, FutureTask<ActionExecutionValue>> oldAction = + buildActionMap.putIfAbsent(primaryOutput, Pair.of(action, actionTask)); + + if (oldAction == null) { + actionTask.run(); + } else if (action == oldAction.first) { + // We only allow the same action to be executed twice if it discovers inputs. We allow that + // because we need to declare additional dependencies on those new inputs. + Preconditions.checkState(action.discoversInputs(), + "Same action shouldn't execute twice in build: %s", action); + actionTask = oldAction.second; + } else { + Preconditions.checkState(Actions.canBeShared(oldAction.first, action), + "Actions cannot be shared: %s %s", oldAction.first, action); + // Wait for other action to finish, so any actions that depend on its outputs can execute. + actionTask = oldAction.second; + } + try { + return actionTask.get(); + } catch (ExecutionException e) { + Throwables.propagateIfPossible(e.getCause(), + ActionExecutionException.class, InterruptedException.class); + throw new IllegalStateException(e); + } finally { + String message = action.getProgressMessage(); + if (message != null) { + // Tell the receiver that the action has completed *before* telling the reporter. + // This way the latter will correctly show the number of completed actions when task + // completion messages are enabled (--show_task_finish). + if (completionReceiver != null) { + completionReceiver.actionCompleted(action); + } + reporter.finishTask(null, prependExecPhaseStats(message)); + } + } + } + + /** + * Returns an ActionExecutionContext suitable for executing a particular action. The caller should + * pass the returned context to {@link #executeAction}, and any other method that needs to execute + * tasks related to that action. + */ + ActionExecutionContext constructActionExecutionContext(final FileAndMetadataCache graphFileCache, + MetadataHandler metadataHandler) { + FileOutErr fileOutErr = actionLogBufferPathGenerator.generate(); + return new ActionExecutionContext( + executorEngine, + new DelegatingPairFileCache(graphFileCache, perBuildFileCache), + metadataHandler, + fileOutErr, + new MiddlemanExpander() { + @Override + public void expand(Artifact middlemanArtifact, + Collection<? super Artifact> output) { + // Legacy code is more permissive regarding "mm" in that it expands any middleman, + // not just inputs of this action. Skyframe doesn't have access to a global action + // graph, therefore this implementation can't expand any middleman, only the + // inputs of this action. + // This is fine though: actions should only hold references to their input + // artifacts, otherwise hermeticity would be violated. + output.addAll(graphFileCache.expandInputMiddleman(middlemanArtifact)); + } + }); + } + + /** + * Returns a MetadataHandler for use when executing a particular action. The caller can pass the + * returned handler in whenever a MetadataHandler is needed in the course of executing the action. + */ + MetadataHandler constructMetadataHandler(MetadataHandler graphFileCache) { + return new UndeclaredInputHandler(graphFileCache, undeclaredInputsMetadata); + } + + /** + * Checks the action cache to see if {@code action} needs to be executed, or is up to date. + * Returns a token with the semantics of {@link ActionCacheChecker#getTokenIfNeedToExecute}: null + * if the action is up to date, and non-null if it needs to be executed, in which case that token + * should be provided to the ActionCacheChecker after execution. + */ + Token checkActionCache(Action action, MetadataHandler metadataHandler, long actionStartTime) { + profiler.startTask(ProfilerTask.ACTION_CHECK, action); + Token token = actionCacheChecker.getTokenIfNeedToExecute( + action, explain ? reporter : null, metadataHandler); + profiler.completeTask(ProfilerTask.ACTION_CHECK); + if (token == null) { + boolean eventPosted = false; + // Notify BlazeRuntimeStatistics about the action middleman 'execution'. + if (action.getActionType().isMiddleman()) { + postEvent(new ActionMiddlemanEvent(action, actionStartTime)); + eventPosted = true; + } + + if (action instanceof NotifyOnActionCacheHit) { + NotifyOnActionCacheHit notify = (NotifyOnActionCacheHit) action; + notify.actionCacheHit(executorEngine); + } + + // We still need to check the outputs so that output file data is available to the value. + checkOutputs(action, metadataHandler); + if (!eventPosted) { + postEvent(new CachedActionEvent(action, actionStartTime)); + } + } + return token; + } + + /** + * Perform dependency discovery for action, which must discover its inputs. + * + * <p>This method is just a wrapper around {@link Action#discoverInputs} that properly processes + * any ActionExecutionException thrown before rethrowing it to the caller. + */ + void discoverInputs(Action action, ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + try { + action.discoverInputs(actionExecutionContext); + } catch (ActionExecutionException e) { + processAndThrow(e, action, actionExecutionContext.getFileOutErr()); + } + } + + /** + * This method should be called if the builder encounters an error during + * execution. This allows the builder to record that it encountered at + * least one error, and may make it swallow its output to prevent + * spamming the user any further. + */ + private void recordExecutionError() { + hadExecutionError = true; + } + + /** + * Returns true if the Builder is winding down (i.e. cancelling outstanding + * actions and preparing to abort.) + * The builder is winding down iff: + * <ul> + * <li>we had an execution error + * <li>we are not running with --keep_going + * </ul> + */ + private boolean isBuilderAborting() { + return hadExecutionError && !keepGoing; + } + + void setFileCache(ActionInputFileCache fileCache) { + this.perBuildFileCache = fileCache; + } + + private class ActionRunner implements Callable<ActionExecutionValue> { + private final Action action; + private final FileAndMetadataCache graphFileCache; + private Token token; + private long actionStartTime; + private ActionExecutionContext actionExecutionContext; + + ActionRunner(Action action, FileAndMetadataCache graphFileCache, Token token, + long actionStartTime, + ActionExecutionContext actionExecutionContext) { + this.action = action; + this.graphFileCache = graphFileCache; + this.token = token; + this.actionStartTime = actionStartTime; + this.actionExecutionContext = actionExecutionContext; + } + + @Override + public ActionExecutionValue call() throws ActionExecutionException, InterruptedException { + profiler.startTask(ProfilerTask.ACTION, action); + try { + if (actionCacheChecker.isActionExecutionProhibited(action)) { + // We can't execute an action (e.g. because --check_???_up_to_date option was used). Fail + // the build instead. + synchronized (reporter) { + TargetOutOfDateException e = new TargetOutOfDateException(action); + reporter.handle(Event.error(e.getMessage())); + recordExecutionError(); + throw e; + } + } + + String message = action.getProgressMessage(); + if (message != null) { + reporter.startTask(null, prependExecPhaseStats(message)); + } + statusReporterRef.get().setPreparing(action); + + createOutputDirectories(action); + + prepareScheduleExecuteAndCompleteAction(action, token, + actionExecutionContext, actionStartTime); + return new ActionExecutionValue( + graphFileCache.getOutputData(), graphFileCache.getAdditionalOutputData()); + } finally { + profiler.completeTask(ProfilerTask.ACTION); + } + } + } + + private void createOutputDirectories(Action action) throws ActionExecutionException { + try { + Set<Path> done = new HashSet<>(); // avoid redundant calls for the same directory. + for (Artifact outputFile : action.getOutputs()) { + Path outputDir = outputFile.getPath().getParentDirectory(); + if (done.add(outputDir)) { + try { + createDirectoryAndParents(outputDir); + continue; + } catch (IOException e) { + /* Fall through to plan B. */ + } + + // Possibly some direct ancestors are not directories. In that case, we unlink all the + // ancestors until we reach a directory, then try again. This handles the case where a + // file becomes a directory, either from one build to another, or within a single build. + // + // Symlinks should not be followed so in order to clean up symlinks pointing to Fileset + // outputs from previous builds. See bug [incremental build of Fileset fails if + // Fileset.out was changed to be a subdirectory of the old value]. + try { + for (Path p = outputDir; !p.isDirectory(Symlinks.NOFOLLOW); + p = p.getParentDirectory()) { + // p may be a file or dangling symlink, or a symlink to an old Fileset output + p.delete(); // throws IOException + } + createDirectoryAndParents(outputDir); + } catch (IOException e) { + throw new ActionExecutionException( + "failed to create output directory '" + outputDir + "'", e, action, false); + } + } + } + } catch (ActionExecutionException ex) { + printError(ex.getMessage(), action, null); + throw ex; + } + } + + private String prependExecPhaseStats(String message) { + if (progressSupplier != null) { + // Prints a progress message like: + // [2608/6445] Compiling foo/bar.cc [host] + return progressSupplier.getProgressString() + " " + message; + } else { + // progressSupplier may be null in tests + return message; + } + } + + /** + * Prepare, schedule, execute, and then complete the action. + * When this function is called, we know that this action needs to be executed. + * This function will prepare for the action's execution (i.e. delete the outputs); + * schedule its execution; execute the action; + * and then do some post-execution processing to complete the action: + * set the outputs readonly and executable, and insert the action results in the + * action cache. + * + * @param action The action to execute + * @param token The non-null token returned by dependencyChecker.getTokenIfNeedToExecute() + * @param context services in the scope of the action + * @param actionStartTime time when we started the first phase of the action execution. + * @throws ActionExecutionException if the execution of the specified action + * failed for any reason. + * @throws InterruptedException if the thread was interrupted. + */ + private void prepareScheduleExecuteAndCompleteAction(Action action, Token token, + ActionExecutionContext context, long actionStartTime) + throws ActionExecutionException, InterruptedException { + Preconditions.checkNotNull(token, action); + // Delete the metadataHandler's cache of the action's outputs, since they are being deleted. + context.getMetadataHandler().discardMetadata(action.getOutputs()); + // Delete the outputs before executing the action, just to ensure that + // the action really does produce the outputs. + try { + action.prepare(context.getExecutor().getExecRoot()); + } catch (IOException e) { + reportError("failed to delete output files before executing action", e, action, null); + } + + postEvent(new ActionStartedEvent(action, actionStartTime)); + ResourceSet estimate = action.estimateResourceConsumption(executorEngine); + ActionExecutionStatusReporter statusReporter = statusReporterRef.get(); + try { + if (estimate == null || estimate == ResourceSet.ZERO) { + statusReporter.setRunningFromBuildData(action); + } else { + // If estimated resource consumption is null, action will manually call + // resource manager when it knows what resources are needed. + resourceManager.acquireResources(action, estimate); + } + boolean outputDumped = executeActionTask(action, context); + completeAction(action, token, context.getMetadataHandler(), + context.getFileOutErr(), outputDumped); + } finally { + if (estimate != null) { + resourceManager.releaseResources(action, estimate); + } + statusReporter.remove(action); + postEvent(new ActionCompletionEvent(action)); + } + } + + private ActionExecutionException processAndThrow( + ActionExecutionException e, Action action, FileOutErr outErrBuffer) + throws ActionExecutionException { + reportActionExecution(action, e, outErrBuffer); + boolean reported = reportErrorIfNotAbortingMode(e, outErrBuffer); + + ActionExecutionException toThrow = e; + if (reported){ + // If we already printed the error for the exception we mark it as already reported + // so that we do not print it again in upper levels. + // Note that we need to report it here since we want immediate feedback of the errors + // and in some cases the upper-level printing mechanism only prints one of the errors. + toThrow = new AlreadyReportedActionExecutionException(e); + } + + // Now, rethrow the exception. + // This can have two effects: + // If we're still building, the exception will get retrieved by the + // completor and rethrown. + // If we're aborting, the exception will never be retrieved from the + // completor, since the completor is waiting for all outstanding jobs + // to finish. After they have finished, it will only rethrow the + // exception that initially caused it to abort will and not check the + // exit status of any actions that had finished in the meantime. + throw toThrow; + } + + /** + * Execute the specified action, in a profiler task. + * The caller is responsible for having already checked that we need to + * execute it and for acquiring/releasing any scheduling locks needed. + * + * <p>This is thread-safe so long as you don't try to execute the same action + * twice at the same time (or overlapping times). + * May execute in a worker thread. + * + * @throws ActionExecutionException if the execution of the specified action + * failed for any reason. + * @throws InterruptedException if the thread was interrupted. + * @return true if the action output was dumped, false otherwise. + */ + private boolean executeActionTask(Action action, ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + profiler.startTask(ProfilerTask.ACTION_EXECUTE, action); + // ActionExecutionExceptions that occur as the thread is interrupted are + // assumed to be a result of that, so we throw InterruptedException + // instead. + FileOutErr outErrBuffer = actionExecutionContext.getFileOutErr(); + try { + action.execute(actionExecutionContext); + + // Action terminated fine, now report the output. + // The .showOutput() method is not necessarily a quick check: in its + // current implementation it uses regular expression matching. + if (outErrBuffer.hasRecordedOutput() + && (action.showsOutputUnconditionally() + || reporter.showOutput(Label.print(action.getOwner().getLabel())))) { + dumpRecordedOutErr(action, outErrBuffer); + return true; + } + // Defer reporting action success until outputs are checked + } catch (ActionExecutionException e) { + processAndThrow(e, action, outErrBuffer); + } finally { + profiler.completeTask(ProfilerTask.ACTION_EXECUTE); + } + return false; + } + + private void completeAction(Action action, Token token, MetadataHandler metadataHandler, + FileOutErr fileOutErr, boolean outputAlreadyDumped) throws ActionExecutionException { + try { + Preconditions.checkState(action.inputsKnown(), + "Action %s successfully executed, but inputs still not known", action); + + profiler.startTask(ProfilerTask.ACTION_COMPLETE, action); + try { + if (!checkOutputs(action, metadataHandler)) { + reportError("not all outputs were created", null, action, + outputAlreadyDumped ? null : fileOutErr); + } + // Prevent accidental stomping on files. + // This will also throw a FileNotFoundException + // if any of the output files doesn't exist. + try { + setOutputsReadOnlyAndExecutable(action, metadataHandler); + } catch (IOException e) { + reportError("failed to set outputs read-only", e, action, null); + } + try { + actionCacheChecker.afterExecution(action, token, metadataHandler); + } catch (IOException e) { + // Skyframe does all the filesystem access needed during the previous calls, and if those + // calls failed, we should already have thrown. So an IOException is impossible here. + throw new IllegalStateException( + "failed to update action cache for " + action.prettyPrint() + + ", but all outputs should already have been checked", e); + } + } finally { + profiler.completeTask(ProfilerTask.ACTION_COMPLETE); + } + reportActionExecution(action, null, fileOutErr); + } catch (ActionExecutionException actionException) { + // Success in execution but failure in completion. + reportActionExecution(action, actionException, fileOutErr); + throw actionException; + } catch (IllegalStateException exception) { + // More serious internal error, but failure still reported. + reportActionExecution(action, + new ActionExecutionException(exception, action, true), fileOutErr); + throw exception; + } + } + + /** + * For each of the action's outputs that is a regular file (not a symbolic + * link or directory), make it read-only and executable. + * + * <p>Making the outputs read-only helps preventing accidental editing of + * them (e.g. in case of generated source code), while making them executable + * helps running generated files (such as generated shell scripts) on the + * command line. + * + * <p>May execute in a worker thread. + * + * <p>Note: setting these bits maintains transparency regarding the locality of the build; + * because the remote execution engine sets them, they should be set for local builds too. + * + * @throws IOException if an I/O error occurred. + */ + private final void setOutputsReadOnlyAndExecutable(Action action, MetadataHandler metadataHandler) + throws IOException { + Preconditions.checkState(!action.getActionType().isMiddleman()); + + for (Artifact output : action.getOutputs()) { + Path path = output.getPath(); + if (metadataHandler.isInjected(output)) { + // We trust the files created by the execution-engine to be non symlinks with expected + // chmod() settings already applied. The follow stanza implies a total of 6 system calls, + // since the UnixFileSystem implementation of setWritable() and setExecutable() both + // do a stat() internally. + continue; + } + if (path.isFile(Symlinks.NOFOLLOW)) { // i.e. regular files only. + path.setWritable(false); + path.setExecutable(true); + } + } + } + + private void reportMissingOutputFile(Action action, Artifact output, Reporter reporter, + boolean isSymlink) { + boolean genrule = action.getMnemonic().equals("Genrule"); + String prefix = (genrule ? "declared output '" : "output '") + output.prettyPrint() + "' "; + if (isSymlink) { + reporter.handle(Event.error( + action.getOwner().getLocation(), prefix + "is a dangling symbolic link")); + } else { + String suffix = genrule ? " by genrule. This is probably " + + "because the genrule actually didn't create this output, or because the output was a " + + "directory and the genrule was run remotely (note that only the contents of " + + "declared file outputs are copied from genrules run remotely)" : ""; + reporter.handle(Event.error( + action.getOwner().getLocation(), prefix + "was not created" + suffix)); + } + } + + /** + * Validates that all action outputs were created. + * + * @return false if some outputs are missing, true - otherwise. + */ + private boolean checkOutputs(Action action, MetadataHandler metadataHandler) { + boolean success = true; + for (Artifact output : action.getOutputs()) { + if (!metadataHandler.artifactExists(output)) { + reportMissingOutputFile(action, output, reporter, output.getPath().isSymbolicLink()); + success = false; + } + } + return success; + } + + private void postEvent(Object event) { + EventBus bus = eventBus.get(); + if (bus != null) { + bus.post(event); + } + } + + /** + * Convenience function for reporting that the action failed due to a + * the exception cause, if there is an additional explanatory message that + * clarifies the message of the exception. Combines the user-provided message + * and the exceptions' message and reports the combination as error. + * Then, throws an ActionExecutionException with the reported error as + * message and the provided exception as the cause. + * + * @param message A small text that explains why the action failed + * @param cause The exception that caused the action to fail + * @param action The action that failed + * @param actionOutput The output of the failed Action. + * May be null, if there is no output to display + */ + private void reportError(String message, Throwable cause, Action action, FileOutErr actionOutput) + throws ActionExecutionException { + ActionExecutionException ex; + if (cause == null) { + ex = new ActionExecutionException(message, action, false); + } else { + ex = new ActionExecutionException(message, cause, action, false); + } + printError(ex.getMessage(), action, actionOutput); + throw ex; + } + + /** + * For the action 'action' that failed due to 'ex' with the output + * 'actionOutput', notify the user about the error. To notify the user, the + * method first displays the output of the action and then reports an error + * via the reporter. The method ensures that the two messages appear next to + * each other by locking the outErr object where the output is displayed. + * + * @param message The reason why the action failed + * @param action The action that failed, must not be null. + * @param actionOutput The output of the failed Action. + * May be null, if there is no output to display + */ + private void printError(String message, Action action, FileOutErr actionOutput) { + synchronized (reporter) { + if (actionOutput != null && actionOutput.hasRecordedOutput()) { + dumpRecordedOutErr(action, actionOutput); + } + if (keepGoing) { + message = "Couldn't " + describeAction(action) + ": " + message; + } + reporter.handle(Event.error(action.getOwner().getLocation(), message)); + recordExecutionError(); + } + } + + /** Describe an action, for use in error messages. */ + private static String describeAction(Action action) { + if (action.getOutputs().isEmpty()) { + return "run " + action.prettyPrint(); + } else if (action.getActionType().isMiddleman()) { + return "build " + action.prettyPrint(); + } else { + return "build file " + action.getPrimaryOutput().prettyPrint(); + } + } + + /** + * Dump the output from the action. + * + * @param action The action whose output is being dumped + * @param outErrBuffer The OutErr that recorded the actions output + */ + private void dumpRecordedOutErr(Action action, FileOutErr outErrBuffer) { + StringBuilder message = new StringBuilder(""); + message.append("From "); + message.append(action.describe()); + message.append(":"); + + // Synchronize this on the reporter, so that the output from multiple + // actions will not be interleaved. + synchronized (reporter) { + // Only print the output if we're not winding down. + if (isBuilderAborting()) { + return; + } + reporter.handle(Event.info(message.toString())); + + OutErr outErr = this.reporter.getOutErr(); + outErrBuffer.dumpOutAsLatin1(outErr.getOutputStream()); + outErrBuffer.dumpErrAsLatin1(outErr.getErrorStream()); + } + } + + private void reportActionExecution(Action action, + ActionExecutionException exception, FileOutErr outErr) { + String stdout = null; + String stderr = null; + + if (outErr.hasRecordedStdout()) { + stdout = outErr.getOutputFile().toString(); + } + if (outErr.hasRecordedStderr()) { + stderr = outErr.getErrorFile().toString(); + } + postEvent(new ActionExecutedEvent(action, exception, stdout, stderr)); + } + + /** + * Returns true if the exception was reported. False otherwise. Currently this is a copy of what + * we did in pre-Skyframe execution. The main implication is that we are printing the error to the + * top level reporter instead of the action reporter. Because of that Skyframe values do not know + * about the errors happening in the execution phase. Even if we change in the future to log to + * the action reporter (that would be done in ActionExecutionFunction.compute() when we get an + * ActionExecutionException), we probably do not want to also store the StdErr output, so + * dumpRecordedOutErr() should still be called here. + */ + private boolean reportErrorIfNotAbortingMode(ActionExecutionException ex, + FileOutErr outErrBuffer) { + // For some actions (e.g. many local actions) the pollInterruptedStatus() + // won't notice that we had an interrupted job. It will continue. + // For that reason we must take care to NOT report errors if we're + // in the 'aborting' mode: Any cancelled action would show up here. + // For some actions (e.g. many local actions) the pollInterruptedStatus() + // won't notice that we had an interrupted job. It will continue. + // For that reason we must take care to NOT report errors if we're + // in the 'aborting' mode: Any cancelled action would show up here. + synchronized (this.reporter) { + if (!isBuilderAborting()) { + // Oops. The action aborted. Report the problem. + printError(ex.getMessage(), ex.getAction(), outErrBuffer); + return true; + } + } + return false; + } + + /** An object supplying data for action execution progress reporting. */ + public interface ProgressSupplier { + /** Returns the progress string to prefix action execution messages with. */ + String getProgressString(); + } + + /** An object that can be notified about action completion. */ + public interface ActionCompletedReceiver { + /** Receives a completed action. */ + void actionCompleted(Action action); + } + + public void setActionExecutionProgressReportingObjects( + @Nullable ProgressSupplier progressSupplier, + @Nullable ActionCompletedReceiver completionReceiver) { + this.progressSupplier = progressSupplier; + this.completionReceiver = completionReceiver; + } + + private static class DelegatingPairFileCache implements ActionInputFileCache { + private final ActionInputFileCache perActionCache; + private final ActionInputFileCache perBuildFileCache; + + private DelegatingPairFileCache(ActionInputFileCache mainCache, + ActionInputFileCache perBuildFileCache) { + this.perActionCache = mainCache; + this.perBuildFileCache = perBuildFileCache; + } + + @Override + public ByteString getDigest(ActionInput actionInput) throws IOException { + ByteString digest = perActionCache.getDigest(actionInput); + return digest != null ? digest : perBuildFileCache.getDigest(actionInput); + } + + @Override + public long getSizeInBytes(ActionInput actionInput) throws IOException { + long size = perActionCache.getSizeInBytes(actionInput); + return size > -1 ? size : perBuildFileCache.getSizeInBytes(actionInput); + } + + @Override + public boolean contentsAvailableLocally(ByteString digest) { + return perActionCache.contentsAvailableLocally(digest) + || perBuildFileCache.contentsAvailableLocally(digest); + } + + @Nullable + @Override + public File getFileFromDigest(ByteString digest) throws IOException { + File file = perActionCache.getFileFromDigest(digest); + return file != null ? file : perBuildFileCache.getFileFromDigest(digest); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java new file mode 100644 index 0000000..857c231 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java
@@ -0,0 +1,510 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.actions.ArtifactPrefixConflictException; +import com.google.devtools.build.lib.actions.MutableActionGraph; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.AnalysisFailureEvent; +import com.google.devtools.build.lib.analysis.Aspect; +import com.google.devtools.build.lib.analysis.CachingAnalysisEnvironment; +import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.ConfiguredTargetFactory; +import com.google.devtools.build.lib.analysis.LabelAndConfiguration; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.ViewCreationFailedException; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; +import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.skyframe.ActionLookupValue.ActionLookupKey; +import com.google.devtools.build.lib.skyframe.BuildInfoCollectionValue.BuildInfoKeyAndConfig; +import com.google.devtools.build.lib.skyframe.ConfiguredTargetFunction.ConfiguredValueCreationException; +import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ConflictException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.CycleInfo; +import com.google.devtools.build.skyframe.ErrorInfo; +import com.google.devtools.build.skyframe.EvaluationProgressReceiver; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.SkyFunction.Environment; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Skyframe-based driver of analysis. + * + * <p>Covers enough functionality to work as a substitute for {@code BuildView#configureTargets}. + */ +public final class SkyframeBuildView { + + private final ConfiguredTargetFactory factory; + private final ArtifactFactory artifactFactory; + @Nullable private EventHandler warningListener; + private final SkyframeExecutor skyframeExecutor; + private final Runnable legacyDataCleaner; + private final BinTools binTools; + private boolean enableAnalysis = false; + + // This hack allows us to see when a configured target has been invalidated, and thus when the set + // of artifact conflicts needs to be recomputed (whenever a configured target has been invalidated + // or newly evaluated). + private final EvaluationProgressReceiver invalidationReceiver = + new ConfiguredTargetValueInvalidationReceiver(); + private final Set<SkyKey> evaluatedConfiguredTargets = Sets.newConcurrentHashSet(); + // Used to see if checks of graph consistency need to be done after analysis. + private volatile boolean someConfiguredTargetEvaluated = false; + + // We keep the set of invalidated configuration targets so that we can know if something + // has been invalidated after graph pruning has been executed. + private Set<ConfiguredTargetValue> dirtyConfiguredTargets = Sets.newConcurrentHashSet(); + private volatile boolean anyConfiguredTargetDeleted = false; + private SkyKey configurationKey = null; + + public SkyframeBuildView(ConfiguredTargetFactory factory, + ArtifactFactory artifactFactory, + SkyframeExecutor skyframeExecutor, Runnable legacyDataCleaner, BinTools binTools) { + this.factory = factory; + this.artifactFactory = artifactFactory; + this.skyframeExecutor = skyframeExecutor; + this.legacyDataCleaner = legacyDataCleaner; + this.binTools = binTools; + skyframeExecutor.setArtifactFactoryAndBinTools(artifactFactory, binTools); + } + + public void setWarningListener(@Nullable EventHandler warningListener) { + this.warningListener = warningListener; + } + + public void setConfigurationSkyKey(SkyKey skyKey) { + this.configurationKey = skyKey; + } + + public void resetEvaluatedConfiguredTargetKeysSet() { + evaluatedConfiguredTargets.clear(); + } + + public Set<SkyKey> getEvaluatedTargetKeys() { + return ImmutableSet.copyOf(evaluatedConfiguredTargets); + } + + private void setDeserializedArtifactOwners() throws ViewCreationFailedException { + Map<PathFragment, Artifact> deserializedArtifactMap = + artifactFactory.getDeserializedArtifacts(); + Set<Artifact> deserializedArtifacts = new HashSet<>(); + for (Artifact artifact : deserializedArtifactMap.values()) { + if (!artifact.getExecPath().getBaseName().endsWith(".gcda")) { + // gcda files are classified as generated artifacts, but are not actually generated. All + // others need owners. + deserializedArtifacts.add(artifact); + } + } + if (deserializedArtifacts.isEmpty()) { + // If there are no deserialized artifacts to process, don't pay the price of iterating over + // the graph. + return; + } + for (Map.Entry<SkyKey, ActionLookupValue> entry : + skyframeExecutor.getActionLookupValueMap().entrySet()) { + for (Action action : entry.getValue().getActionsForFindingArtifactOwners()) { + for (Artifact output : action.getOutputs()) { + Artifact deserializedArtifact = deserializedArtifactMap.get(output.getExecPath()); + if (deserializedArtifact != null) { + deserializedArtifact.setArtifactOwner((ActionLookupKey) entry.getKey().argument()); + deserializedArtifacts.remove(deserializedArtifact); + } + } + } + } + if (!deserializedArtifacts.isEmpty()) { + throw new ViewCreationFailedException("These artifacts were read in from the FDO profile but" + + " have no generating action that could be found. If you are confident that your profile was" + + " collected from the same source state at which you're building, please report this:\n" + + Artifact.asExecPaths(deserializedArtifacts)); + } + artifactFactory.clearDeserializedArtifacts(); + } + + /** + * Analyzes the specified targets using Skyframe as the driving framework. + * + * @return the configured targets that should be built + */ + public Collection<ConfiguredTarget> configureTargets(List<ConfiguredTargetKey> values, + EventBus eventBus, boolean keepGoing) + throws InterruptedException, ViewCreationFailedException { + enableAnalysis(true); + EvaluationResult<ConfiguredTargetValue> result; + try { + result = skyframeExecutor.configureTargets(values, keepGoing); + } finally { + enableAnalysis(false); + } + // For Skyframe m1, note that we already reported action conflicts during action registration + // in the legacy action graph. + ImmutableMap<Action, ConflictException> badActions = skyframeExecutor.findArtifactConflicts(); + + // Filter out all CTs that have a bad action and convert to a list of configured targets. This + // code ensures that the resulting list of configured targets has the same order as the incoming + // list of values, i.e., that the order is deterministic. + Collection<ConfiguredTarget> goodCts = Lists.newArrayListWithCapacity(values.size()); + for (ConfiguredTargetKey value : values) { + ConfiguredTargetValue ctValue = result.get(ConfiguredTargetValue.key(value)); + if (ctValue == null) { + continue; + } + goodCts.add(ctValue.getConfiguredTarget()); + } + + if (!result.hasError() && badActions.isEmpty()) { + setDeserializedArtifactOwners(); + return goodCts; + } + + // --nokeep_going so we fail with an exception for the first error. + // TODO(bazel-team): We might want to report the other errors through the event bus but + // for keeping this code in parity with legacy we just report the first error for now. + if (!keepGoing) { + for (Map.Entry<Action, ConflictException> bad : badActions.entrySet()) { + ConflictException ex = bad.getValue(); + try { + ex.rethrowTyped(); + } catch (MutableActionGraph.ActionConflictException ace) { + ace.reportTo(skyframeExecutor.getReporter()); + String errorMsg = "Analysis of target '" + bad.getKey().getOwner().getLabel() + + "' failed; build aborted"; + throw new ViewCreationFailedException(errorMsg); + } catch (ArtifactPrefixConflictException apce) { + skyframeExecutor.getReporter().handle(Event.error(apce.getMessage())); + } + throw new ViewCreationFailedException(ex.getMessage()); + } + + Map.Entry<SkyKey, ErrorInfo> error = result.errorMap().entrySet().iterator().next(); + SkyKey topLevel = error.getKey(); + ErrorInfo errorInfo = error.getValue(); + assertSaneAnalysisError(errorInfo, topLevel); + skyframeExecutor.getCyclesReporter().reportCycles(errorInfo.getCycleInfo(), topLevel, + skyframeExecutor.getReporter()); + Throwable cause = errorInfo.getException(); + Preconditions.checkState(cause != null || !Iterables.isEmpty(errorInfo.getCycleInfo()), + errorInfo); + String errorMsg = "Analysis of target '" + ConfiguredTargetValue.extractLabel(topLevel) + + "' failed; build aborted"; + throw new ViewCreationFailedException(errorMsg); + } + + // --keep_going : We notify the error and return a ConfiguredTargetValue + for (Map.Entry<SkyKey, ErrorInfo> errorEntry : result.errorMap().entrySet()) { + if (values.contains(errorEntry.getKey().argument())) { + SkyKey errorKey = errorEntry.getKey(); + ConfiguredTargetKey label = (ConfiguredTargetKey) errorKey.argument(); + ErrorInfo errorInfo = errorEntry.getValue(); + assertSaneAnalysisError(errorInfo, errorKey); + + skyframeExecutor.getCyclesReporter().reportCycles(errorInfo.getCycleInfo(), errorKey, + skyframeExecutor.getReporter()); + // We try to get the root cause key first from ErrorInfo rootCauses. If we don't have one + // we try to use the cycle culprit if the error is a cycle. Otherwise we use the top-level + // error key. + Label root; + if (!Iterables.isEmpty(errorEntry.getValue().getRootCauses())) { + SkyKey culprit = Preconditions.checkNotNull(Iterables.getFirst( + errorEntry.getValue().getRootCauses(), null)); + root = ((ConfiguredTargetKey) culprit.argument()).getLabel(); + } else { + root = maybeGetConfiguredTargetCycleCulprit(errorInfo.getCycleInfo()); + } + if (warningListener != null) { + warningListener.handle(Event.warn("errors encountered while analyzing target '" + + label + "': it will not be built")); + } + eventBus.post(new AnalysisFailureEvent( + LabelAndConfiguration.of(label.getLabel(), label.getConfiguration()), root)); + } + } + + Collection<Exception> reportedExceptions = Sets.newHashSet(); + for (Map.Entry<Action, ConflictException> bad : badActions.entrySet()) { + ConflictException ex = bad.getValue(); + try { + ex.rethrowTyped(); + } catch (MutableActionGraph.ActionConflictException ace) { + ace.reportTo(skyframeExecutor.getReporter()); + if (warningListener != null) { + warningListener.handle(Event.warn("errors encountered while analyzing target '" + + bad.getKey().getOwner().getLabel() + "': it will not be built")); + } + } catch (ArtifactPrefixConflictException apce) { + if (reportedExceptions.add(apce)) { + skyframeExecutor.getReporter().handle(Event.error(apce.getMessage())); + } + } + } + + if (!badActions.isEmpty()) { + // In order to determine the set of configured targets transitively error free from action + // conflict issues, we run a post-processing update() that uses the bad action map. + EvaluationResult<PostConfiguredTargetValue> actionConflictResult = + skyframeExecutor.postConfigureTargets(values, keepGoing, badActions); + + goodCts = Lists.newArrayListWithCapacity(values.size()); + for (ConfiguredTargetKey value : values) { + PostConfiguredTargetValue postCt = + actionConflictResult.get(PostConfiguredTargetValue.key(value)); + if (postCt != null) { + goodCts.add(postCt.getCt()); + } + } + } + setDeserializedArtifactOwners(); + return goodCts; + } + + @Nullable + Label maybeGetConfiguredTargetCycleCulprit(Iterable<CycleInfo> cycleInfos) { + for (CycleInfo cycleInfo : cycleInfos) { + SkyKey culprit = Iterables.getFirst(cycleInfo.getCycle(), null); + if (culprit == null) { + continue; + } + if (culprit.functionName().equals(SkyFunctions.CONFIGURED_TARGET)) { + return ((LabelAndConfiguration) culprit.argument()).getLabel(); + } + } + return null; + } + + private static void assertSaneAnalysisError(ErrorInfo errorInfo, SkyKey key) { + Throwable cause = errorInfo.getException(); + if (cause != null) { + // We should only be trying to configure targets when the loading phase succeeds, meaning + // that the only errors should be analysis errors. + Preconditions.checkState(cause instanceof ConfiguredValueCreationException, + "%s -> %s", key, errorInfo); + } + } + + ArtifactFactory getArtifactFactory() { + return artifactFactory; + } + + @Nullable + EventHandler getWarningListener() { + return warningListener; + } + + /** + * Because we don't know what build-info artifacts this configured target may request, we + * conservatively register a dep on all of them. + */ + // TODO(bazel-team): Allow analysis to return null so the value builder can exit and wait for a + // restart deps are not present. + private boolean getWorkspaceStatusValues(Environment env) { + env.getValue(WorkspaceStatusValue.SKY_KEY); + Map<BuildInfoKey, BuildInfoFactory> buildInfoFactories = + PrecomputedValue.BUILD_INFO_FACTORIES.get(env); + if (buildInfoFactories == null) { + return false; + } + BuildConfigurationCollection configurations = getBuildConfigurationCollection(env); + if (configurations == null) { + return false; + } + // These factories may each create their own build info artifacts, all depending on the basic + // build-info.txt and build-changelist.txt. + List<SkyKey> depKeys = Lists.newArrayList(); + for (BuildInfoKey key : buildInfoFactories.keySet()) { + for (BuildConfiguration config : configurations.getAllConfigurations()) { + if (buildInfoFactories.get(key).isEnabled(config)) { + depKeys.add(BuildInfoCollectionValue.key(new BuildInfoKeyAndConfig(key, config))); + } + } + } + env.getValues(depKeys); + return !env.valuesMissing(); + } + + /** Returns null if any build-info values are not ready. */ + @Nullable + CachingAnalysisEnvironment createAnalysisEnvironment(ArtifactOwner owner, + boolean isSystemEnv, boolean extendedSanityChecks, EventHandler eventHandler, + Environment env, boolean allowRegisteringActions) { + if (!getWorkspaceStatusValues(env)) { + return null; + } + return new CachingAnalysisEnvironment( + artifactFactory, owner, isSystemEnv, extendedSanityChecks, eventHandler, env, + allowRegisteringActions, binTools); + } + + /** + * Invokes the appropriate constructor to create a {@link ConfiguredTarget} instance. + * + * <p>For use in {@code ConfiguredTargetFunction}. + * + * <p>Returns null if Skyframe deps are missing or upon certain errors. + */ + @Nullable + ConfiguredTarget createConfiguredTarget(Target target, BuildConfiguration configuration, + CachingAnalysisEnvironment analysisEnvironment, + ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap, + Set<ConfigMatchingProvider> configConditions) + throws InterruptedException { + Preconditions.checkState(enableAnalysis, + "Already in execution phase %s %s", target, configuration); + return factory.createConfiguredTarget(analysisEnvironment, artifactFactory, target, + configuration, prerequisiteMap, configConditions); + } + + @Nullable + public Aspect createAspect( + AnalysisEnvironment env, RuleConfiguredTarget associatedTarget, + ConfiguredAspectFactory aspectFactory, + ListMultimap<Attribute, ConfiguredTarget> prerequisiteMap, + Set<ConfigMatchingProvider> configConditions) { + return factory.createAspect( + env, associatedTarget, aspectFactory, prerequisiteMap, configConditions); + } + + @Nullable + private BuildConfigurationCollection getBuildConfigurationCollection(Environment env) { + ConfigurationCollectionValue configurationsValue = + (ConfigurationCollectionValue) env.getValue(configurationKey); + return configurationsValue == null ? null : configurationsValue.getConfigurationCollection(); + } + + @Nullable + SkyframeDependencyResolver createDependencyResolver(Environment env) { + BuildConfigurationCollection configurations = getBuildConfigurationCollection(env); + return configurations == null ? null : new SkyframeDependencyResolver(env); + } + + /** + * Workaround to clear all legacy data, like the action graph and the artifact factory. We need + * to clear them to avoid conflicts. + * TODO(bazel-team): Remove this workaround. [skyframe-execution] + */ + void clearLegacyData() { + legacyDataCleaner.run(); + } + + /** + * Hack to invalidate actions in legacy action graph when their values are invalidated in + * skyframe. + */ + EvaluationProgressReceiver getInvalidationReceiver() { + return invalidationReceiver; + } + + /** Clear the invalidated configured targets detected during loading and analysis phases. */ + public void clearInvalidatedConfiguredTargets() { + dirtyConfiguredTargets = Sets.newConcurrentHashSet(); + anyConfiguredTargetDeleted = false; + } + + public boolean isSomeConfiguredTargetInvalidated() { + return anyConfiguredTargetDeleted || !dirtyConfiguredTargets.isEmpty(); + } + + /** + * Called from SkyframeExecutor to see whether the graph needs to be checked for artifact + * conflicts. Returns true if some configured target has been evaluated since the last time the + * graph was checked for artifact conflicts (with that last time marked by a call to + * {@link #resetEvaluatedConfiguredTargetFlag()}). + */ + boolean isSomeConfiguredTargetEvaluated() { + Preconditions.checkState(!enableAnalysis); + return someConfiguredTargetEvaluated; + } + + /** + * Called from SkyframeExecutor after the graph is checked for artifact conflicts so that + * the next time {@link #isSomeConfiguredTargetEvaluated} is called, it will return true only if + * some configured target has been evaluated since the last check for artifact conflicts. + */ + void resetEvaluatedConfiguredTargetFlag() { + someConfiguredTargetEvaluated = false; + } + + /** + * {@link #createConfiguredTarget} will only create configured targets if this is set to true. It + * should be set to true before any Skyframe update call that might call into {@link + * #createConfiguredTarget}, and false immediately after the call. Use it to fail-fast in the case + * that a target is requested for analysis not during the analysis phase. + */ + void enableAnalysis(boolean enable) { + this.enableAnalysis = enable; + } + + private class ConfiguredTargetValueInvalidationReceiver implements EvaluationProgressReceiver { + @Override + public void invalidated(SkyValue value, InvalidationState state) { + if (value instanceof ConfiguredTargetValue) { + ConfiguredTargetValue ctValue = (ConfiguredTargetValue) value; + // If the value was just dirtied and not deleted, then it may not be truly invalid, since + // it may later get re-validated. + if (state == InvalidationState.DELETED) { + anyConfiguredTargetDeleted = true; + } else { + dirtyConfiguredTargets.add(ctValue); + } + } + } + + @Override + public void enqueueing(SkyKey skyKey) {} + + @Override + public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) { + if (skyKey.functionName() == SkyFunctions.CONFIGURED_TARGET) { + if (state == EvaluationState.BUILT) { + evaluatedConfiguredTargets.add(skyKey); + // During multithreaded operation, this is only set to true, so no concurrency issues. + someConfiguredTargetEvaluated = true; + } + Preconditions.checkNotNull(value, "%s %s", skyKey, state); + ConfiguredTargetValue ctValue = (ConfiguredTargetValue) value; + dirtyConfiguredTargets.remove(ctValue); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeDependencyResolver.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeDependencyResolver.java new file mode 100644 index 0000000..1c7cbfa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeDependencyResolver.java
@@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.analysis.DependencyResolver; +import com.google.devtools.build.lib.analysis.TargetAndConfiguration; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyFunction.Environment; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import javax.annotation.Nullable; + +/** + * A dependency resolver for use within Skyframe. Loads packages lazily when possible. + */ +public final class SkyframeDependencyResolver extends DependencyResolver { + + private final Environment env; + + public SkyframeDependencyResolver(Environment env) { + this.env = env; + } + + @Override + protected void invalidVisibilityReferenceHook(TargetAndConfiguration value, Label label) { + env.getListener().handle( + Event.error(TargetUtils.getLocationMaybe(value.getTarget()), String.format( + "Label '%s' in visibility attribute does not refer to a package group", label))); + } + + @Override + protected void invalidPackageGroupReferenceHook(TargetAndConfiguration value, Label label) { + env.getListener().handle( + Event.error(TargetUtils.getLocationMaybe(value.getTarget()), String.format( + "label '%s' does not refer to a package group", label))); + } + + @Nullable + @Override + protected Target getTarget(Label label) throws NoSuchThingException { + if (env.getValue(TargetMarkerValue.key(label)) == null) { + return null; + } + SkyKey key = PackageValue.key(label.getPackageIdentifier()); + SkyValue value = env.getValue(key); + if (value == null) { + return null; + } + PackageValue packageValue = (PackageValue) value; + return packageValue.getPackage().getTarget(label.getName()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java new file mode 100644 index 0000000..29b4c23 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -0,0 +1,1476 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionCacheChecker; +import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.ActionLogBufferPathGenerator; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceManager; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.Aspect; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.BuildView.Options; +import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.TopLevelArtifactContext; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationFactory; +import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Preprocessor; +import com.google.devtools.build.lib.packages.RuleVisibility; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.pkgcache.PackageManager; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader; +import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ActionCompletedReceiver; +import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ProgressSupplier; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.ResourceUsage; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.BatchStat; +import com.google.devtools.build.lib.vfs.ModifiedFileSet; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.lib.vfs.UnixGlob; +import com.google.devtools.build.skyframe.BuildDriver; +import com.google.devtools.build.skyframe.CycleInfo; +import com.google.devtools.build.skyframe.CyclesReporter; +import com.google.devtools.build.skyframe.Differencer; +import com.google.devtools.build.skyframe.ErrorInfo; +import com.google.devtools.build.skyframe.EvaluationProgressReceiver; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.Injectable; +import com.google.devtools.build.skyframe.MemoizingEvaluator; +import com.google.devtools.build.skyframe.MemoizingEvaluator.EvaluatorSupplier; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +/** + * A helper object to support Skyframe-driven execution. + * + * <p>This object is mostly used to inject external state, such as the executor engine or + * some additional artifacts (workspace status and build info artifacts) into SkyFunctions + * for use during the build. + */ +public abstract class SkyframeExecutor { + private final EvaluatorSupplier evaluatorSupplier; + protected MemoizingEvaluator memoizingEvaluator; + private final MemoizingEvaluator.EmittedEventState emittedEventState = + new MemoizingEvaluator.EmittedEventState(); + protected final Reporter reporter; + private final PackageFactory pkgFactory; + private final WorkspaceStatusAction.Factory workspaceStatusActionFactory; + private final BlazeDirectories directories; + @Nullable + private BatchStat batchStatter; + + // TODO(bazel-team): Figure out how to handle value builders that block internally. Blocking + // operations may need to be handled in another (bigger?) thread pool. Also, we should detect + // the number of cores and use that as the thread-pool size for CPU-bound operations. + // I just bumped this to 200 to get reasonable execution phase performance; that may cause + // significant overhead for CPU-bound processes (i.e. analysis). [skyframe-analysis] + @VisibleForTesting + public static final int DEFAULT_THREAD_COUNT = 200; + + // Stores Packages between reruns of the PackageFunction (because of missing dependencies, + // within the same evaluate() run) to avoid loading the same package twice (first time loading + // to find subincludes and declare value dependencies). + // TODO(bazel-team): remove this cache once we have skyframe-native package loading + // [skyframe-loading] + private final ConcurrentMap<PackageIdentifier, Package.LegacyBuilder> packageFunctionCache = + Maps.newConcurrentMap(); + private final AtomicInteger numPackagesLoaded = new AtomicInteger(0); + + protected SkyframeBuildView skyframeBuildView; + private EventHandler errorEventListener; + private ActionLogBufferPathGenerator actionLogBufferPathGenerator; + + protected BuildDriver buildDriver; + + // AtomicReferences are used here as mutable boxes shared with value builders. + private final AtomicBoolean showLoadingProgress = new AtomicBoolean(); + protected final AtomicReference<UnixGlob.FilesystemCalls> syscalls = + new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS); + protected final AtomicReference<PathPackageLocator> pkgLocator = + new AtomicReference<>(); + protected final AtomicReference<ImmutableSet<String>> deletedPackages = + new AtomicReference<>(ImmutableSet.<String>of()); + private final AtomicReference<EventBus> eventBus = new AtomicReference<>(); + + private final ImmutableList<BuildInfoFactory> buildInfoFactories; + // Under normal circumstances, the artifact factory persists for the life of a Blaze server, but + // since it is not yet created when we create the value builders, we have to use a supplier, + // initialized when the build view is created. + private final MutableSupplier<ArtifactFactory> artifactFactory = new MutableSupplier<>(); + // Used to give to WriteBuildInfoAction via a supplier. Relying on BuildVariableValue.BUILD_ID + // would be preferable, but we have no way to have the Action depend on that value directly. + // Having the BuildInfoFunction own the supplier is currently not possible either, because then + // it would be invalidated on every build, since it would depend on the build id value. + private MutableSupplier<UUID> buildId = new MutableSupplier<>(); + + protected boolean active = true; + private final PackageManager packageManager; + + private final Preprocessor.Factory.Supplier preprocessorFactorySupplier; + private Preprocessor.Factory preprocessorFactory; + + protected final TimestampGranularityMonitor tsgm; + + private final ResourceManager resourceManager; + + /** Used to lock evaluator on legacy calls to get existing values. */ + private final Object valueLookupLock = new Object(); + private final AtomicReference<ActionExecutionStatusReporter> statusReporterRef = + new AtomicReference<>(); + private final SkyframeActionExecutor skyframeActionExecutor; + protected SkyframeProgressReceiver progressReceiver; + private final AtomicReference<CyclesReporter> cyclesReporter = new AtomicReference<>(); + + private final Set<Path> immutableDirectories; + + private BinTools binTools = null; + private boolean needToInjectEmbeddedArtifacts = true; + private boolean needToInjectPrecomputedValuesForAnalysis = true; + protected int modifiedFiles; + private final Predicate<PathFragment> allowedMissingInputs; + + private final ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions; + private final ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues; + + protected SkyframeIncrementalBuildMonitor incrementalBuildMonitor = + new SkyframeIncrementalBuildMonitor(); + + private MutableSupplier<ConfigurationFactory> configurationFactory = new MutableSupplier<>(); + private MutableSupplier<Map<String, String>> clientEnv = new MutableSupplier<>(); + private MutableSupplier<ImmutableList<ConfigurationFragmentFactory>> configurationFragments = + new MutableSupplier<>(); + private MutableSupplier<Set<Package>> configurationPackages = new MutableSupplier<>(); + private SkyKey configurationSkyKey = null; + + private static final Logger LOG = Logger.getLogger(SkyframeExecutor.class.getName()); + + protected SkyframeExecutor( + Reporter reporter, + EvaluatorSupplier evaluatorSupplier, + PackageFactory pkgFactory, + TimestampGranularityMonitor tsgm, + BlazeDirectories directories, + Factory workspaceStatusActionFactory, + ImmutableList<BuildInfoFactory> buildInfoFactories, + Set<Path> immutableDirectories, + Predicate<PathFragment> allowedMissingInputs, + Preprocessor.Factory.Supplier preprocessorFactorySupplier, + ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions, + ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) { + // Strictly speaking, these arguments are not required for initialization, but all current + // callsites have them at hand, so we might as well set them during construction. + this.reporter = Preconditions.checkNotNull(reporter); + this.evaluatorSupplier = evaluatorSupplier; + this.pkgFactory = pkgFactory; + this.pkgFactory.setSyscalls(syscalls); + this.tsgm = tsgm; + this.workspaceStatusActionFactory = workspaceStatusActionFactory; + this.packageManager = new SkyframePackageManager( + new SkyframePackageLoader(), new SkyframeTransitivePackageLoader(), + new SkyframeTargetPatternEvaluator(this), syscalls, cyclesReporter, pkgLocator, + numPackagesLoaded, this); + this.errorEventListener = this.reporter; + this.resourceManager = ResourceManager.instance(); + this.skyframeActionExecutor = new SkyframeActionExecutor(reporter, resourceManager, eventBus, + statusReporterRef); + this.directories = Preconditions.checkNotNull(directories); + this.buildInfoFactories = buildInfoFactories; + this.immutableDirectories = immutableDirectories; + this.allowedMissingInputs = allowedMissingInputs; + this.preprocessorFactorySupplier = preprocessorFactorySupplier; + this.extraSkyFunctions = extraSkyFunctions; + this.extraPrecomputedValues = extraPrecomputedValues; + } + + private ImmutableMap<SkyFunctionName, SkyFunction> skyFunctions( + Root buildDataDirectory, + PackageFactory pkgFactory, + Predicate<PathFragment> allowedMissingInputs) { + ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator, + immutableDirectories); + // We use an immutable map builder for the nice side effect that it throws if a duplicate key + // is inserted. + ImmutableMap.Builder<SkyFunctionName, SkyFunction> map = ImmutableMap.builder(); + map.put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()); + map.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper)); + map.put(SkyFunctions.DIRECTORY_LISTING_STATE, + new DirectoryListingStateFunction(externalFilesHelper)); + map.put(SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS, + new FileSymlinkCycleUniquenessFunction()); + map.put(SkyFunctions.FILE, new FileFunction(pkgLocator, externalFilesHelper)); + map.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction()); + map.put(SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(deletedPackages)); + map.put(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, new ContainingPackageLookupFunction()); + map.put(SkyFunctions.AST_FILE_LOOKUP, new ASTFileLookupFunction( + pkgLocator, packageManager, pkgFactory.getRuleClassProvider())); + map.put(SkyFunctions.SKYLARK_IMPORTS_LOOKUP, new SkylarkImportLookupFunction( + pkgFactory.getRuleClassProvider(), pkgFactory)); + map.put(SkyFunctions.GLOB, new GlobFunction()); + map.put(SkyFunctions.TARGET_PATTERN, new TargetPatternFunction(pkgLocator)); + map.put(SkyFunctions.RECURSIVE_PKG, new RecursivePkgFunction()); + map.put(SkyFunctions.PACKAGE, new PackageFunction( + reporter, pkgFactory, packageManager, showLoadingProgress, packageFunctionCache, + eventBus, numPackagesLoaded)); + map.put(SkyFunctions.TARGET_MARKER, new TargetMarkerFunction()); + map.put(SkyFunctions.TRANSITIVE_TARGET, new TransitiveTargetFunction()); + map.put(SkyFunctions.CONFIGURED_TARGET, + new ConfiguredTargetFunction(new BuildViewProvider())); + map.put(SkyFunctions.ASPECT, new AspectFunction(new BuildViewProvider())); + map.put(SkyFunctions.POST_CONFIGURED_TARGET, + new PostConfiguredTargetFunction(new BuildViewProvider())); + map.put(SkyFunctions.CONFIGURATION_COLLECTION, new ConfigurationCollectionFunction( + configurationFactory, clientEnv, configurationPackages)); + map.put(SkyFunctions.CONFIGURATION_FRAGMENT, new ConfigurationFragmentFunction( + configurationFragments, configurationPackages)); + map.put(SkyFunctions.WORKSPACE_FILE, new WorkspaceFileFunction(pkgFactory)); + map.put(SkyFunctions.TARGET_COMPLETION, new TargetCompletionFunction(eventBus)); + map.put(SkyFunctions.TEST_COMPLETION, new TestCompletionFunction()); + map.put(SkyFunctions.ARTIFACT, new ArtifactFunction(allowedMissingInputs)); + map.put(SkyFunctions.BUILD_INFO_COLLECTION, new BuildInfoCollectionFunction(artifactFactory, + buildDataDirectory)); + map.put(SkyFunctions.BUILD_INFO, new WorkspaceStatusFunction()); + map.put(SkyFunctions.COVERAGE_REPORT, new CoverageReportFunction()); + map.put(SkyFunctions.ACTION_EXECUTION, + new ActionExecutionFunction(skyframeActionExecutor, tsgm)); + map.put(SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL, + new RecursiveFilesystemTraversalFunction()); + map.put(SkyFunctions.FILESET_ENTRY, new FilesetEntryFunction()); + map.putAll(extraSkyFunctions); + return map.build(); + } + + @ThreadCompatible + public void setActive(boolean active) { + this.active = active; + } + + protected void checkActive() { + Preconditions.checkState(active); + } + + public void setFileCache(ActionInputFileCache fileCache) { + this.skyframeActionExecutor.setFileCache(fileCache); + } + + public void dump(boolean summarize, PrintStream out) { + memoizingEvaluator.dump(summarize, out); + } + + public abstract void dumpPackages(PrintStream out); + + public void setBatchStatter(@Nullable BatchStat batchStatter) { + this.batchStatter = batchStatter; + } + + /** + * Notify listeners about changed files, and release any associated memory afterwards. + */ + public void drainChangedFiles() { + incrementalBuildMonitor.alertListeners(getEventBus()); + incrementalBuildMonitor = null; + } + + @VisibleForTesting + public BuildDriver getDriverForTesting() { + return buildDriver; + } + + /** + * This method exists only to allow a module to make a top-level Skyframe call during the + * transition to making it fully Skyframe-compatible. Do not add additional callers! + */ + public <E extends Exception> SkyValue evaluateSkyKeyForCodeMigration(final SkyKey key, + final Class<E> clazz) throws E { + try { + return callUninterruptibly(new Callable<SkyValue>() { + @Override + public SkyValue call() throws E, InterruptedException { + synchronized (valueLookupLock) { + // We evaluate in keepGoing mode because in the case that the graph does not store its + // edges, nokeepGoing builds are not allowed, whereas keepGoing builds are always + // permitted. + EvaluationResult<ActionLookupValue> result = buildDriver.evaluate( + ImmutableList.of(key), true, ResourceUsage.getAvailableProcessors(), + errorEventListener); + if (!result.hasError()) { + return Preconditions.checkNotNull(result.get(key), "%s %s", result, key); + } + ErrorInfo errorInfo = Preconditions.checkNotNull(result.getError(key), + "%s %s", key, result); + Throwables.propagateIfPossible(errorInfo.getException(), clazz); + if (errorInfo.getException() != null) { + throw new IllegalStateException(errorInfo.getException()); + } + throw new IllegalStateException(errorInfo.toString()); + } + } + }); + } catch (Exception e) { + Throwables.propagateIfPossible(e, clazz); + throw new IllegalStateException(e); + } + } + + class BuildViewProvider { + /** + * Returns the current {@link SkyframeBuildView} instance. + */ + SkyframeBuildView getSkyframeBuildView() { + return skyframeBuildView; + } + } + + /** + * Must be called before the {@link SkyframeExecutor} can be used (should only be called in + * factory methods and as an implementation detail of {@link #resetEvaluator}). + */ + protected void init() { + progressReceiver = new SkyframeProgressReceiver(); + Map<SkyFunctionName, SkyFunction> skyFunctions = skyFunctions( + directories.getBuildDataDirectory(), pkgFactory, allowedMissingInputs); + memoizingEvaluator = evaluatorSupplier.create( + skyFunctions, evaluatorDiffer(), progressReceiver, emittedEventState, + hasIncrementalState()); + buildDriver = newBuildDriver(); + } + + /** + * Reinitializes the Skyframe evaluator, dropping all previously computed values. + * + * <p>Be careful with this method as it also deletes all injected values. You need to make sure + * that any necessary precomputed values are reinjected before the next build. Constants can be + * put in {@link #reinjectConstantValuesLazily}. + */ + public void resetEvaluator() { + init(); + emittedEventState.clear(); + if (skyframeBuildView != null) { + skyframeBuildView.clearLegacyData(); + } + reinjectConstantValuesLazily(); + } + + protected abstract Differencer evaluatorDiffer(); + + protected abstract BuildDriver newBuildDriver(); + + /** + * Values whose values are known at startup and guaranteed constant are still wiped from the + * evaluator when we create a new one, so they must be re-injected each time we create a new + * evaluator. + */ + private void reinjectConstantValuesLazily() { + needToInjectEmbeddedArtifacts = true; + needToInjectPrecomputedValuesForAnalysis = true; + } + + /** + * Deletes all ConfiguredTarget values from the Skyframe cache. This is done to save memory (e.g. + * on a configuration change); since the configuration is part of the key, these key/value pairs + * will be sitting around doing nothing until the configuration changes back to the previous + * value. + * + * <p>The next evaluation will delete all invalid values. + */ + public abstract void dropConfiguredTargets(); + + /** + * Removes ConfigurationFragmentValuess and ConfigurationCollectionValues from the cache. + */ + @VisibleForTesting + public void invalidateConfigurationCollection() { + invalidate(SkyFunctionName.functionIsIn(ImmutableSet.of(SkyFunctions.CONFIGURATION_FRAGMENT, + SkyFunctions.CONFIGURATION_COLLECTION))); + } + + /** + * Decides if graph edges should be stored for this build. If not, re-creates the graph to not + * store graph edges. Necessary conditions to not store graph edges are: + * (1) batch (since incremental builds are not possible); + * (2) skyframe build (since otherwise the memory savings are too slight to bother); + * (3) keep-going (since otherwise bubbling errors up may require edges of done nodes); + * (4) discard_analysis_cache (since otherwise user isn't concerned about saving memory this way). + */ + public void decideKeepIncrementalState(boolean batch, Options viewOptions) { + // Assume incrementality. + } + + public boolean hasIncrementalState() { + return true; + } + + @VisibleForTesting + protected abstract Injectable injectable(); + + /** + * Saves memory by clearing analysis objects from Skyframe. If using legacy execution, actually + * deletes the relevant values. If using Skyframe execution, clears their data without deleting + * them (they will be deleted on the next build). + */ + public abstract void clearAnalysisCache(Collection<ConfiguredTarget> topLevelTargets); + + /** + * Injects the contents of the computed tools/defaults package. + */ + @VisibleForTesting + public void setupDefaultPackage(String defaultsPackageContents) { + PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.set(injectable(), defaultsPackageContents); + } + + /** + * Injects the top-level artifact options. + */ + public void injectTopLevelContext(TopLevelArtifactContext options) { + PrecomputedValue.TOP_LEVEL_CONTEXT.set(injectable(), options); + } + + public void injectWorkspaceStatusData() { + PrecomputedValue.WORKSPACE_STATUS_KEY.set(injectable(), + workspaceStatusActionFactory.createWorkspaceStatusAction( + artifactFactory.get(), WorkspaceStatusValue.ARTIFACT_OWNER, buildId)); + } + + public void injectCoverageReportData(Action action) { + PrecomputedValue.COVERAGE_REPORT_KEY.set(injectable(), action); + } + + /** + * Sets the default visibility. + */ + private void setDefaultVisibility(RuleVisibility defaultVisibility) { + PrecomputedValue.DEFAULT_VISIBILITY.set(injectable(), defaultVisibility); + } + + private void maybeInjectPrecomputedValuesForAnalysis() { + if (needToInjectPrecomputedValuesForAnalysis) { + injectBuildInfoFactories(); + injectExtraPrecomputedValues(); + needToInjectPrecomputedValuesForAnalysis = false; + } + } + + private void injectExtraPrecomputedValues() { + for (PrecomputedValue.Injected injected : extraPrecomputedValues) { + injected.inject(injectable()); + } + } + + /** + * Injects the build info factory map that will be used when constructing build info + * actions/artifacts. Unchanged across the life of the Blaze server, although it must be injected + * each time the evaluator is created. + */ + private void injectBuildInfoFactories() { + ImmutableMap.Builder<BuildInfoKey, BuildInfoFactory> factoryMapBuilder = + ImmutableMap.builder(); + for (BuildInfoFactory factory : buildInfoFactories) { + factoryMapBuilder.put(factory.getKey(), factory); + } + PrecomputedValue.BUILD_INFO_FACTORIES.set(injectable(), factoryMapBuilder.build()); + } + + private void setShowLoadingProgress(boolean showLoadingProgressValue) { + showLoadingProgress.set(showLoadingProgressValue); + } + + @VisibleForTesting + public void setCommandId(UUID commandId) { + PrecomputedValue.BUILD_ID.set(injectable(), commandId); + buildId.set(commandId); + } + + /** Returns the build-info.txt and build-changelist.txt artifacts. */ + public Collection<Artifact> getWorkspaceStatusArtifacts() throws InterruptedException { + // Should already be present, unless the user didn't request any targets for analysis. + EvaluationResult<WorkspaceStatusValue> result = buildDriver.evaluate( + ImmutableList.of(WorkspaceStatusValue.SKY_KEY), /*keepGoing=*/true, /*numThreads=*/1, + reporter); + WorkspaceStatusValue value = + Preconditions.checkNotNull(result.get(WorkspaceStatusValue.SKY_KEY)); + return ImmutableList.of(value.getStableArtifact(), value.getVolatileArtifact()); + } + + // TODO(bazel-team): Make this take a PackageIdentifier. + public Map<PathFragment, Root> getArtifactRoots(Iterable<PathFragment> execPaths) { + final List<SkyKey> packageKeys = new ArrayList<>(); + for (PathFragment execPath : execPaths) { + Preconditions.checkArgument(!execPath.isAbsolute(), execPath); + packageKeys.add(ContainingPackageLookupValue.key( + PackageIdentifier.createInDefaultRepo(execPath))); + } + + EvaluationResult<ContainingPackageLookupValue> result; + try { + result = callUninterruptibly(new Callable<EvaluationResult<ContainingPackageLookupValue>>() { + @Override + public EvaluationResult<ContainingPackageLookupValue> call() throws InterruptedException { + return buildDriver.evaluate(packageKeys, /*keepGoing=*/true, /*numThreads=*/1, reporter); + } + }); + } catch (Exception e) { + throw new IllegalStateException(e); // Should never happen. + } + + Map<PathFragment, Root> roots = new HashMap<>(); + for (PathFragment execPath : execPaths) { + ContainingPackageLookupValue value = result.get(ContainingPackageLookupValue.key( + PackageIdentifier.createInDefaultRepo(execPath))); + if (value.hasContainingPackage()) { + roots.put(execPath, Root.asSourceRoot(value.getContainingPackageRoot())); + } else { + roots.put(execPath, null); + } + } + return roots; + } + + @VisibleForTesting + public WorkspaceStatusAction getLastWorkspaceStatusActionForTesting() { + PrecomputedValue value = (PrecomputedValue) buildDriver.getGraphForTesting() + .getExistingValueForTesting(PrecomputedValue.WORKSPACE_STATUS_KEY.getKeyForTesting()); + return (WorkspaceStatusAction) value.get(); + } + + /** + * Informs user about number of modified files (source and output files). + */ + // Note, that number of modified files in some cases can be bigger than actual number of + // modified files for targets in current request. Skyframe may check for modification all files + // from previous requests. + protected void informAboutNumberOfModifiedFiles() { + LOG.info(String.format("Found %d modified files from last build", modifiedFiles)); + } + + public Reporter getReporter() { + return reporter; + } + + public EventBus getEventBus() { + return eventBus.get(); + } + + /** + * The map from package names to the package root where each package was found; this is used to + * set up the symlink tree. + */ + public ImmutableMap<PackageIdentifier, Path> getPackageRoots() { + // Make a map of the package names to their root paths. + ImmutableMap.Builder<PackageIdentifier, Path> packageRoots = ImmutableMap.builder(); + for (Package pkg : configurationPackages.get()) { + packageRoots.put(pkg.getPackageIdentifier(), pkg.getSourceRoot()); + } + return packageRoots.build(); + } + + @VisibleForTesting + ImmutableList<Path> getPathEntries() { + return pkgLocator.get().getPathEntries(); + } + + protected abstract void invalidate(Predicate<SkyKey> pred); + + protected static Iterable<SkyKey> getSkyKeysPotentiallyAffected( + Iterable<PathFragment> modifiedSourceFiles, final Path pathEntry) { + // TODO(bazel-team): change ModifiedFileSet to work with RootedPaths instead of PathFragments. + Iterable<SkyKey> fileStateSkyKeys = Iterables.transform(modifiedSourceFiles, + new Function<PathFragment, SkyKey>() { + @Override + public SkyKey apply(PathFragment pathFragment) { + Preconditions.checkState(!pathFragment.isAbsolute(), + "found absolute PathFragment: %s", pathFragment); + return FileStateValue.key(RootedPath.toRootedPath(pathEntry, pathFragment)); + } + }); + // TODO(bazel-team): Strictly speaking, we only need to invalidate directory values when a file + // has been created or deleted, not when it has been modified. Unfortunately we + // do not have that information here, although fancy filesystems could provide it with a + // hypothetically modified DiffAwareness interface. + // TODO(bazel-team): Even if we don't have that information, we could avoid invalidating + // directories when the state of a file does not change by statting them and comparing + // the new filetype (nonexistent/file/symlink/directory) with the old one. + Iterable<SkyKey> dirListingStateSkyKeys = Iterables.transform( + modifiedSourceFiles, + new Function<PathFragment, SkyKey>() { + @Override + public SkyKey apply(PathFragment pathFragment) { + Preconditions.checkState(!pathFragment.isAbsolute(), + "found absolute PathFragment: %s", pathFragment); + return DirectoryListingStateValue.key(RootedPath.toRootedPath(pathEntry, + pathFragment.getParentDirectory())); + } + }); + return Iterables.concat(fileStateSkyKeys, dirListingStateSkyKeys); + } + + protected static SkyKey createFileStateKey(RootedPath rootedPath) { + return FileStateValue.key(rootedPath); + } + + protected static SkyKey createDirectoryListingStateKey(RootedPath rootedPath) { + return DirectoryListingStateValue.key(rootedPath); + } + + /** + * Creates a FileValue pointing of type directory. No matter that the rootedPath points to a + * symlink. + * + * <p> Use it with caution as it would prevent invalidation when the destination file in the + * symlink changes. + */ + protected static FileValue createFileDirValue(RootedPath rootedPath) { + return FileValue.value(rootedPath, FileStateValue.DIRECTORY_FILE_STATE_NODE, + rootedPath, FileStateValue.DIRECTORY_FILE_STATE_NODE); + } + + /** + * Sets the packages that should be treated as deleted and ignored. + */ + @VisibleForTesting // productionVisibility = Visibility.PRIVATE + public abstract void setDeletedPackages(Iterable<String> pkgs); + + /** + * Prepares the evaluator for loading. + * + * <p>MUST be run before every incremental build. + */ + @VisibleForTesting // productionVisibility = Visibility.PRIVATE + public void preparePackageLoading(PathPackageLocator pkgLocator, RuleVisibility defaultVisibility, + boolean showLoadingProgress, + String defaultsPackageContents, UUID commandId) { + Preconditions.checkNotNull(pkgLocator); + setActive(true); + + maybeInjectPrecomputedValuesForAnalysis(); + setCommandId(commandId); + setShowLoadingProgress(showLoadingProgress); + setDefaultVisibility(defaultVisibility); + setupDefaultPackage(defaultsPackageContents); + setPackageLocator(pkgLocator); + + syscalls.set(new PerBuildSyscallCache()); + checkPreprocessorFactory(); + emittedEventState.clear(); + + // If the PackageFunction was interrupted, there may be stale entries here. + packageFunctionCache.clear(); + numPackagesLoaded.set(0); + + // Reset the stateful SkyframeCycleReporter, which contains cycles from last run. + cyclesReporter.set(createCyclesReporter()); + } + + @SuppressWarnings("unchecked") + private void setPackageLocator(PathPackageLocator pkgLocator) { + PathPackageLocator oldLocator = this.pkgLocator.getAndSet(pkgLocator); + PrecomputedValue.PATH_PACKAGE_LOCATOR.set(injectable(), pkgLocator); + + if (!pkgLocator.equals(oldLocator)) { + // The package path is read not only by SkyFunctions but also by some other code paths. + // We need to take additional steps to keep the corresponding data structures in sync. + // (Some of the additional steps are carried out by ConfiguredTargetValueInvalidationListener, + // and some by BuildView#buildHasIncompatiblePackageRoots and #updateSkyframe.) + onNewPackageLocator(oldLocator, pkgLocator); + } + } + + protected abstract void onNewPackageLocator(PathPackageLocator oldLocator, + PathPackageLocator pkgLocator); + + private void checkPreprocessorFactory() { + if (preprocessorFactory == null) { + Preprocessor.Factory newPreprocessorFactory = preprocessorFactorySupplier.getFactory( + packageManager); + pkgFactory.setPreprocessorFactory(newPreprocessorFactory); + preprocessorFactory = newPreprocessorFactory; + } else if (!preprocessorFactory.isStillValid()) { + Preprocessor.Factory newPreprocessorFactory = preprocessorFactorySupplier.getFactory( + packageManager); + invalidate(SkyFunctionName.functionIs(SkyFunctions.PACKAGE)); + pkgFactory.setPreprocessorFactory(newPreprocessorFactory); + preprocessorFactory = newPreprocessorFactory; + } + } + + /** + * Specifies the current {@link SkyframeBuildView} instance. This should only be set once over the + * lifetime of the Blaze server, except in tests. + */ + public void setSkyframeBuildView(SkyframeBuildView skyframeBuildView) { + this.skyframeBuildView = skyframeBuildView; + setConfigurationSkyKey(configurationSkyKey); + this.artifactFactory.set(skyframeBuildView.getArtifactFactory()); + if (skyframeBuildView.getWarningListener() != null) { + setErrorEventListener(skyframeBuildView.getWarningListener()); + } + } + + /** + * Sets the eventBus to use for posting events. + */ + public void setEventBus(EventBus eventBus) { + this.eventBus.set(eventBus); + } + + /** + * Sets the eventHandler to use for reporting errors. + */ + public void setErrorEventListener(EventHandler eventHandler) { + this.errorEventListener = eventHandler; + } + + /** + * Sets the path for action log buffers. + */ + public void setActionOutputRoot(Path actionOutputRoot) { + Preconditions.checkNotNull(actionOutputRoot); + this.actionLogBufferPathGenerator = new ActionLogBufferPathGenerator(actionOutputRoot); + this.skyframeActionExecutor.setActionLogBufferPathGenerator(actionLogBufferPathGenerator); + } + + private void setConfigurationSkyKey(SkyKey skyKey) { + this.configurationSkyKey = skyKey; + if (skyframeBuildView != null) { + skyframeBuildView.setConfigurationSkyKey(skyKey); + } + } + + @VisibleForTesting + public void setConfigurationDataForTesting(BuildOptions options, + BlazeDirectories directories, ConfigurationFactory configurationFactory) { + SkyKey skyKey = ConfigurationCollectionValue.key(options, ImmutableSet.<String>of()); + setConfigurationSkyKey(skyKey); + PrecomputedValue.BLAZE_DIRECTORIES.set(injectable(), directories); + this.configurationFactory.set(configurationFactory); + this.configurationFragments.set(ImmutableList.copyOf(configurationFactory.getFactories())); + this.configurationPackages.set(Sets.<Package>newConcurrentHashSet()); + } + + @VisibleForTesting + public BuildConfigurationCollection createConfigurations( + ConfigurationFactory configurationFactory, BuildConfigurationKey configurationKey) + throws InvalidConfigurationException, InterruptedException { + return createConfigurations(false, configurationFactory, configurationKey); + } + + /** + * Asks the Skyframe evaluator to build the value for BuildConfigurationCollection and + * returns result. Also invalidates {@link PrecomputedValue#TEST_ENVIRONMENT_VARIABLES} and + * {@link PrecomputedValue#BLAZE_DIRECTORIES} if they have changed. + */ + public BuildConfigurationCollection createConfigurations(boolean keepGoing, + ConfigurationFactory configurationFactory, BuildConfigurationKey configurationKey) + throws InvalidConfigurationException, InterruptedException { + + this.configurationPackages.set(Sets.<Package>newConcurrentHashSet()); + this.clientEnv.set(configurationKey.getClientEnv()); + this.configurationFactory.set(configurationFactory); + this.configurationFragments.set(ImmutableList.copyOf(configurationFactory.getFactories())); + BuildOptions buildOptions = configurationKey.getBuildOptions(); + Map<String, String> testEnv = BuildConfiguration.getTestEnv( + buildOptions.get(BuildConfiguration.Options.class).testEnvironment, + configurationKey.getClientEnv()); + // TODO(bazel-team): find a way to use only BuildConfigurationKey instead of + // TestEnvironmentVariables and BlazeDirectories. There is a problem only with + // TestEnvironmentVariables because BuildConfigurationKey stores client environment variables + // and we don't want to rebuild everything when any variable changes. + PrecomputedValue.TEST_ENVIRONMENT_VARIABLES.set(injectable(), testEnv); + PrecomputedValue.BLAZE_DIRECTORIES.set(injectable(), configurationKey.getDirectories()); + + SkyKey skyKey = ConfigurationCollectionValue.key(configurationKey.getBuildOptions(), + configurationKey.getMultiCpu()); + setConfigurationSkyKey(skyKey); + EvaluationResult<ConfigurationCollectionValue> result = buildDriver.evaluate( + Arrays.asList(skyKey), keepGoing, DEFAULT_THREAD_COUNT, errorEventListener); + if (result.hasError()) { + Throwable e = result.getError(skyKey).getException(); + // Wrap loading failed exceptions + if (e instanceof NoSuchThingException) { + e = new InvalidConfigurationException(e); + } + Throwables.propagateIfInstanceOf(e, InvalidConfigurationException.class); + throw new IllegalStateException( + "Unknown error during ConfigurationCollectionValue evaluation", e); + } + Preconditions.checkState(result.values().size() == 1, + "Result of evaluate() must contain exactly one value %s", result); + ConfigurationCollectionValue configurationValue = + Iterables.getOnlyElement(result.values()); + this.configurationPackages.set( + Sets.newConcurrentHashSet(configurationValue.getConfigurationPackages())); + return configurationValue.getConfigurationCollection(); + } + + private Iterable<ActionLookupValue> getActionLookupValues() { + // This filter keeps subclasses of ActionLookupValue. + return Iterables.filter(memoizingEvaluator.getDoneValues().values(), ActionLookupValue.class); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + Map<SkyKey, ActionLookupValue> getActionLookupValueMap() { + return (Map) Maps.filterValues(memoizingEvaluator.getDoneValues(), + Predicates.instanceOf(ActionLookupValue.class)); + } + + /** + * Checks the actions in Skyframe for conflicts between their output artifacts. Delegates to + * {@link SkyframeActionExecutor#findAndStoreArtifactConflicts} to do the work, since any + * conflicts found will only be reported during execution. + */ + ImmutableMap<Action, SkyframeActionExecutor.ConflictException> findArtifactConflicts() + throws InterruptedException { + if (skyframeBuildView.isSomeConfiguredTargetEvaluated() + || skyframeBuildView.isSomeConfiguredTargetInvalidated()) { + // This operation is somewhat expensive, so we only do it if the graph might have changed in + // some way -- either we analyzed a new target or we invalidated an old one. + skyframeActionExecutor.findAndStoreArtifactConflicts(getActionLookupValues()); + skyframeBuildView.resetEvaluatedConfiguredTargetFlag(); + // The invalidated configured targets flag will be reset later in the evaluate() call. + } + return skyframeActionExecutor.badActions(); + } + + /** + * Asks the Skyframe evaluator to build the given artifacts and targets, and to test the + * given test targets. + */ + public EvaluationResult<?> buildArtifacts( + Executor executor, + Set<Artifact> artifactsToBuild, + Collection<ConfiguredTarget> targetsToBuild, + Collection<ConfiguredTarget> targetsToTest, + boolean exclusiveTesting, + boolean keepGoing, + boolean explain, + int numJobs, + ActionCacheChecker actionCacheChecker, + @Nullable EvaluationProgressReceiver executionProgressReceiver) throws InterruptedException { + checkActive(); + Preconditions.checkState(actionLogBufferPathGenerator != null); + + skyframeActionExecutor.prepareForExecution(executor, keepGoing, explain, actionCacheChecker); + + resourceManager.resetResourceUsage(); + try { + progressReceiver.executionProgressReceiver = executionProgressReceiver; + Iterable<SkyKey> artifactKeys = ArtifactValue.mandatoryKeys(artifactsToBuild); + Iterable<SkyKey> targetKeys = TargetCompletionValue.keys(targetsToBuild); + Iterable<SkyKey> testKeys = TestCompletionValue.keys(targetsToTest, exclusiveTesting); + return buildDriver.evaluate(Iterables.concat(artifactKeys, targetKeys, testKeys), keepGoing, + numJobs, errorEventListener); + } finally { + progressReceiver.executionProgressReceiver = null; + // Also releases thread locks. + resourceManager.resetResourceUsage(); + skyframeActionExecutor.executionOver(); + } + } + + @VisibleForTesting + public void prepareBuildingForTestingOnly(Executor executor, boolean keepGoing, boolean explain, + ActionCacheChecker checker) { + skyframeActionExecutor.prepareForExecution(executor, keepGoing, explain, checker); + } + + EvaluationResult<TargetPatternValue> targetPatterns(Iterable<SkyKey> patternSkyKeys, + boolean keepGoing, EventHandler eventHandler) throws InterruptedException { + checkActive(); + return buildDriver.evaluate(patternSkyKeys, keepGoing, DEFAULT_THREAD_COUNT, + eventHandler); + } + + /** + * Returns the {@link ConfiguredTarget}s corresponding to the given keys. + * + * <p>For use for legacy support from {@code BuildView} only. + * + * <p>If a requested configured target is in error, the corresponding value is omitted from the + * returned list. + */ + @ThreadSafety.ThreadSafe + public ImmutableList<ConfiguredTarget> getConfiguredTargets(Iterable<Dependency> keys) { + checkActive(); + if (skyframeBuildView == null) { + // If build view has not yet been initialized, no configured targets can have been created. + // This is most likely to happen after a failed loading phase. + return ImmutableList.of(); + } + final List<SkyKey> skyKeys = new ArrayList<>(); + for (Dependency key : keys) { + skyKeys.add(ConfiguredTargetValue.key(key.getLabel(), key.getConfiguration())); + for (Class<? extends ConfiguredAspectFactory> aspect : key.getAspects()) { + skyKeys.add(AspectValue.key(key.getLabel(), key.getConfiguration(), aspect)); + } + } + + EvaluationResult<SkyValue> result; + try { + result = callUninterruptibly(new Callable<EvaluationResult<SkyValue>>() { + @Override + public EvaluationResult<SkyValue> call() throws Exception { + synchronized (valueLookupLock) { + try { + skyframeBuildView.enableAnalysis(true); + return buildDriver.evaluate(skyKeys, false, DEFAULT_THREAD_COUNT, + errorEventListener); + } finally { + skyframeBuildView.enableAnalysis(false); + } + } + } + }); + } catch (Exception e) { + throw new IllegalStateException(e); // Should never happen. + } + + ImmutableList.Builder<ConfiguredTarget> cts = ImmutableList.builder(); + + DependentNodeLoop: + for (Dependency key : keys) { + SkyKey configuredTargetKey = ConfiguredTargetValue.key( + key.getLabel(), key.getConfiguration()); + if (result.get(configuredTargetKey) == null) { + continue; + } + + ConfiguredTarget configuredTarget = + ((ConfiguredTargetValue) result.get(configuredTargetKey)).getConfiguredTarget(); + List<Aspect> aspects = new ArrayList<>(); + + for (Class<? extends ConfiguredAspectFactory> aspect : key.getAspects()) { + SkyKey aspectKey = AspectValue.key(key.getLabel(), key.getConfiguration(), aspect); + if (result.get(aspectKey) == null) { + continue DependentNodeLoop; + } + + aspects.add(((AspectValue) result.get(aspectKey)).get()); + } + + cts.add(RuleConfiguredTarget.mergeAspects(configuredTarget, aspects)); + } + + return cts.build(); + } + + /** + * Returns a particular configured target. + * + * <p>Used only for testing. + */ + @VisibleForTesting + @Nullable + public ConfiguredTarget getConfiguredTargetForTesting( + Label label, BuildConfiguration configuration) { + if (memoizingEvaluator.getExistingValueForTesting( + PrecomputedValue.WORKSPACE_STATUS_KEY.getKeyForTesting()) == null) { + injectWorkspaceStatusData(); + } + return Iterables.getFirst(getConfiguredTargets(ImmutableList.of( + new Dependency(label, configuration))), null); + } + + /** + * Invalidates Skyframe values corresponding to the given set of modified files under the given + * path entry. + * + * <p>May throw an {@link InterruptedException}, which means that no values have been invalidated. + */ + @VisibleForTesting + public abstract void invalidateFilesUnderPathForTesting(ModifiedFileSet modifiedFileSet, + Path pathEntry) throws InterruptedException; + + /** + * Invalidates SkyFrame values that may have failed for transient reasons. + */ + public abstract void invalidateTransientErrors(); + + @VisibleForTesting + public TimestampGranularityMonitor getTimestampGranularityMonitorForTesting() { + return tsgm; + } + + /** + * Configures a given set of configured targets. + */ + public EvaluationResult<ConfiguredTargetValue> configureTargets( + List<ConfiguredTargetKey> values, boolean keepGoing) throws InterruptedException { + checkActive(); + + // Make sure to not run too many analysis threads. This can cause memory thrashing. + return buildDriver.evaluate(ConfiguredTargetValue.keys(values), keepGoing, + ResourceUsage.getAvailableProcessors(), errorEventListener); + } + + /** + * Post-process the targets. Values in the EvaluationResult are known to be transitively + * error-free from action conflicts. + */ + public EvaluationResult<PostConfiguredTargetValue> postConfigureTargets( + List<ConfiguredTargetKey> values, boolean keepGoing, + ImmutableMap<Action, SkyframeActionExecutor.ConflictException> badActions) + throws InterruptedException { + checkActive(); + PrecomputedValue.BAD_ACTIONS.set(injectable(), badActions); + // Make sure to not run too many analysis threads. This can cause memory thrashing. + EvaluationResult<PostConfiguredTargetValue> result = + buildDriver.evaluate(PostConfiguredTargetValue.keys(values), keepGoing, + ResourceUsage.getAvailableProcessors(), errorEventListener); + + // Remove all post-configured target values immediately for memory efficiency. We are OK with + // this mini-phase being non-incremental as the failure mode of action conflict is rare. + memoizingEvaluator.delete(SkyFunctionName.functionIs(SkyFunctions.POST_CONFIGURED_TARGET)); + + return result; + } + + /** + * Returns a Skyframe-based {@link SkyframeTransitivePackageLoader} implementation. + */ + @VisibleForTesting + public TransitivePackageLoader pkgLoader() { + checkActive(); + return new SkyframeLabelVisitor(new SkyframeTransitivePackageLoader(), cyclesReporter); + } + + class SkyframeTransitivePackageLoader { + /** + * Loads the specified {@link TransitiveTargetValue}s. + */ + EvaluationResult<TransitiveTargetValue> loadTransitiveTargets( + Iterable<Target> targetsToVisit, Iterable<Label> labelsToVisit, boolean keepGoing) + throws InterruptedException { + List<SkyKey> valueNames = new ArrayList<>(); + for (Target target : targetsToVisit) { + valueNames.add(TransitiveTargetValue.key(target.getLabel())); + } + for (Label label : labelsToVisit) { + valueNames.add(TransitiveTargetValue.key(label)); + } + + return buildDriver.evaluate(valueNames, keepGoing, DEFAULT_THREAD_COUNT, + errorEventListener); + } + + public Set<Package> retrievePackages(Set<PackageIdentifier> packageIds) { + final List<SkyKey> valueNames = new ArrayList<>(); + for (PackageIdentifier pkgId : packageIds) { + valueNames.add(PackageValue.key(pkgId)); + } + + try { + return callUninterruptibly(new Callable<Set<Package>>() { + @Override + public Set<Package> call() throws Exception { + EvaluationResult<PackageValue> result = buildDriver.evaluate( + valueNames, false, ResourceUsage.getAvailableProcessors(), errorEventListener); + Preconditions.checkState(!result.hasError(), + "unexpected errors: %s", result.errorMap()); + Set<Package> packages = Sets.newHashSet(); + for (PackageValue value : result.values()) { + packages.add(value.getPackage()); + } + return packages; + } + }); + } catch (Exception e) { + throw new IllegalStateException(e); + } + + } + } + + /** + * Returns the generating {@link Action} of the given {@link Artifact}. + * + * <p>For use for legacy support from {@code BuildView} only. + */ + @ThreadSafety.ThreadSafe + public Action getGeneratingAction(final Artifact artifact) { + if (artifact.isSourceArtifact()) { + return null; + } + + try { + return callUninterruptibly(new Callable<Action>() { + @Override + public Action call() throws InterruptedException { + ArtifactOwner artifactOwner = artifact.getArtifactOwner(); + Preconditions.checkState(artifactOwner instanceof ActionLookupValue.ActionLookupKey, + "%s %s", artifact, artifactOwner); + SkyKey actionLookupKey = + ActionLookupValue.key((ActionLookupValue.ActionLookupKey) artifactOwner); + + synchronized (valueLookupLock) { + // Note that this will crash (attempting to run a configured target value builder after + // analysis) after a failed --nokeep_going analysis in which the configured target that + // failed was a (transitive) dependency of the configured target that should generate + // this action. We don't expect callers to query generating actions in such cases. + EvaluationResult<ActionLookupValue> result = buildDriver.evaluate( + ImmutableList.of(actionLookupKey), false, ResourceUsage.getAvailableProcessors(), + errorEventListener); + return result.hasError() + ? null + : result.get(actionLookupKey).getGeneratingAction(artifact); + } + } + }); + } catch (Exception e) { + throw new IllegalStateException("Error getting generating action: " + artifact.prettyPrint(), + e); + } + } + + public PackageManager getPackageManager() { + return packageManager; + } + + class SkyframePackageLoader { + /** + * Looks up a particular package (mostly used after the loading phase, so packages should + * already be present, but occasionally used pre-loading phase). Use should be discouraged, + * since this cannot be used inside a Skyframe evaluation, and concurrent calls are + * synchronized. + * + * <p>Note that this method needs to be synchronized since InMemoryMemoizingEvaluator.evaluate() + * method does not support concurrent calls. + */ + Package getPackage(EventHandler eventHandler, PackageIdentifier pkgName) + throws InterruptedException, NoSuchPackageException { + synchronized (valueLookupLock) { + SkyKey key = PackageValue.key(pkgName); + // Any call to this method post-loading phase should either be error-free or be in a + // keep_going build, since otherwise the build would have failed during loading. Thus + // we set keepGoing=true unconditionally. + EvaluationResult<PackageValue> result = + buildDriver.evaluate(ImmutableList.of(key), /*keepGoing=*/true, + DEFAULT_THREAD_COUNT, eventHandler); + if (result.hasError()) { + ErrorInfo error = result.getError(); + if (!Iterables.isEmpty(error.getCycleInfo())) { + reportCycles(result.getError().getCycleInfo(), key); + // This can only happen if a package is freshly loaded outside of the target parsing + // or loading phase + throw new BuildFileContainsErrorsException(pkgName.toString(), + "Cycle encountered while loading package " + pkgName); + } + Throwable e = error.getException(); + // PackageFunction should be catching, swallowing, and rethrowing all transitive + // errors as NoSuchPackageExceptions. + Throwables.propagateIfInstanceOf(e, NoSuchPackageException.class); + throw new IllegalStateException("Unexpected Exception type from PackageValue for '" + + pkgName + "'' with root causes: " + Iterables.toString(error.getRootCauses()), e); + } + return result.get(key).getPackage(); + } + } + + Package getLoadedPackage(final PackageIdentifier pkgName) throws NoSuchPackageException { + // Note that in Skyframe there is no way to tell if the package has been loaded before or not, + // so this will never throw for packages that are not loaded. However, no code currently + // relies on having the exception thrown. + try { + return callUninterruptibly(new Callable<Package>() { + @Override + public Package call() throws Exception { + return getPackage(errorEventListener, pkgName); + } + }); + } catch (NoSuchPackageException e) { + if (e.getPackage() != null) { + return e.getPackage(); + } + throw e; + } catch (Exception e) { + throw new IllegalStateException(e); // Should never happen. + } + } + + /** + * Returns whether the given package should be consider deleted and thus should be ignored. + */ + public boolean isPackageDeleted(String packageName) { + return deletedPackages.get().contains(packageName); + } + + /** Same as {@link PackageManager#partiallyClear}. */ + void partiallyClear() { + packageFunctionCache.clear(); + } + } + + /** + * Calls the given callable uninterruptibly. + * + * <p>If the callable throws {@link InterruptedException}, calls it again, until the callable + * returns a result. Sets the {@code currentThread().interrupted()} bit if the callable threw + * {@link InterruptedException} at least once. + * + * <p>This is almost identical to {@code Uninterruptibles#getUninterruptibly}. + */ + protected static final <T> T callUninterruptibly(Callable<T> callable) throws Exception { + boolean interrupted = false; + try { + while (true) { + try { + return callable.call(); + } catch (InterruptedException e) { + interrupted = true; + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + @VisibleForTesting + public MemoizingEvaluator getEvaluatorForTesting() { + return memoizingEvaluator; + } + + /** + * Stores the set of loaded packages and, if needed, evicts ConfiguredTarget values. + * + * <p>The set represents all packages from the transitive closure of the top-level targets from + * the latest build. + */ + @ThreadCompatible + public abstract void updateLoadedPackageSet(Set<PackageIdentifier> loadedPackages); + + public void sync(PackageCacheOptions packageCacheOptions, Path workingDirectory, + String defaultsPackageContents, UUID commandId) throws InterruptedException, + AbruptExitException{ + + preparePackageLoading( + createPackageLocator(packageCacheOptions, directories.getWorkspace(), workingDirectory), + packageCacheOptions.defaultVisibility, packageCacheOptions.showLoadingProgress, + defaultsPackageContents, commandId); + setDeletedPackages(ImmutableSet.copyOf(packageCacheOptions.deletedPackages)); + + incrementalBuildMonitor = new SkyframeIncrementalBuildMonitor(); + invalidateTransientErrors(); + } + + protected PathPackageLocator createPackageLocator(PackageCacheOptions packageCacheOptions, + Path workspace, Path workingDirectory) throws AbruptExitException{ + return PathPackageLocator.create( + packageCacheOptions.packagePath, getReporter(), workspace, workingDirectory); + } + + private CyclesReporter createCyclesReporter() { + return new CyclesReporter( + new TransitiveTargetCycleReporter(packageManager), + new ActionArtifactCycleReporter(packageManager), + new SkylarkModuleCycleReporter()); + } + + CyclesReporter getCyclesReporter() { + return cyclesReporter.get(); + } + + /** Convenience method with same semantics as {@link CyclesReporter#reportCycles}. */ + public void reportCycles(Iterable<CycleInfo> cycles, SkyKey topLevelKey) { + getCyclesReporter().reportCycles(cycles, topLevelKey, errorEventListener); + } + + public void setActionExecutionProgressReportingObjects(@Nullable ProgressSupplier supplier, + @Nullable ActionCompletedReceiver completionReceiver, + @Nullable ActionExecutionStatusReporter statusReporter) { + skyframeActionExecutor.setActionExecutionProgressReportingObjects(supplier, completionReceiver); + this.statusReporterRef.set(statusReporter); + } + + /** + * This should be called at most once in the lifetime of the SkyframeExecutor (except for + * tests), and it should be called before the execution phase. + */ + void setArtifactFactoryAndBinTools(ArtifactFactory artifactFactory, BinTools binTools) { + this.artifactFactory.set(artifactFactory); + this.binTools = binTools; + } + + public void prepareExecution(boolean checkOutputFiles) throws AbruptExitException, + InterruptedException { + maybeInjectEmbeddedArtifacts(); + + if (checkOutputFiles) { + // Detect external modifications in the output tree. + FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm); + invalidateDirtyActions(fsnc.getDirtyActionValues(batchStatter)); + modifiedFiles += fsnc.getNumberOfModifiedOutputFiles(); + } + informAboutNumberOfModifiedFiles(); + } + + protected abstract void invalidateDirtyActions(Iterable<SkyKey> dirtyActionValues); + + @VisibleForTesting void maybeInjectEmbeddedArtifacts() throws AbruptExitException { + // The blaze client already ensures that the contents of the embedded binaries never change, + // so we just need to make sure that the appropriate artifacts are present in the skyframe + // graph. + + if (!needToInjectEmbeddedArtifacts) { + return; + } + + Preconditions.checkNotNull(artifactFactory.get()); + Preconditions.checkNotNull(binTools); + Map<SkyKey, SkyValue> values = Maps.newHashMap(); + // Blaze separately handles the symlinks that target these binaries. See BinTools#setupTool. + for (Artifact artifact : binTools.getAllEmbeddedArtifacts(artifactFactory.get())) { + FileArtifactValue fileArtifactValue; + try { + fileArtifactValue = FileArtifactValue.create(artifact); + } catch (IOException e) { + // See ExtractData in blaze.cc. + String message = "Error: corrupt installation: file " + artifact.getPath() + " missing. " + + "Please remove '" + directories.getInstallBase() + "' and try again."; + throw new AbruptExitException(message, ExitCode.LOCAL_ENVIRONMENTAL_ERROR, e); + } + values.put(ArtifactValue.key(artifact, /*isMandatory=*/true), fileArtifactValue); + } + injectable().inject(values); + needToInjectEmbeddedArtifacts = false; + } + + /** + * Mark dirty values for deletion if they've been dirty for longer than N versions. + * + * <p>Specifying a value N means, if the current version is V and a value was dirtied (and + * has remained so) in version U, and U + N <= V, then the value will be marked for deletion + * and purged in version V+1. + */ + public abstract void deleteOldNodes(long versionWindowForDirtyGc); + + /** + * A progress received to track analysis invalidation and update progress messages. + */ + protected class SkyframeProgressReceiver implements EvaluationProgressReceiver { + /** + * This flag is needed in order to avoid invalidating legacy data when we clear the + * analysis cache because of --discard_analysis_cache flag. For that case we want to keep + * the legacy data but get rid of the Skyframe data. + */ + protected boolean ignoreInvalidations = false; + /** This receiver is only needed for execution, so it is null otherwise. */ + @Nullable EvaluationProgressReceiver executionProgressReceiver = null; + + @Override + public void invalidated(SkyValue value, InvalidationState state) { + if (ignoreInvalidations) { + return; + } + if (skyframeBuildView != null) { + skyframeBuildView.getInvalidationReceiver().invalidated(value, state); + } + } + + @Override + public void enqueueing(SkyKey skyKey) { + if (ignoreInvalidations) { + return; + } + if (skyframeBuildView != null) { + skyframeBuildView.getInvalidationReceiver().enqueueing(skyKey); + } + if (executionProgressReceiver != null) { + executionProgressReceiver.enqueueing(skyKey); + } + } + + @Override + public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) { + if (ignoreInvalidations) { + return; + } + if (skyframeBuildView != null) { + skyframeBuildView.getInvalidationReceiver().evaluated(skyKey, value, state); + } + if (executionProgressReceiver != null) { + executionProgressReceiver.evaluated(skyKey, value, state); + } + } + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutorFactory.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutorFactory.java new file mode 100644 index 0000000..a1615cf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutorFactory.java
@@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.Preprocessor; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; + +import java.util.Set; + +/** +* A factory that creates instances of SkyframeExecutor. +*/ +public interface SkyframeExecutorFactory { + + /** + * Creates an instance of SkyframeExecutor + * + * @param reporter the reporter to be used by the executor + * @param pkgFactory the package factory + * @param skyframeBuild use Skyframe for the build phase. Should be always true after we are in + * the skyframe full mode. + * @param tsgm timestamp granularity monitor + * @param directories Blaze directories + * @param workspaceStatusActionFactory a factory for creating WorkspaceStatusAction objects + * @param buildInfoFactories list of BuildInfoFactories + * @param diffAwarenessFactories + * @param allowedMissingInputs + * @param preprocessorFactorySupplier + * @param extraSkyFunctions + * @param extraPrecomputedValues + * @return an instance of the SkyframeExecutor + * @throws AbruptExitException if the executor cannot be created + */ + SkyframeExecutor create(Reporter reporter, PackageFactory pkgFactory, + TimestampGranularityMonitor tsgm, BlazeDirectories directories, + Factory workspaceStatusActionFactory, + ImmutableList<BuildInfoFactory> buildInfoFactories, + Set<Path> immutableDirectories, + Iterable<? extends DiffAwareness.Factory> diffAwarenessFactories, + Predicate<PathFragment> allowedMissingInputs, + Preprocessor.Factory.Supplier preprocessorFactorySupplier, + ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions, + ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) throws AbruptExitException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeIncrementalBuildMonitor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeIncrementalBuildMonitor.java new file mode 100644 index 0000000..c0fea26 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeIncrementalBuildMonitor.java
@@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.actions.ChangedFilesMessage; +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyKey; + +import java.util.HashSet; +import java.util.Set; + +/** + * A package-private class intended to track a small number of modified files during the build. This + * class should stop recording changed files if there are too many of them, instead of holding onto + * a large collection of files. + */ +@ThreadSafety.ThreadCompatible +class SkyframeIncrementalBuildMonitor { + private Set<PathFragment> files = new HashSet<>(); + private static final int MAX_FILES = 100; + + public void accrue(Iterable<SkyKey> invalidatedValues) { + for (SkyKey skyKey : invalidatedValues) { + if (skyKey.functionName() == SkyFunctions.FILE_STATE) { + RootedPath file = (RootedPath) skyKey.argument(); + maybeAddFile(file.getRelativePath()); + } + } + } + + private void maybeAddFile(PathFragment path) { + if (files != null) { + files.add(path); + if (files.size() >= MAX_FILES) { + files = null; + } + } + } + + public void alertListeners(EventBus eventBus) { + if (files != null) { + eventBus.post(new ChangedFilesMessage(files)); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeLabelVisitor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeLabelVisitor.java new file mode 100644 index 0000000..2844cc0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeLabelVisitor.java
@@ -0,0 +1,262 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor.SkyframeTransitivePackageLoader; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.CycleInfo; +import com.google.devtools.build.skyframe.CyclesReporter; +import com.google.devtools.build.skyframe.ErrorInfo; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.SkyKey; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.Nullable; + +/** + * Skyframe-based transitive package loader. + */ +final class SkyframeLabelVisitor implements TransitivePackageLoader { + + private final SkyframeTransitivePackageLoader transitivePackageLoader; + private final AtomicReference<CyclesReporter> skyframeCyclesReporter; + + private Set<PackageIdentifier> allVisitedPackages; + private Set<PackageIdentifier> errorFreeVisitedPackages; + private Set<Label> visitedTargets; + private Set<TransitiveTargetValue> previousBuildTargetValueSet = null; + private boolean lastBuildKeepGoing = false; + private final Multimap<Label, Label> rootCauses = HashMultimap.create(); + + SkyframeLabelVisitor(SkyframeTransitivePackageLoader transitivePackageLoader, + AtomicReference<CyclesReporter> skyframeCyclesReporter) { + this.transitivePackageLoader = transitivePackageLoader; + this.skyframeCyclesReporter = skyframeCyclesReporter; + } + + @Override + public boolean sync(EventHandler eventHandler, Set<Target> targetsToVisit, + Set<Label> labelsToVisit, boolean keepGoing, int parallelThreads, int maxDepth) + throws InterruptedException { + rootCauses.clear(); + lastBuildKeepGoing = false; + EvaluationResult<TransitiveTargetValue> result = + transitivePackageLoader.loadTransitiveTargets(targetsToVisit, labelsToVisit, keepGoing); + updateVisitedValues(result.values()); + lastBuildKeepGoing = keepGoing; + + if (!hasErrors(result)) { + return true; + } + + Set<Entry<SkyKey, ErrorInfo>> errors = result.errorMap().entrySet(); + if (!keepGoing) { + // We may have multiple errors, but in non keep_going builds, we're obligated to print only + // one of them. + Preconditions.checkState(!errors.isEmpty(), result); + Entry<SkyKey, ErrorInfo> error = errors.iterator().next(); + ErrorInfo errorInfo = error.getValue(); + SkyKey topLevel = error.getKey(); + Label topLevelLabel = (Label) topLevel.argument(); + if (!Iterables.isEmpty(errorInfo.getCycleInfo())) { + skyframeCyclesReporter.get().reportCycles(errorInfo.getCycleInfo(), topLevel, eventHandler); + errorAboutLoadingFailure(topLevelLabel, null, eventHandler); + } else if (isDirectErrorFromTopLevelLabel(topLevelLabel, labelsToVisit, errorInfo)) { + // An error caused by a non-top-level label has already been reported during error + // bubbling but an error caused by the top-level non-target label itself hasn't been + // reported yet. Note that errors from top-level targets have already been reported + // during target parsing. + errorAboutLoadingFailure(topLevelLabel, errorInfo.getException(), eventHandler); + } + return false; + } + + for (Entry<SkyKey, ErrorInfo> errorEntry : errors) { + SkyKey key = errorEntry.getKey(); + ErrorInfo errorInfo = errorEntry.getValue(); + Preconditions.checkState(key.functionName().equals(SkyFunctions.TRANSITIVE_TARGET), errorEntry); + Label topLevelLabel = (Label) key.argument(); + if (!Iterables.isEmpty(errorInfo.getCycleInfo())) { + skyframeCyclesReporter.get().reportCycles(errorInfo.getCycleInfo(), key, eventHandler); + for (Label rootCause : getRootCausesOfCycles(topLevelLabel, errorInfo.getCycleInfo())) { + rootCauses.put(topLevelLabel, rootCause); + } + } + if (isDirectErrorFromTopLevelLabel(topLevelLabel, labelsToVisit, errorInfo)) { + // Unlike top-level targets, which have already gone through target parsing, + // errors directly coming from top-level labels have not been reported yet. + // + // See the note in the --nokeep_going case above. + eventHandler.handle(Event.error(errorInfo.getException().getMessage())); + } + warnAboutLoadingFailure(topLevelLabel, eventHandler); + for (SkyKey badKey : errorInfo.getRootCauses()) { + Preconditions.checkState(badKey.argument() instanceof Label, + "%s %s %s", key, errorInfo, badKey); + rootCauses.put(topLevelLabel, (Label) badKey.argument()); + } + } + for (Label topLevelLabel : result.<Label>keyNames()) { + SkyKey topLevelTransitiveTargetKey = TransitiveTargetValue.key(topLevelLabel); + TransitiveTargetValue topLevelTransitiveTargetValue = result.get(topLevelTransitiveTargetKey); + if (topLevelTransitiveTargetValue.getTransitiveRootCauses() != null) { + for (Label rootCause : topLevelTransitiveTargetValue.getTransitiveRootCauses()) { + rootCauses.put(topLevelLabel, rootCause); + } + warnAboutLoadingFailure(topLevelLabel, eventHandler); + } + } + return false; + } + + private static boolean hasErrors(EvaluationResult<TransitiveTargetValue> result) { + if (result.hasError()) { + return true; + } + for (TransitiveTargetValue transitiveTargetValue : result.values()) { + if (transitiveTargetValue.getTransitiveRootCauses() != null) { + return true; + } + } + return false; + } + + private static boolean isDirectErrorFromTopLevelLabel(Label label, Set<Label> topLevelLabels, + ErrorInfo errorInfo) { + return errorInfo.getException() != null && topLevelLabels.contains(label) + && Iterables.contains(errorInfo.getRootCauses(), TransitiveTargetValue.key(label)); + } + + private static void errorAboutLoadingFailure(Label topLevelLabel, @Nullable Throwable throwable, + EventHandler eventHandler) { + eventHandler.handle(Event.error( + "Loading of target '" + topLevelLabel + "' failed; build aborted" + + (throwable == null ? "" : ": " + throwable.getMessage()))); + } + + private static void warnAboutLoadingFailure(Label label, EventHandler eventHandler) { + eventHandler.handle(Event.warn( + // TODO(bazel-team): We use 'analyzing' here so that we print the same message as legacy + // Blaze. Once we get rid of legacy we should be able to change to 'loading' or + // similar. + "errors encountered while analyzing target '" + label + "': it will not be built")); + } + + private static Set<Label> getRootCausesOfCycles(Label labelToLoad, Iterable<CycleInfo> cycles) { + ImmutableSet.Builder<Label> builder = ImmutableSet.builder(); + for (CycleInfo cycleInfo : cycles) { + // The root cause of a cycle depends on the type of a cycle. + + SkyKey culprit = Iterables.getFirst(cycleInfo.getCycle(), null); + if (culprit == null) { + continue; + } + if (culprit.functionName().equals(SkyFunctions.TRANSITIVE_TARGET)) { + // For a cycle between build targets, the root cause is the first element of the cycle. + builder.add((Label) culprit.argument()); + } else { + // For other types of cycles (e.g. file symlink cycles), the root cause is the furthest + // target dependency that itself depended on the cycle. + Label furthestTarget = labelToLoad; + for (SkyKey skyKey : cycleInfo.getPathToCycle()) { + if (skyKey.functionName().equals(SkyFunctions.TRANSITIVE_TARGET)) { + furthestTarget = (Label) skyKey.argument(); + } else { + break; + } + } + builder.add(furthestTarget); + } + } + return builder.build(); + } + + // Unfortunately we have to do an effective O(TC) visitation after the eval() call above to + // determine all of the packages in the closure. + private void updateVisitedValues(Collection<TransitiveTargetValue> targetValues) { + Set<TransitiveTargetValue> currentBuildTargetValueSet = new HashSet<>(targetValues); + if (Objects.equals(previousBuildTargetValueSet, currentBuildTargetValueSet)) { + // The next stanza is slow (and scales with the edge count of the target graph), so avoid + // the computation if the previous build already did it. + return; + } + NestedSetBuilder<PackageIdentifier> nestedAllPkgsBuilder = NestedSetBuilder.stableOrder(); + NestedSetBuilder<PackageIdentifier> nestedErrorFreePkgsBuilder = NestedSetBuilder.stableOrder(); + NestedSetBuilder<Label> nestedTargetBuilder = NestedSetBuilder.stableOrder(); + for (TransitiveTargetValue value : targetValues) { + nestedAllPkgsBuilder.addTransitive(value.getTransitiveSuccessfulPackages()); + nestedAllPkgsBuilder.addTransitive(value.getTransitiveUnsuccessfulPackages()); + nestedErrorFreePkgsBuilder.addTransitive(value.getTransitiveSuccessfulPackages()); + nestedTargetBuilder.addTransitive(value.getTransitiveTargets()); + } + allVisitedPackages = nestedAllPkgsBuilder.build().toSet(); + errorFreeVisitedPackages = nestedErrorFreePkgsBuilder.build().toSet(); + visitedTargets = nestedTargetBuilder.build().toSet(); + previousBuildTargetValueSet = currentBuildTargetValueSet; + } + + + @Override + public Set<PackageIdentifier> getVisitedPackageNames() { + return allVisitedPackages; + } + + @Override + public Set<Package> getErrorFreeVisitedPackages() { + return transitivePackageLoader.retrievePackages(errorFreeVisitedPackages); + } + + /** + * Doesn't necessarily include all top-level targets visited in error, because of issues with + * skyframe semantics (e.g. impossible to load a target if it transitively depends on a file + * symlink cycle). This is actually fine for the non-test usages of this method since such bad + * targets get filtered out. + */ + @Override + public Set<Label> getVisitedTargets() { + return visitedTargets; + } + + @Override + public Multimap<Label, Label> getRootCauses(final Collection<Label> targetsToLoad) { + Preconditions.checkState(lastBuildKeepGoing); + return Multimaps.filterKeys(rootCauses, + new Predicate<Label>() { + @Override + public boolean apply(Label label) { + return targetsToLoad.contains(label); + } + }); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageLoaderWithValueEnvironment.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageLoaderWithValueEnvironment.java new file mode 100644 index 0000000..e467ae0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageLoaderWithValueEnvironment.java
@@ -0,0 +1,120 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.Fragment; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.analysis.config.PackageProviderForConfigurations; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor.SkyframePackageLoader; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.MemoizingEvaluator; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyKey; + +import java.io.IOException; +import java.util.Set; + +/** + * Repeats functionality of {@link SkyframePackageLoader} but uses + * {@link SkyFunction.Environment#getValue} instead of {@link MemoizingEvaluator#evaluate} + * for node evaluation + */ +class SkyframePackageLoaderWithValueEnvironment implements + PackageProviderForConfigurations { + private final SkyFunction.Environment env; + private final Set<Package> packages; + + public SkyframePackageLoaderWithValueEnvironment(SkyFunction.Environment env, + Set<Package> packages) { + this.env = env; + this.packages = packages; + } + + private Package getPackage(PackageIdentifier pkgIdentifier) throws NoSuchPackageException{ + SkyKey key = PackageValue.key(pkgIdentifier); + PackageValue value = (PackageValue) env.getValueOrThrow(key, NoSuchPackageException.class); + if (value != null) { + packages.add(value.getPackage()); + return value.getPackage(); + } + return null; + } + + @Override + public Package getLoadedPackage(final PackageIdentifier pkgIdentifier) + throws NoSuchPackageException { + try { + return getPackage(pkgIdentifier); + } catch (NoSuchPackageException e) { + if (e.getPackage() != null) { + return e.getPackage(); + } + throw e; + } + } + + @Override + public Target getLoadedTarget(Label label) throws NoSuchPackageException, + NoSuchTargetException { + Package pkg = getLoadedPackage(label.getPackageIdentifier()); + return pkg == null ? null : pkg.getTarget(label.getName()); + } + + @Override + public boolean isTargetCurrent(Target target) { + throw new UnsupportedOperationException(); + } + + @Override + public void addDependency(Package pkg, String fileName) throws SyntaxException, IOException { + RootedPath fileRootedPath = RootedPath.toRootedPath(pkg.getSourceRoot(), + pkg.getNameFragment().getRelative(fileName)); + FileValue result = (FileValue) env.getValue(FileValue.key(fileRootedPath)); + if (result != null && !result.exists()) { + throw new IOException(); + } + } + + @SuppressWarnings("unchecked") + @Override + public <T extends Fragment> T getFragment(BuildOptions buildOptions, Class<T> fragmentType) + throws InvalidConfigurationException { + ConfigurationFragmentValue fragmentNode = (ConfigurationFragmentValue) env.getValueOrThrow( + ConfigurationFragmentValue.key(buildOptions, fragmentType), + InvalidConfigurationException.class); + if (fragmentNode == null) { + return null; + } + return (T) fragmentNode.getFragment(); + } + + @Override + public BlazeDirectories getDirectories() { + return PrecomputedValue.BLAZE_DIRECTORIES.get(env); + } + + @Override + public boolean valuesMissing() { + return env.valuesMissing(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java new file mode 100644 index 0000000..cc32bf8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframePackageManager.java
@@ -0,0 +1,177 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.cmdline.LabelValidator; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.PackageManager; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator; +import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor.SkyframePackageLoader; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.UnixGlob; +import com.google.devtools.build.skyframe.CyclesReporter; + +import java.io.PrintStream; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Skyframe-based package manager. + * + * <p>This is essentially a compatibility shim between the native Skyframe and non-Skyframe + * parts of Blaze and should not be long-lived. + */ +class SkyframePackageManager implements PackageManager { + + private final SkyframePackageLoader packageLoader; + private final SkyframeExecutor.SkyframeTransitivePackageLoader transitiveLoader; + private final TargetPatternEvaluator patternEvaluator; + private final AtomicReference<UnixGlob.FilesystemCalls> syscalls; + private final AtomicReference<CyclesReporter> skyframeCyclesReporter; + private final AtomicReference<PathPackageLocator> pkgLocator; + private final AtomicInteger numPackagesLoaded; + private final SkyframeExecutor skyframeExecutor; + + public SkyframePackageManager(SkyframePackageLoader packageLoader, + SkyframeExecutor.SkyframeTransitivePackageLoader transitiveLoader, + TargetPatternEvaluator patternEvaluator, + AtomicReference<UnixGlob.FilesystemCalls> syscalls, + AtomicReference<CyclesReporter> skyframeCyclesReporter, + AtomicReference<PathPackageLocator> pkgLocator, + AtomicInteger numPackagesLoaded, + SkyframeExecutor skyframeExecutor) { + this.packageLoader = packageLoader; + this.transitiveLoader = transitiveLoader; + this.patternEvaluator = patternEvaluator; + this.skyframeCyclesReporter = skyframeCyclesReporter; + this.pkgLocator = pkgLocator; + this.syscalls = syscalls; + this.numPackagesLoaded = numPackagesLoaded; + this.skyframeExecutor = skyframeExecutor; + } + + @Override + public Package getLoadedPackage(PackageIdentifier pkgIdentifier) throws NoSuchPackageException { + return packageLoader.getLoadedPackage(pkgIdentifier); + } + + @ThreadSafe + @Override + public Package getPackage(EventHandler eventHandler, PackageIdentifier packageIdentifier) + throws NoSuchPackageException, InterruptedException { + try { + return packageLoader.getPackage(eventHandler, packageIdentifier); + } catch (NoSuchPackageException e) { + if (e.getPackage() != null) { + return e.getPackage(); + } + throw e; + } + } + + @Override + public Target getLoadedTarget(Label label) throws NoSuchPackageException, NoSuchTargetException { + return getLoadedPackage(label.getPackageIdentifier()).getTarget(label.getName()); + } + + @Override + public Target getTarget(EventHandler eventHandler, Label label) + throws NoSuchPackageException, NoSuchTargetException, InterruptedException { + return getPackage(eventHandler, label.getPackageIdentifier()).getTarget(label.getName()); + } + + @Override + public boolean isTargetCurrent(Target target) { + Package pkg = target.getPackage(); + try { + return getLoadedPackage(target.getLabel().getPackageIdentifier()) == pkg; + } catch (NoSuchPackageException e) { + return false; + } + } + + @Override + public void partiallyClear() { + packageLoader.partiallyClear(); + } + + @Override + public PackageManagerStatistics getStatistics() { + return new PackageManagerStatistics() { + @Override + public int getPackagesLoaded() { + return numPackagesLoaded.get(); + } + + @Override + public int getPackagesLookedUp() { + return -1; + } + + @Override + public int getCacheSize() { + return -1; + } + }; + } + + @Override + public boolean isPackage(String packageName) { + return getBuildFileForPackage(packageName) != null; + } + + @Override + public void dump(PrintStream printStream) { + skyframeExecutor.dumpPackages(printStream); + } + + @ThreadSafe + @Override + public Path getBuildFileForPackage(String packageName) { + // Note that this method needs to be thread-safe, as it is currently used concurrently by + // legacy blaze code. + if (packageLoader.isPackageDeleted(packageName) + || LabelValidator.validatePackageName(packageName) != null) { + return null; + } + // TODO(bazel-team): Use a PackageLookupValue here [skyframe-loading] + // TODO(bazel-team): The implementation in PackageCache also checks for duplicate packages, see + // BuildFileCache#getBuildFile [skyframe-loading] + return pkgLocator.get().getPackageBuildFileNullable(packageName, syscalls); + } + + @Override + public PathPackageLocator getPackagePath() { + return pkgLocator.get(); + } + + @Override + public TransitivePackageLoader newTransitiveLoader() { + return new SkyframeLabelVisitor(transitiveLoader, skyframeCyclesReporter); + } + + @Override + public TargetPatternEvaluator getTargetPatternEvaluator() { + return patternEvaluator; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeTargetPatternEvaluator.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeTargetPatternEvaluator.java new file mode 100644 index 0000000..9e619e3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeTargetPatternEvaluator.java
@@ -0,0 +1,146 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.cmdline.ResolvedTargets; +import com.google.devtools.build.lib.cmdline.TargetParsingException; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.FilteringPolicies; +import com.google.devtools.build.lib.pkgcache.FilteringPolicy; +import com.google.devtools.build.lib.pkgcache.ParseFailureListener; +import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.ErrorInfo; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.SkyKey; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * Skyframe-based target pattern parsing. + */ +final class SkyframeTargetPatternEvaluator implements TargetPatternEvaluator { + private final SkyframeExecutor skyframeExecutor; + private String offset = ""; + + SkyframeTargetPatternEvaluator(SkyframeExecutor skyframeExecutor) { + this.skyframeExecutor = skyframeExecutor; + } + + @Override + public ResolvedTargets<Target> parseTargetPatternList(EventHandler eventHandler, + List<String> targetPatterns, FilteringPolicy policy, boolean keepGoing) + throws TargetParsingException, InterruptedException { + return parseTargetPatternList(offset, eventHandler, targetPatterns, policy, keepGoing); + } + + @Override + public ResolvedTargets<Target> parseTargetPattern(EventHandler eventHandler, + String pattern, boolean keepGoing) throws TargetParsingException, InterruptedException { + return parseTargetPatternList(eventHandler, ImmutableList.of(pattern), + FilteringPolicies.NO_FILTER, keepGoing); + } + + @Override + public void updateOffset(PathFragment relativeWorkingDirectory) { + offset = relativeWorkingDirectory.getPathString(); + } + + @Override + public String getOffset() { + return offset; + } + + @Override + public Map<String, ResolvedTargets<Target>> preloadTargetPatterns(EventHandler eventHandler, + Collection<String> patterns, boolean keepGoing) + throws TargetParsingException, InterruptedException { + // TODO(bazel-team): This is used only in "blaze query". There are plans to dramatically change + // how query works on Skyframe, in which case this method is likely to go away. + // We cannot use an ImmutableMap here because there may be null values. + Map<String, ResolvedTargets<Target>> result = Maps.newHashMapWithExpectedSize(patterns.size()); + for (String pattern : patterns) { + // TODO(bazel-team): This could be parallelized to improve performance. [skyframe-loading] + result.put(pattern, parseTargetPattern(eventHandler, pattern, keepGoing)); + } + return result; + } + + /** + * Loads a list of target patterns (eg, "foo/..."). + */ + ResolvedTargets<Target> parseTargetPatternList(String offset, EventHandler eventHandler, + List<String> targetPatterns, FilteringPolicy policy, boolean keepGoing) + throws InterruptedException, TargetParsingException { + Iterable<SkyKey> patternSkyKeys = TargetPatternValue.keys(targetPatterns, policy, offset); + EvaluationResult<TargetPatternValue> result = + skyframeExecutor.targetPatterns(patternSkyKeys, keepGoing, eventHandler); + + String errorMessage = null; + ResolvedTargets.Builder<Target> builder = ResolvedTargets.builder(); + for (SkyKey key : patternSkyKeys) { + TargetPatternValue resultValue = result.get(key); + if (resultValue != null) { + ResolvedTargets<Target> results = resultValue.getTargets(); + if (((TargetPatternValue.TargetPattern) key.argument()).isNegative()) { + builder.filter(Predicates.not(Predicates.in(results.getTargets()))); + } else { + builder.merge(results); + } + } else { + TargetPatternValue.TargetPattern pattern = + (TargetPatternValue.TargetPattern) key.argument(); + String rawPattern = pattern.getPattern(); + ErrorInfo error = result.errorMap().get(key); + if (error == null) { + Preconditions.checkState(!keepGoing); + continue; + } + if (error.getException() != null) { + errorMessage = error.getException().getMessage(); + } else if (!Iterables.isEmpty(error.getCycleInfo())) { + errorMessage = "cycles detected during target parsing"; + skyframeExecutor.getCyclesReporter().reportCycles( + error.getCycleInfo(), key, eventHandler); + } else { + throw new IllegalStateException(error.toString()); + } + if (keepGoing) { + eventHandler.handle(Event.error("Skipping '" + rawPattern + "': " + errorMessage)); + } + builder.setError(); + + if (eventHandler instanceof ParseFailureListener) { + ParseFailureListener parseListener = (ParseFailureListener) eventHandler; + parseListener.parsingError(rawPattern, errorMessage); + } + } + } + + if (!keepGoing && result.hasError()) { + Preconditions.checkState(errorMessage != null, "unexpected errors: %s", result.errorMap()); + throw new TargetParsingException(errorMessage); + } + return builder.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkFileDependency.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkFileDependency.java new file mode 100644 index 0000000..02d2e91 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkFileDependency.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.syntax.Label; + +/** + * A simple value class to store the direct Skylark file dependencies of a Skylark + * extension file. It also contains a Label identifying the extension file. + */ +class SkylarkFileDependency { + + private final Label label; + private final ImmutableList<SkylarkFileDependency> dependencies; + + SkylarkFileDependency(Label label, ImmutableList<SkylarkFileDependency> dependencies) { + this.label = label; + this.dependencies = dependencies; + } + + /** + * Returns the list of direct Skylark file dependencies of the Skylark extension file + * corresponding to this object. + */ + ImmutableList<SkylarkFileDependency> getDependencies() { + return dependencies; + } + + /** + * Returns the Label of the Skylark extension file corresponding to this object. + */ + Label getLabel() { + return label; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupFunction.java new file mode 100644 index 0000000..02d41e6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupFunction.java
@@ -0,0 +1,238 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.packages.BuildFileNotFoundException; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; +import com.google.devtools.build.lib.packages.RuleClassProvider; +import com.google.devtools.build.lib.skyframe.ASTFileLookupValue.ASTLookupInputException; +import com.google.devtools.build.lib.syntax.BuildFileAST; +import com.google.devtools.build.lib.syntax.Function; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.HashMap; +import java.util.Map; + +/** + * A Skyframe function to look up and import a single Skylark extension. + */ +public class SkylarkImportLookupFunction implements SkyFunction { + + private final RuleClassProvider ruleClassProvider; + private final ImmutableList<Function> nativeRuleFunctions; + + public SkylarkImportLookupFunction( + RuleClassProvider ruleClassProvider, PackageFactory packageFactory) { + this.ruleClassProvider = ruleClassProvider; + this.nativeRuleFunctions = packageFactory.collectNativeRuleFunctions(); + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, + InterruptedException { + PackageIdentifier arg = (PackageIdentifier) skyKey.argument(); + PathFragment file = arg.getPackageFragment(); + ASTFileLookupValue astLookupValue = null; + try { + SkyKey astLookupKey = ASTFileLookupValue.key(file); + astLookupValue = (ASTFileLookupValue) env.getValueOrThrow(astLookupKey, + ErrorReadingSkylarkExtensionException.class, InconsistentFilesystemException.class); + } catch (ErrorReadingSkylarkExtensionException e) { + throw new SkylarkImportLookupFunctionException(SkylarkImportFailedException.errorReadingFile( + file, e.getMessage())); + } catch (InconsistentFilesystemException e) { + throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT); + } catch (ASTLookupInputException e) { + throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT); + } + if (astLookupValue == null) { + return null; + } + if (astLookupValue == ASTFileLookupValue.NO_FILE) { + // Skylark import files have to exist. + throw new SkylarkImportLookupFunctionException(SkylarkImportFailedException.noFile(file)); + } + + Map<PathFragment, SkylarkEnvironment> importMap = new HashMap<>(); + ImmutableList.Builder<SkylarkFileDependency> fileDependencies = ImmutableList.builder(); + BuildFileAST ast = astLookupValue.getAST(); + // TODO(bazel-team): Refactor this code and PackageFunction to reduce code duplications. + for (PathFragment importFile : ast.getImports()) { + try { + SkyKey importsLookupKey = SkylarkImportLookupValue.key(arg.getRepository(), importFile); + SkylarkImportLookupValue importsLookupValue; + importsLookupValue = (SkylarkImportLookupValue) env.getValueOrThrow( + importsLookupKey, ASTLookupInputException.class); + if (importsLookupValue != null) { + importMap.put(importFile, importsLookupValue.getImportedEnvironment()); + fileDependencies.add(importsLookupValue.getDependency()); + } + } catch (ASTLookupInputException e) { + throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT); + } + } + Label label = pathFragmentToLabel(arg.getRepository(), file, env); + if (env.valuesMissing()) { + // This means some imports are unavailable. + return null; + } + + if (ast.containsErrors()) { + throw new SkylarkImportLookupFunctionException(SkylarkImportFailedException.skylarkErrors( + file)); + } + + SkylarkEnvironment extensionEnv = createEnv(ast, importMap, env); + // Skylark UserDefinedFunctions are sharing function definition Environments, so it's extremely + // important not to modify them from this point. Ideally they should be only used to import + // symbols and serve as global Environments of UserDefinedFunctions. + return new SkylarkImportLookupValue( + extensionEnv, new SkylarkFileDependency(label, fileDependencies.build())); + } + + /** + * Converts the PathFragment of the Skylark file to a Label using the BUILD file closest to the + * Skylark file in its directory hierarchy - finds the package to which the Skylark file belongs. + * Throws an exception if no such BUILD file exists. + */ + private Label pathFragmentToLabel(RepositoryName repo, PathFragment file, Environment env) + throws SkylarkImportLookupFunctionException { + ContainingPackageLookupValue containingPackageLookupValue = null; + try { + PackageIdentifier newPkgId = new PackageIdentifier(repo, file.getParentDirectory()); + containingPackageLookupValue = (ContainingPackageLookupValue) env.getValueOrThrow( + ContainingPackageLookupValue.key(newPkgId), + BuildFileNotFoundException.class, InconsistentFilesystemException.class); + } catch (BuildFileNotFoundException e) { + // Thrown when there are IO errors looking for BUILD files. + throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT); + } catch (InconsistentFilesystemException e) { + throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT); + } + + if (containingPackageLookupValue == null) { + return null; + } + + if (!containingPackageLookupValue.hasContainingPackage()) { + throw new SkylarkImportLookupFunctionException( + SkylarkImportFailedException.noBuildFile(file)); + } + + PathFragment pkgName = + containingPackageLookupValue.getContainingPackageName().getPackageFragment(); + PathFragment fileInPkg = file.relativeTo(pkgName); + + try { + // This code relies on PackageIdentifier.RepositoryName.toString() + return Label.parseRepositoryLabel(repo + "//" + pkgName.getPathString() + ":" + fileInPkg); + } catch (SyntaxException e) { + throw new IllegalStateException(e); + } + } + + /** + * Creates the SkylarkEnvironment to be imported. After it's returned, the Environment + * must not be modified. + */ + private SkylarkEnvironment createEnv(BuildFileAST ast, + Map<PathFragment, SkylarkEnvironment> importMap, Environment env) + throws InterruptedException { + StoredEventHandler eventHandler = new StoredEventHandler(); + // TODO(bazel-team): this method overestimates the changes which can affect the + // Skylark RuleClass. For example changes to comments or unused functions can modify the hash. + // A more accurate - however much more complicated - way would be to calculate a hash based on + // the transitive closure of the accessible AST nodes. + SkylarkEnvironment extensionEnv = ruleClassProvider + .createSkylarkRuleClassEnvironment(eventHandler, ast.getContentHashCode()); + // Adding native rules module for build extensions. + // TODO(bazel-team): this might not be the best place to do this. + extensionEnv.update("native", ruleClassProvider.getNativeModule()); + for (Function function : nativeRuleFunctions) { + extensionEnv.registerFunction( + ruleClassProvider.getNativeModule().getClass(), function.getName(), function); + } + extensionEnv.setImportedExtensions(importMap); + ast.exec(extensionEnv, eventHandler); + // Don't fail just replay the events so the original package lookup can fail. + Event.replayEventsOn(env.getListener(), eventHandler.getEvents()); + return extensionEnv; + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + static final class SkylarkImportFailedException extends Exception { + private SkylarkImportFailedException(String errorMessage) { + super(errorMessage); + } + + static SkylarkImportFailedException errorReadingFile(PathFragment file, String error) { + return new SkylarkImportFailedException( + String.format("Encountered error while reading extension file '%s': %s", file, error)); + } + + static SkylarkImportFailedException noFile(PathFragment file) { + return new SkylarkImportFailedException( + String.format("Extension file not found: '%s'", file)); + } + + static SkylarkImportFailedException noBuildFile(PathFragment file) { + return new SkylarkImportFailedException( + String.format("Every .bzl file must have a corresponding package, but '%s' " + + "does not have one. Please create a BUILD file in the same or any parent directory." + + " Note that this BUILD file does not need to do anything except exist.", file)); + } + + static SkylarkImportFailedException skylarkErrors(PathFragment file) { + return new SkylarkImportFailedException(String.format("Extension '%s' has errors", file)); + } + } + + private static final class SkylarkImportLookupFunctionException extends SkyFunctionException { + private SkylarkImportLookupFunctionException(SkylarkImportFailedException cause) { + super(cause, Transience.PERSISTENT); + } + + private SkylarkImportLookupFunctionException(InconsistentFilesystemException e, + Transience transience) { + super(e, transience); + } + + private SkylarkImportLookupFunctionException(ASTLookupInputException e, + Transience transience) { + super(e, transience); + } + + private SkylarkImportLookupFunctionException(BuildFileNotFoundException e, + Transience transience) { + super(e, transience); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupValue.java new file mode 100644 index 0000000..3c87431 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupValue.java
@@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName; +import com.google.devtools.build.lib.skyframe.ASTFileLookupValue.ASTLookupInputException; +import com.google.devtools.build.lib.syntax.SkylarkEnvironment; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * A value that represents a Skylark import lookup result. The lookup value corresponds to + * exactly one Skylark file, identified by the PathFragment SkyKey argument. + */ +public class SkylarkImportLookupValue implements SkyValue { + + private final SkylarkEnvironment importedEnvironment; + /** + * The immediate Skylark file dependency descriptor class corresponding to this value. + * Using this reference it's possible to reach the transitive closure of Skylark files + * on which this Skylark file depends. + */ + private final SkylarkFileDependency dependency; + + public SkylarkImportLookupValue( + SkylarkEnvironment importedEnvironment, SkylarkFileDependency dependency) { + this.importedEnvironment = Preconditions.checkNotNull(importedEnvironment); + this.dependency = Preconditions.checkNotNull(dependency); + } + + /** + * Returns the imported SkylarkEnvironment. + */ + public SkylarkEnvironment getImportedEnvironment() { + return importedEnvironment; + } + + /** + * Returns the immediate Skylark file dependency corresponding to this import lookup value. + */ + public SkylarkFileDependency getDependency() { + return dependency; + } + + static SkyKey key(PackageIdentifier pkgIdentifier) throws ASTLookupInputException { + return key(pkgIdentifier.getRepository(), pkgIdentifier.getPackageFragment()); + } + + static SkyKey key(RepositoryName repo, PathFragment fileToImport) throws ASTLookupInputException { + // Skylark import lookup keys need to be valid AST file lookup keys. + ASTFileLookupValue.checkInputArgument(fileToImport); + return new SkyKey( + SkyFunctions.SKYLARK_IMPORTS_LOOKUP, + new PackageIdentifier(repo, fileToImport)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkModuleCycleReporter.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkModuleCycleReporter.java new file mode 100644 index 0000000..a0f37a9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkylarkModuleCycleReporter.java
@@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.skyframe.CycleInfo; +import com.google.devtools.build.skyframe.CyclesReporter; +import com.google.devtools.build.skyframe.SkyKey; + +/** + * Reports cycles of recursive import of Skylark files. + */ +public class SkylarkModuleCycleReporter implements CyclesReporter.SingleCycleReporter { + + private static final Predicate<SkyKey> IS_SKYLARK_MODULE_SKY_KEY = + SkyFunctions.isSkyFunction(SkyFunctions.SKYLARK_IMPORTS_LOOKUP); + + private static final Predicate<SkyKey> IS_PACKAGE_SKY_KEY = + SkyFunctions.isSkyFunction(SkyFunctions.PACKAGE); + + @Override + public boolean maybeReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo, boolean alreadyReported, + EventHandler eventHandler) { + ImmutableList<SkyKey> pathToCycle = cycleInfo.getPathToCycle(); + if (pathToCycle.size() == 0) { + return false; + } + SkyKey lastPathElement = cycleInfo.getPathToCycle().get(pathToCycle.size() - 1); + if (alreadyReported) { + return true; + } else if (Iterables.all(cycleInfo.getCycle(), IS_SKYLARK_MODULE_SKY_KEY) + // The last element of the path to the cycle has to be a PackageFunction. + && IS_PACKAGE_SKY_KEY.apply(lastPathElement)) { + StringBuilder cycleMessage = new StringBuilder() + .append(((PackageIdentifier) lastPathElement.argument()).toString() + "/BUILD: ") + .append("cycle in referenced extension files: "); + + AbstractLabelCycleReporter.printCycle(cycleInfo.getCycle(), cycleMessage, + new Function<SkyKey, String>() { + @Override + public String apply(SkyKey input) { + return ((PackageIdentifier) input.argument()).toString(); + } + }); + + // TODO(bazel-team): it would be nice to pass the Location of the load Statement in the + // BUILD file. + eventHandler.handle(Event.error(null, cycleMessage.toString())); + return true; + } + return false; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionFunction.java new file mode 100644 index 0000000..2e4aea9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionFunction.java
@@ -0,0 +1,138 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.MissingInputFileException; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.LabelAndConfiguration; +import com.google.devtools.build.lib.analysis.TargetCompleteEvent; +import com.google.devtools.build.lib.analysis.TopLevelArtifactContext; +import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.skyframe.ValueOrException2; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.Nullable; + +/** + * TargetCompletionFunction builds the artifactsToBuild collection of a {@link ConfiguredTarget}. + */ +public final class TargetCompletionFunction implements SkyFunction { + + private final AtomicReference<EventBus> eventBusRef; + + public TargetCompletionFunction(AtomicReference<EventBus> eventBusRef) { + this.eventBusRef = eventBusRef; + } + + @Nullable + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws TargetCompletionFunctionException { + LabelAndConfiguration lac = (LabelAndConfiguration) skyKey.argument(); + ConfiguredTargetValue ctValue = (ConfiguredTargetValue) + env.getValue(ConfiguredTargetValue.key(lac.getLabel(), lac.getConfiguration())); + TopLevelArtifactContext topLevelContext = PrecomputedValue.TOP_LEVEL_CONTEXT.get(env); + if (env.valuesMissing()) { + return null; + } + + Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> inputDeps = + env.getValuesOrThrow(ArtifactValue.mandatoryKeys( + TopLevelArtifactHelper.getAllArtifactsToBuild( + ctValue.getConfiguredTarget(), topLevelContext)), + MissingInputFileException.class, ActionExecutionException.class); + + int missingCount = 0; + ActionExecutionException firstActionExecutionException = null; + MissingInputFileException missingInputException = null; + NestedSetBuilder<Label> rootCausesBuilder = NestedSetBuilder.stableOrder(); + for (Map.Entry<SkyKey, ValueOrException2<MissingInputFileException, + ActionExecutionException>> depsEntry : inputDeps.entrySet()) { + Artifact input = ArtifactValue.artifact(depsEntry.getKey()); + try { + depsEntry.getValue().get(); + } catch (MissingInputFileException e) { + missingCount++; + if (input.getOwner() != null) { + rootCausesBuilder.add(input.getOwner()); + env.getListener().handle(Event.error( + ctValue.getConfiguredTarget().getTarget().getLocation(), + String.format("%s: missing input file '%s'", + lac.getLabel(), input.getOwner()))); + } + } catch (ActionExecutionException e) { + rootCausesBuilder.addTransitive(e.getRootCauses()); + if (firstActionExecutionException == null) { + firstActionExecutionException = e; + } + } + } + + if (missingCount > 0) { + missingInputException = new MissingInputFileException( + ctValue.getConfiguredTarget().getTarget().getLocation() + " " + missingCount + + " input file(s) do not exist", ctValue.getConfiguredTarget().getTarget().getLocation()); + } + + NestedSet<Label> rootCauses = rootCausesBuilder.build(); + if (!rootCauses.isEmpty()) { + eventBusRef.get().post( + TargetCompleteEvent.createFailed(ctValue.getConfiguredTarget(), rootCauses)); + if (firstActionExecutionException != null) { + throw new TargetCompletionFunctionException(firstActionExecutionException); + } else { + throw new TargetCompletionFunctionException(missingInputException); + } + } + + return env.valuesMissing() ? null : new TargetCompletionValue(ctValue.getConfiguredTarget()); + } + + @Override + public String extractTag(SkyKey skyKey) { + return Label.print(((LabelAndConfiguration) skyKey.argument()).getLabel()); + } + + private static final class TargetCompletionFunctionException extends SkyFunctionException { + + private final ActionExecutionException actionException; + + public TargetCompletionFunctionException(ActionExecutionException e) { + super(e, Transience.PERSISTENT); + this.actionException = e; + } + + public TargetCompletionFunctionException(MissingInputFileException e) { + super(e, Transience.TRANSIENT); + this.actionException = null; + } + + @Override + public boolean isCatastrophic() { + return actionException != null && actionException.isCatastrophe(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionValue.java new file mode 100644 index 0000000..5d7153d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetCompletionValue.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.LabelAndConfiguration; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.Collection; + +/** + * The value of a TargetCompletion. Currently this just stores a ConfiguredTarget. + */ +public class TargetCompletionValue implements SkyValue { + private final ConfiguredTarget ct; + + TargetCompletionValue(ConfiguredTarget ct) { + this.ct = ct; + } + + public ConfiguredTarget getConfiguredTarget() { + return ct; + } + + public static SkyKey key(LabelAndConfiguration labelAndConfiguration) { + return new SkyKey(SkyFunctions.TARGET_COMPLETION, labelAndConfiguration); + } + + public static Iterable<SkyKey> keys(Collection<ConfiguredTarget> targets) { + return Iterables.transform(targets, new Function<ConfiguredTarget, SkyKey>() { + @Override + public SkyKey apply(ConfiguredTarget ct) { + return new SkyKey(SkyFunctions.TARGET_COMPLETION, new LabelAndConfiguration(ct)); + } + }); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerFunction.java new file mode 100644 index 0000000..3f9f22f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerFunction.java
@@ -0,0 +1,134 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.packages.BuildFileNotFoundException; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * A SkyFunction for {@link TargetMarkerValue}s. + */ +public final class TargetMarkerFunction implements SkyFunction { + + public TargetMarkerFunction() { + } + + @Override + public SkyValue compute(SkyKey key, Environment env) throws TargetMarkerFunctionException { + Label label = (Label) key.argument(); + PathFragment pkgForLabel = label.getPackageFragment(); + + if (label.getName().contains("/")) { + // This target is in a subdirectory, therefore it could potentially be invalidated by + // a new BUILD file appearing in the hierarchy. + PathFragment containingDirectory = label.toPathFragment().getParentDirectory(); + ContainingPackageLookupValue containingPackageLookupValue = null; + try { + PackageIdentifier newPkgId = new PackageIdentifier( + label.getPackageIdentifier().getRepository(), containingDirectory); + containingPackageLookupValue = (ContainingPackageLookupValue) env.getValueOrThrow( + ContainingPackageLookupValue.key(newPkgId), + BuildFileNotFoundException.class, InconsistentFilesystemException.class); + } catch (BuildFileNotFoundException e) { + // Thrown when there are IO errors looking for BUILD files. + throw new TargetMarkerFunctionException(e); + } catch (InconsistentFilesystemException e) { + throw new TargetMarkerFunctionException(new NoSuchTargetException(label, + e.getMessage())); + } + if (containingPackageLookupValue == null) { + return null; + } + if (!containingPackageLookupValue.hasContainingPackage()) { + // This means the label's package doesn't exist. E.g. there is no package 'a' and we are + // trying to build the target for label 'a:b/foo'. + throw new TargetMarkerFunctionException(new BuildFileNotFoundException( + pkgForLabel.getPathString(), "BUILD file not found on package path for '" + + pkgForLabel.getPathString() + "'")); + } + if (!containingPackageLookupValue.getContainingPackageName().equals( + label.getPackageIdentifier())) { + throw new TargetMarkerFunctionException(new NoSuchTargetException(label, + String.format("Label '%s' crosses boundary of subpackage '%s'", label, + containingPackageLookupValue.getContainingPackageName()))); + } + } + + SkyKey pkgSkyKey = PackageValue.key(label.getPackageIdentifier()); + NoSuchPackageException nspe = null; + Package pkg; + try { + PackageValue value = (PackageValue) + env.getValueOrThrow(pkgSkyKey, NoSuchPackageException.class); + if (value == null) { + return null; + } + pkg = value.getPackage(); + } catch (NoSuchPackageException e) { + // For consistency with pre-Skyframe Blaze, we can return a valid Target from a Package + // containing errors. + pkg = e.getPackage(); + if (pkg == null) { + // Re-throw this exception with our key because root causes should be targets, not packages. + throw new TargetMarkerFunctionException(e); + } + nspe = e; + } + + Target target; + try { + target = pkg.getTarget(label.getName()); + } catch (NoSuchTargetException e) { + throw new TargetMarkerFunctionException(e); + } + + if (nspe != null) { + // There is a target, but its package is in error. We rethrow so that the root cause is the + // target, not the package. Note that targets are only in error when their package is + // "in error" (because a package is in error if there was an error evaluating the package, or + // if one of its targets was in error). + throw new TargetMarkerFunctionException(new NoSuchTargetException(target, nspe)); + } + return TargetMarkerValue.TARGET_MARKER_INSTANCE; + } + + @Override + public String extractTag(SkyKey skyKey) { + return Label.print((Label) skyKey.argument()); + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link TargetMarkerFunction#compute}. + */ + private static final class TargetMarkerFunctionException extends SkyFunctionException { + public TargetMarkerFunctionException(NoSuchTargetException e) { + super(e, Transience.PERSISTENT); + } + + public TargetMarkerFunctionException(NoSuchPackageException e) { + super(e, Transience.PERSISTENT); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerValue.java new file mode 100644 index 0000000..eef7084 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetMarkerValue.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * Value represents visited target in the Skyframe graph after error checking. + */ +@Immutable +@ThreadSafe +public final class TargetMarkerValue implements SkyValue { + + static final TargetMarkerValue TARGET_MARKER_INSTANCE = new TargetMarkerValue(); + + private TargetMarkerValue() { + } + + @ThreadSafe + public static SkyKey key(Label label) { + return new SkyKey(SkyFunctions.TARGET_MARKER, label); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternFunction.java new file mode 100644 index 0000000..6e631ed --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternFunction.java
@@ -0,0 +1,278 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.cmdline.LabelValidator; +import com.google.devtools.build.lib.cmdline.ResolvedTargets; +import com.google.devtools.build.lib.cmdline.TargetParsingException; +import com.google.devtools.build.lib.cmdline.TargetPattern; +import com.google.devtools.build.lib.cmdline.TargetPatternResolver; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.FilteringPolicies; +import com.google.devtools.build.lib.pkgcache.FilteringPolicy; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.pkgcache.TargetPatternResolverUtil; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.Nullable; + +/** + * TargetPatternFunction translates a target pattern (eg, "foo/...") into a set of resolved + * Targets. + */ +public class TargetPatternFunction implements SkyFunction { + + private final AtomicReference<PathPackageLocator> pkgPath; + + public TargetPatternFunction(AtomicReference<PathPackageLocator> pkgPath) { + this.pkgPath = pkgPath; + } + + @Override + public SkyValue compute(SkyKey key, Environment env) throws TargetPatternFunctionException, + InterruptedException { + TargetPatternValue.TargetPattern patternKey = + ((TargetPatternValue.TargetPattern) key.argument()); + + TargetPattern.Parser parser = new TargetPattern.Parser(patternKey.getOffset()); + try { + Resolver resolver = new Resolver(env, patternKey.getPolicy(), pkgPath); + TargetPattern resolvedPattern = parser.parse(patternKey.getPattern()); + return new TargetPatternValue(resolvedPattern.eval(resolver)); + } catch (TargetParsingException e) { + throw new TargetPatternFunctionException(e); + } catch (TargetPatternResolver.MissingDepException e) { + return null; + } + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + private static class Resolver implements TargetPatternResolver<Target> { + private final Environment env; + private final FilteringPolicy policy; + private final AtomicReference<PathPackageLocator> pkgPath; + + public Resolver(Environment env, FilteringPolicy policy, + AtomicReference<PathPackageLocator> pkgPath) { + this.policy = policy; + this.env = env; + this.pkgPath = pkgPath; + } + + @Override + public void warn(String msg) { + env.getListener().handle(Event.warn(msg)); + } + + /** + * Gets a Package via the Skyframe env. May return a Package that has errors. + */ + private Package getPackage(PackageIdentifier pkgIdentifier) + throws MissingDepException, NoSuchThingException { + SkyKey pkgKey = PackageValue.key(pkgIdentifier); + Package pkg; + try { + PackageValue pkgValue = + (PackageValue) env.getValueOrThrow(pkgKey, NoSuchThingException.class); + if (pkgValue == null) { + throw new MissingDepException(); + } + pkg = pkgValue.getPackage(); + } catch (NoSuchPackageException e) { + pkg = e.getPackage(); + if (pkg == null) { + throw e; + } + } + return pkg; + } + + @Override + public Target getTargetOrNull(String targetName) throws InterruptedException, + MissingDepException { + try { + Label label = Label.parseAbsolute(targetName); + if (!isPackage(label.getPackageName())) { + return null; + } + Package pkg = getPackage(label.getPackageIdentifier()); + return pkg.getTarget(label.getName()); + } catch (Label.SyntaxException | NoSuchThingException e) { + return null; + } + } + + @Override + public ResolvedTargets<Target> getExplicitTarget(String targetName) + throws TargetParsingException, InterruptedException, MissingDepException { + Label label = TargetPatternResolverUtil.label(targetName); + try { + Package pkg = getPackage(label.getPackageIdentifier()); + Target target = pkg.getTarget(label.getName()); + return policy.shouldRetain(target, true) + ? ResolvedTargets.of(target) + : ResolvedTargets.<Target>empty(); + } catch (NoSuchThingException e) { + throw new TargetParsingException(e.getMessage(), e); + } + } + + @Override + public ResolvedTargets<Target> getTargetsInPackage(String originalPattern, String packageName, + boolean rulesOnly) + throws TargetParsingException, InterruptedException, MissingDepException { + FilteringPolicy actualPolicy = rulesOnly + ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy) + : policy; + return getTargetsInPackage(originalPattern, packageName, actualPolicy); + } + + private ResolvedTargets<Target> getTargetsInPackage(String originalPattern, String packageName, + FilteringPolicy policy) + throws TargetParsingException, MissingDepException { + // Normalise, e.g "foo//bar" -> "foo/bar"; "foo/" -> "foo": + PathFragment packageNameFragment = new PathFragment(packageName); + packageName = packageNameFragment.toString(); + + // It's possible for this check to pass, but for + // Label.validatePackageNameFull to report an error because the + // package name is illegal. That's a little weird, but we can live with + // that for now--see test case: testBadPackageNameButGoodEnoughForALabel. + // (BTW I tried duplicating that validation logic in Label but it was + // extremely tricky.) + if (LabelValidator.validatePackageName(packageName) != null) { + throw new TargetParsingException("'" + packageName + "' is not a valid package name"); + } + if (!isPackage(packageName)) { + throw new TargetParsingException( + TargetPatternResolverUtil.getParsingErrorMessage( + "no such package '" + packageName + "': BUILD file not found on package path", + originalPattern)); + } + + try { + Package pkg = getPackage( + PackageIdentifier.createInDefaultRepo(packageNameFragment.toString())); + return TargetPatternResolverUtil.resolvePackageTargets(pkg, policy); + } catch (NoSuchThingException e) { + String message = TargetPatternResolverUtil.getParsingErrorMessage( + "package contains errors", originalPattern); + throw new TargetParsingException(message, e); + } + } + + @Override + public boolean isPackage(String packageName) throws MissingDepException { + SkyKey packageLookupKey; + packageLookupKey = PackageLookupValue.key(new PathFragment(packageName)); + PackageLookupValue packageLookupValue = (PackageLookupValue) env.getValue(packageLookupKey); + if (packageLookupValue == null) { + throw new MissingDepException(); + } + return packageLookupValue.packageExists(); + } + + @Override + public String getTargetKind(Target target) { + return target.getTargetKind(); + } + + @Override + public ResolvedTargets<Target> findTargetsBeneathDirectory( + String originalPattern, String pathPrefix, boolean rulesOnly) + throws TargetParsingException, MissingDepException { + FilteringPolicy actualPolicy = rulesOnly + ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy) + : policy; + + PathFragment directory = new PathFragment(pathPrefix); + if (directory.containsUplevelReferences()) { + throw new TargetParsingException("up-level references are not permitted: '" + + directory.getPathString() + "'"); + } + if (!pathPrefix.isEmpty() && (LabelValidator.validatePackageName(pathPrefix) != null)) { + throw new TargetParsingException("'" + pathPrefix + "' is not a valid package name"); + } + + ResolvedTargets.Builder<Target> builder = ResolvedTargets.builder(); + + List<RecursivePkgValue> lookupValues = new ArrayList<>(); + for (Path root : pkgPath.get().getPathEntries()) { + SkyKey key = RecursivePkgValue.key(RootedPath.toRootedPath(root, directory)); + RecursivePkgValue lookup = (RecursivePkgValue) env.getValue(key); + if (lookup != null) { + lookupValues.add(lookup); + } + } + if (env.valuesMissing()) { + throw new MissingDepException(); + } + + for (RecursivePkgValue value : lookupValues) { + for (String pkg : value.getPackages()) { + builder.merge(getTargetsInPackage(originalPattern, pkg, FilteringPolicies.NO_FILTER)); + } + } + + if (builder.isEmpty()) { + throw new TargetParsingException("no targets found beneath '" + directory + "'"); + } + + // Apply the transform after the check so we only return the + // error if the tree really contains no targets. + ResolvedTargets<Target> intermediateResult = builder.build(); + ResolvedTargets.Builder<Target> filteredBuilder = ResolvedTargets.builder(); + if (intermediateResult.hasError()) { + filteredBuilder.setError(); + } + for (Target target : intermediateResult.getTargets()) { + if (actualPolicy.shouldRetain(target, false)) { + filteredBuilder.add(target); + } + } + return filteredBuilder.build(); + } + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link TargetPatternFunction#compute}. + */ + private static final class TargetPatternFunctionException extends SkyFunctionException { + public TargetPatternFunctionException(TargetParsingException e) { + super(e, Transience.PERSISTENT); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternValue.java new file mode 100644 index 0000000..c97194f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternValue.java
@@ -0,0 +1,212 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.cmdline.ResolvedTargets; +import com.google.devtools.build.lib.cmdline.ResolvedTargets.Builder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.FilteringPolicies; +import com.google.devtools.build.lib.pkgcache.FilteringPolicy; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * A value referring to a computed set of resolved targets. This is used for the results of target + * pattern parsing. + */ +@Immutable +@ThreadSafe +public final class TargetPatternValue implements SkyValue { + + private ResolvedTargets<Target> targets; + + TargetPatternValue(ResolvedTargets<Target> targets) { + this.targets = Preconditions.checkNotNull(targets); + } + + private void writeObject(ObjectOutputStream out) throws IOException { + Set<Package> packages = new LinkedHashSet<>(); + List<String> ts = new ArrayList<>(); + List<String> filteredTs = new ArrayList<>(); + for (Target target : targets.getTargets()) { + packages.add(target.getPackage()); + ts.add(target.getLabel().toString()); + } + for (Target target : targets.getFilteredTargets()) { + packages.add(target.getPackage()); + filteredTs.add(target.getLabel().toString()); + } + + out.writeObject(packages); + out.writeObject(ts); + out.writeObject(filteredTs); + } + + @SuppressWarnings("unchecked") + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + Set<Package> packages = (Set<Package>) in.readObject(); + List<String> ts = (List<String>) in.readObject(); + List<String> filteredTs = (List<String>) in.readObject(); + + Map<String, Package> packageMap = new HashMap<>(); + for (Package p : packages) { + packageMap.put(p.getName(), p); + } + + Builder<Target> builder = ResolvedTargets.<Target>builder(); + for (String labelString : ts) { + builder.add(lookupTarget(packageMap, labelString)); + } + + for (String labelString : filteredTs) { + builder.remove(lookupTarget(packageMap, labelString)); + } + this.targets = builder.build(); + } + + private static Target lookupTarget(Map<String, Package> packageMap, String labelString) { + Label label = Label.parseAbsoluteUnchecked(labelString); + Package p = packageMap.get(label.getPackageName()); + try { + return p.getTarget(label.getName()); + } catch (NoSuchTargetException e) { + throw new IllegalStateException(e); + } + } + + @SuppressWarnings("unused") + private void readObjectNoData() { + throw new IllegalStateException(); + } + + /** + * Create a target pattern value key. + * + * @param pattern The pattern, eg "-foo/biz...". If the first character is "-", the pattern + * is treated as a negative pattern. + * @param policy The filtering policy, eg "only return test targets" + * @param offset The offset to apply to relative target patterns. + */ + @ThreadSafe + public static SkyKey key(String pattern, + FilteringPolicy policy, + String offset) { + return new SkyKey(SkyFunctions.TARGET_PATTERN, + pattern.startsWith("-") + // Don't apply filters to negative patterns. + ? new TargetPattern(pattern.substring(1), FilteringPolicies.NO_FILTER, true, offset) + : new TargetPattern(pattern, policy, false, offset)); + } + + /** + * Like above, but accepts a collection of target patterns for the same filtering policy. + * + * @param patterns The collection of patterns, eg "-foo/biz...". If the first character is "-", + * the pattern is treated as a negative pattern. + * @param policy The filtering policy, eg "only return test targets" + * @param offset The offset to apply to relative target patterns. + */ + @ThreadSafe + public static Iterable<SkyKey> keys(Collection<String> patterns, + FilteringPolicy policy, + String offset) { + List<SkyKey> keys = Lists.newArrayListWithCapacity(patterns.size()); + for (String pattern : patterns) { + keys.add(key(pattern, policy, offset)); + } + return keys; + } + + public ResolvedTargets<Target> getTargets() { + return targets; + } + + /** + * A TargetPattern is a tuple of pattern (eg, "foo/..."), filtering policy, a relative pattern + * offset, and whether it is a positive or negative match. + */ + @ThreadSafe + public static class TargetPattern implements Serializable { + private final String pattern; + private final FilteringPolicy policy; + private final boolean isNegative; + + private final String offset; + + public TargetPattern(String pattern, FilteringPolicy policy, + boolean isNegative, String offset) { + this.pattern = Preconditions.checkNotNull(pattern); + this.policy = Preconditions.checkNotNull(policy); + this.isNegative = isNegative; + this.offset = offset; + } + + public String getPattern() { + return pattern; + } + + public boolean isNegative() { + return isNegative; + } + + public FilteringPolicy getPolicy() { + return policy; + } + + public String getOffset() { + return offset; + } + + @Override + public String toString() { + return (isNegative ? "-" : "") + pattern; + } + + @Override + public int hashCode() { + return Objects.hash(pattern, isNegative, policy, offset); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof TargetPattern)) { + return false; + } + TargetPattern other = (TargetPattern) obj; + + return other.isNegative == this.isNegative && other.pattern.equals(this.pattern) && + other.offset.equals(this.offset) && other.policy.equals(this.policy); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionFunction.java new file mode 100644 index 0000000..b6f9606 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionFunction.java
@@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.LabelAndConfiguration; +import com.google.devtools.build.lib.rules.test.TestProvider; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** + * TargetCompletionFunction builds all relevant test artifacts of a {@link + * com.google.devtools.build.lib.analysis.ConfiguredTarget}. This includes test shards and repeated + * runs. + */ +public final class TestCompletionFunction implements SkyFunction { + + public TestCompletionFunction() { + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + TestCompletionValue.TestCompletionKey key = + (TestCompletionValue.TestCompletionKey) skyKey.argument(); + LabelAndConfiguration lac = key.getLabelAndConfiguration(); + if (env.getValue(TargetCompletionValue.key(lac)) == null) { + return null; + } + + ConfiguredTargetValue ctValue = (ConfiguredTargetValue) + env.getValue(ConfiguredTargetValue.key(lac.getLabel(), lac.getConfiguration())); + if (ctValue == null) { + return null; + } + + ConfiguredTarget ct = ctValue.getConfiguredTarget(); + if (key.isExclusiveTesting()) { + // Request test artifacts iteratively if testing exclusively. + for (Artifact testArtifact : TestProvider.getTestStatusArtifacts(ct)) { + if (env.getValue(ArtifactValue.key(testArtifact, /*isMandatory=*/true)) == null) { + return null; + } + } + } else { + env.getValues(ArtifactValue.mandatoryKeys(TestProvider.getTestStatusArtifacts(ct))); + if (env.valuesMissing()) { + return null; + } + } + return TestCompletionValue.TEST_COMPLETION_MARKER; + } + + @Override + public String extractTag(SkyKey skyKey) { + return Label.print(((LabelAndConfiguration) skyKey.argument()).getLabel()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionValue.java new file mode 100644 index 0000000..da944ed --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TestCompletionValue.java
@@ -0,0 +1,66 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.LabelAndConfiguration; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.util.Collection; + +/** + * A test completion value represents the completion of a test target. This includes the execution + * of all test shards and repeated runs, if applicable. + */ +public class TestCompletionValue implements SkyValue { + static final TestCompletionValue TEST_COMPLETION_MARKER = new TestCompletionValue(); + + private TestCompletionValue() { } + + public static SkyKey key(LabelAndConfiguration lac, boolean exclusive) { + return new SkyKey(SkyFunctions.TEST_COMPLETION, new TestCompletionKey(lac, exclusive)); + } + + public static Iterable<SkyKey> keys(Collection<ConfiguredTarget> targets, + final boolean exclusive) { + return Iterables.transform(targets, new Function<ConfiguredTarget, SkyKey>() { + @Override + public SkyKey apply(ConfiguredTarget ct) { + return new SkyKey(SkyFunctions.TEST_COMPLETION, + new TestCompletionKey(new LabelAndConfiguration(ct), exclusive)); + } + }); + } + + static class TestCompletionKey { + private final LabelAndConfiguration lac; + private final boolean exclusiveTesting; + + TestCompletionKey(LabelAndConfiguration lac, boolean exclusive) { + this.lac = lac; + this.exclusiveTesting = exclusive; + } + + public LabelAndConfiguration getLabelAndConfiguration() { + return lac; + } + + public boolean isExclusiveTesting() { + return exclusiveTesting; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetCycleReporter.java b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetCycleReporter.java new file mode 100644 index 0000000..03bbd25 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetCycleReporter.java
@@ -0,0 +1,86 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.packages.PackageGroup; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.CycleInfo; +import com.google.devtools.build.skyframe.SkyKey; + +import java.util.List; + +/** + * Reports cycles between {@link TransitiveTargetValue}s. These indicates cycles between targets + * (e.g. '//a:foo' depends on '//b:bar' and '//b:bar' depends on '//a:foo'). + */ +class TransitiveTargetCycleReporter extends AbstractLabelCycleReporter { + + private static final Predicate<SkyKey> IS_TRANSITIVE_TARGET_SKY_KEY = + SkyFunctions.isSkyFunction(SkyFunctions.TRANSITIVE_TARGET); + + TransitiveTargetCycleReporter(LoadedPackageProvider loadedPackageProvider) { + super(loadedPackageProvider); + } + + @Override + protected boolean canReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo) { + return Iterables.all(Iterables.concat(ImmutableList.of(topLevelKey), + cycleInfo.getPathToCycle(), cycleInfo.getCycle()), + IS_TRANSITIVE_TARGET_SKY_KEY); + } + + @Override + public String prettyPrint(SkyKey key) { + return getLabel(key).toString(); + } + + @Override + protected Label getLabel(SkyKey key) { + return (Label) key.argument(); + } + + @Override + protected String getAdditionalMessageAboutCycle(SkyKey topLevelKey, CycleInfo cycleInfo) { + Target currentTarget = getTargetForLabel(getLabel(topLevelKey)); + List<SkyKey> keys = Lists.newArrayList(); + if (!cycleInfo.getPathToCycle().isEmpty()) { + keys.add(topLevelKey); + keys.addAll(cycleInfo.getPathToCycle()); + } + keys.addAll(cycleInfo.getCycle()); + // Make sure we check the edge from the last element of the cycle to the first element of the + // cycle. + keys.add(cycleInfo.getCycle().get(0)); + for (SkyKey nextKey : keys) { + Label nextLabel = getLabel(nextKey); + Target nextTarget = getTargetForLabel(nextLabel); + // This is inefficient but it's no big deal since we only do this when there's a cycle. + if (currentTarget.getVisibility().getDependencyLabels().contains(nextLabel) + && !nextTarget.getTargetKind().equals(PackageGroup.targetKind())) { + return "\nThe cycle is caused by a visibility edge from " + currentTarget.getLabel() + + " to the non-package-group target " + nextTarget.getLabel() + " . Note that " + + "visibility labels are supposed to be package group targets (which prevents cycles " + + "of this form)"; + } + currentTarget = nextTarget; + } + return ""; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetFunction.java new file mode 100644 index 0000000..417cfca --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetFunction.java
@@ -0,0 +1,234 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.PackageGroup; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.skyframe.ValueOrException; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * This class builds transitive Target values such that evaluating a Target value is similar to + * running it through the LabelVisitor. + */ +public class TransitiveTargetFunction implements SkyFunction { + + @Override + public SkyValue compute(SkyKey key, Environment env) throws TransitiveTargetFunctionException { + Label label = (Label) key.argument(); + SkyKey packageKey = PackageValue.key(label.getPackageIdentifier()); + SkyKey targetKey = TargetMarkerValue.key(label); + Target target; + boolean packageLoadedSuccessfully; + boolean successfulTransitiveLoading = true; + NestedSetBuilder<Label> transitiveRootCauses = NestedSetBuilder.stableOrder(); + NoSuchTargetException errorLoadingTarget = null; + try { + TargetMarkerValue targetValue = (TargetMarkerValue) env.getValueOrThrow(targetKey, + NoSuchThingException.class); + if (targetValue == null) { + return null; + } + PackageValue packageValue = (PackageValue) env.getValueOrThrow(packageKey, + NoSuchThingException.class); + if (packageValue == null) { + return null; + } + + packageLoadedSuccessfully = true; + target = packageValue.getPackage().getTarget(label.getName()); + } catch (NoSuchTargetException e) { + target = e.getTarget(); + if (target == null) { + throw new TransitiveTargetFunctionException(e); + } + successfulTransitiveLoading = false; + transitiveRootCauses.add(label); + errorLoadingTarget = e; + packageLoadedSuccessfully = e.getPackageLoadedSuccessfully(); + } catch (NoSuchPackageException e) { + throw new TransitiveTargetFunctionException(e); + } catch (NoSuchThingException e) { + throw new IllegalStateException(e + + " not NoSuchTargetException or NoSuchPackageException"); + } + + NestedSetBuilder<PackageIdentifier> transitiveSuccessfulPkgs = NestedSetBuilder.stableOrder(); + NestedSetBuilder<PackageIdentifier> transitiveUnsuccessfulPkgs = NestedSetBuilder.stableOrder(); + NestedSetBuilder<Label> transitiveTargets = NestedSetBuilder.stableOrder(); + + PackageIdentifier packageId = target.getPackage().getPackageIdentifier(); + if (packageLoadedSuccessfully) { + transitiveSuccessfulPkgs.add(packageId); + } else { + transitiveUnsuccessfulPkgs.add(packageId); + } + transitiveTargets.add(target.getLabel()); + for (Map.Entry<SkyKey, ValueOrException<NoSuchThingException>> entry : + env.getValuesOrThrow(getLabelDepKeys(target), NoSuchThingException.class).entrySet()) { + Label depLabel = (Label) entry.getKey().argument(); + TransitiveTargetValue transitiveTargetValue; + try { + transitiveTargetValue = (TransitiveTargetValue) entry.getValue().get(); + if (transitiveTargetValue == null) { + continue; + } + } catch (NoSuchPackageException | NoSuchTargetException e) { + successfulTransitiveLoading = false; + transitiveRootCauses.add(depLabel); + maybeReportErrorAboutMissingEdge(target, depLabel, e, env.getListener()); + continue; + } catch (NoSuchThingException e) { + throw new IllegalStateException("Unexpected Exception type from TransitiveTargetValue.", e); + } + transitiveSuccessfulPkgs.addTransitive( + transitiveTargetValue.getTransitiveSuccessfulPackages()); + transitiveUnsuccessfulPkgs.addTransitive( + transitiveTargetValue.getTransitiveUnsuccessfulPackages()); + transitiveTargets.addTransitive(transitiveTargetValue.getTransitiveTargets()); + NestedSet<Label> rootCauses = transitiveTargetValue.getTransitiveRootCauses(); + if (rootCauses != null) { + successfulTransitiveLoading = false; + transitiveRootCauses.addTransitive(rootCauses); + if (transitiveTargetValue.getErrorLoadingTarget() != null) { + maybeReportErrorAboutMissingEdge(target, depLabel, + transitiveTargetValue.getErrorLoadingTarget(), env.getListener()); + } + } + } + + if (env.valuesMissing()) { + return null; + } + + NestedSet<PackageIdentifier> successfullyLoadedPackages = transitiveSuccessfulPkgs.build(); + NestedSet<PackageIdentifier> unsuccessfullyLoadedPackages = transitiveUnsuccessfulPkgs.build(); + NestedSet<Label> loadedTargets = transitiveTargets.build(); + if (successfulTransitiveLoading) { + return TransitiveTargetValue.successfulTransitiveLoading(successfullyLoadedPackages, + unsuccessfullyLoadedPackages, loadedTargets); + } else { + NestedSet<Label> rootCauses = transitiveRootCauses.build(); + return TransitiveTargetValue.unsuccessfulTransitiveLoading(successfullyLoadedPackages, + unsuccessfullyLoadedPackages, loadedTargets, rootCauses, errorLoadingTarget); + } + } + + @Override + public String extractTag(SkyKey skyKey) { + return Label.print(((Label) skyKey.argument())); + } + + private static void maybeReportErrorAboutMissingEdge(Target target, Label depLabel, + NoSuchThingException e, EventHandler eventHandler) { + if (e instanceof NoSuchTargetException) { + NoSuchTargetException nste = (NoSuchTargetException) e; + if (depLabel.equals(nste.getLabel())) { + eventHandler.handle(Event.error(TargetUtils.getLocationMaybe(target), + TargetUtils.formatMissingEdge(target, depLabel, e))); + } + } else if (e instanceof NoSuchPackageException) { + NoSuchPackageException nspe = (NoSuchPackageException) e; + if (nspe.getPackageName().equals(depLabel.getPackageName())) { + eventHandler.handle(Event.error(TargetUtils.getLocationMaybe(target), + TargetUtils.formatMissingEdge(target, depLabel, e))); + } + } + } + + private static Iterable<SkyKey> getLabelDepKeys(Target target) { + List<SkyKey> depKeys = Lists.newArrayList(); + for (Label depLabel : getLabelDeps(target)) { + depKeys.add(TransitiveTargetValue.key(depLabel)); + } + return depKeys; + } + + // TODO(bazel-team): Unify this logic with that in LabelVisitor, and possibly DependencyResolver. + private static Iterable<Label> getLabelDeps(Target target) { + final Set<Label> labels = new HashSet<>(); + if (target instanceof OutputFile) { + Rule rule = ((OutputFile) target).getGeneratingRule(); + labels.add(rule.getLabel()); + visitTargetVisibility(target, labels); + } else if (target instanceof InputFile) { + visitTargetVisibility(target, labels); + } else if (target instanceof Rule) { + visitTargetVisibility(target, labels); + labels.addAll(((Rule) target).getLabels(Rule.NO_NODEP_ATTRIBUTES)); + } else if (target instanceof PackageGroup) { + visitPackageGroup((PackageGroup) target, labels); + } + return labels; + } + + private static void visitTargetVisibility(Target target, Set<Label> labels) { + for (Label label : target.getVisibility().getDependencyLabels()) { + labels.add(label); + } + } + + private static void visitPackageGroup(PackageGroup packageGroup, Set<Label> labels) { + for (final Label include : packageGroup.getIncludes()) { + labels.add(include); + } + } + + /** + * Used to declare all the exception types that can be wrapped in the exception thrown by + * {@link TransitiveTargetFunction#compute}. + */ + private static class TransitiveTargetFunctionException extends SkyFunctionException { + /** + * Used to propagate an error from a direct target dependency to the + * target that depended on it. + */ + public TransitiveTargetFunctionException(NoSuchPackageException e) { + super(e, Transience.PERSISTENT); + } + + /** + * In nokeep_going mode, used to propagate an error from a direct target dependency to the + * target that depended on it. + * + * In keep_going mode, used the same way, but only for targets that could not be loaded at all + * (we proceed with transitive loading on targets that contain errors). + */ + public TransitiveTargetFunctionException(NoSuchTargetException e) { + super(e, Transience.PERSISTENT); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetValue.java new file mode 100644 index 0000000..69b9638 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveTargetValue.java
@@ -0,0 +1,142 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A <i>transitive</i> target reference that, when built in skyframe, loads the entire + * transitive closure of a target. + * + * This will probably be unnecessary once other refactorings occur throughout the codebase + * which make loading/analysis interleaving more feasible, or we will migrate "blaze query" to + * use this to evaluate its Target graph. + */ +@Immutable +@ThreadSafe +public class TransitiveTargetValue implements SkyValue { + + // Non-final for serialization purposes. + private NestedSet<PackageIdentifier> transitiveSuccessfulPkgs; + private NestedSet<PackageIdentifier> transitiveUnsuccessfulPkgs; + private NestedSet<Label> transitiveTargets; + @Nullable private NestedSet<Label> transitiveRootCauses; + @Nullable private NoSuchTargetException errorLoadingTarget; + + private TransitiveTargetValue(NestedSet<PackageIdentifier> transitiveSuccessfulPkgs, + NestedSet<PackageIdentifier> transitiveUnsuccessfulPkgs, NestedSet<Label> transitiveTargets, + @Nullable NestedSet<Label> transitiveRootCauses, + @Nullable NoSuchTargetException errorLoadingTarget) { + this.transitiveSuccessfulPkgs = transitiveSuccessfulPkgs; + this.transitiveUnsuccessfulPkgs = transitiveUnsuccessfulPkgs; + this.transitiveTargets = transitiveTargets; + this.transitiveRootCauses = transitiveRootCauses; + this.errorLoadingTarget = errorLoadingTarget; + } + + private void writeObject(ObjectOutputStream out) throws IOException { + // It helps to flatten the transitiveSuccessfulPkgs nested set as it has lots of duplicates. + Set<PackageIdentifier> successfulPkgs = transitiveSuccessfulPkgs.toSet(); + out.writeInt(successfulPkgs.size()); + for (PackageIdentifier pkg : successfulPkgs) { + out.writeObject(pkg); + } + + out.writeObject(transitiveUnsuccessfulPkgs); + // Deliberately do not write out transitiveTargets. There is a lot of those and they drive + // serialization costs through the roof, both in terms of space and of time. + // TODO(bazel-team): Deal with this properly once we have efficient serialization of NestedSets. + out.writeObject(transitiveRootCauses); + out.writeObject(errorLoadingTarget); + } + + @SuppressWarnings("unchecked") + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + int successfulPkgCount = in.readInt(); + NestedSetBuilder<PackageIdentifier> pkgs = NestedSetBuilder.stableOrder(); + for (int i = 0; i < successfulPkgCount; i++) { + pkgs.add((PackageIdentifier) in.readObject()); + } + transitiveSuccessfulPkgs = pkgs.build(); + transitiveUnsuccessfulPkgs = (NestedSet<PackageIdentifier>) in.readObject(); + // TODO(bazel-team): Deal with transitiveTargets properly. + transitiveTargets = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + transitiveRootCauses = (NestedSet<Label>) in.readObject(); + errorLoadingTarget = (NoSuchTargetException) in.readObject(); + } + + static TransitiveTargetValue unsuccessfulTransitiveLoading( + NestedSet<PackageIdentifier> transitiveSuccessfulPkgs, + NestedSet<PackageIdentifier> transitiveUnsuccessfulPkgs, NestedSet<Label> transitiveTargets, + NestedSet<Label> rootCauses, @Nullable NoSuchTargetException errorLoadingTarget) { + return new TransitiveTargetValue(transitiveSuccessfulPkgs, transitiveUnsuccessfulPkgs, + transitiveTargets, rootCauses, errorLoadingTarget); + } + + static TransitiveTargetValue successfulTransitiveLoading( + NestedSet<PackageIdentifier> transitiveSuccessfulPkgs, + NestedSet<PackageIdentifier> transitiveUnsuccessfulPkgs, + NestedSet<Label> transitiveTargets) { + return new TransitiveTargetValue(transitiveSuccessfulPkgs, transitiveUnsuccessfulPkgs, + transitiveTargets, null, null); + } + + /** Returns the error, if any, from loading the target. */ + @Nullable + public NoSuchTargetException getErrorLoadingTarget() { + return errorLoadingTarget; + } + + /** Returns the packages that were transitively successfully loaded. */ + public NestedSet<PackageIdentifier> getTransitiveSuccessfulPackages() { + return transitiveSuccessfulPkgs; + } + + /** Returns the packages that were transitively successfully loaded. */ + public NestedSet<PackageIdentifier> getTransitiveUnsuccessfulPackages() { + return transitiveUnsuccessfulPkgs; + } + + /** Returns the targets that were transitively loaded. */ + public NestedSet<Label> getTransitiveTargets() { + return transitiveTargets; + } + + /** Returns the root causes, if any, of why targets weren't loaded. */ + @Nullable + public NestedSet<Label> getTransitiveRootCauses() { + return transitiveRootCauses; + } + + @ThreadSafe + public static SkyKey key(Label label) { + return new SkyKey(SkyFunctions.TRANSITIVE_TARGET, label); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java new file mode 100644 index 0000000..f518b8a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java
@@ -0,0 +1,224 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import static com.google.devtools.build.lib.syntax.Environment.NONE; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.cmdline.LabelValidator; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.packages.ExternalPackage.Binding; +import com.google.devtools.build.lib.packages.ExternalPackage.ExternalPackageBuilder; +import com.google.devtools.build.lib.packages.ExternalPackage.ExternalPackageBuilder.NoSuchBindingException; +import com.google.devtools.build.lib.packages.Package.NameConflictException; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleFactory; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.packages.Type.ConversionException; +import com.google.devtools.build.lib.syntax.AbstractFunction; +import com.google.devtools.build.lib.syntax.BuildFileAST; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.FuncallExpression; +import com.google.devtools.build.lib.syntax.Function; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.syntax.MixedModeFunction; +import com.google.devtools.build.lib.syntax.ParserInputSource; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * A SkyFunction to parse WORKSPACE files. + */ +public class WorkspaceFileFunction implements SkyFunction { + + private static final String BIND = "bind"; + + private final PackageFactory packageFactory; + + WorkspaceFileFunction(PackageFactory packageFactory) { + this.packageFactory = packageFactory; + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws WorkspaceFileFunctionException, + InterruptedException { + RootedPath workspaceRoot = (RootedPath) skyKey.argument(); + // Explicitly make skyframe load this file. + if (env.getValue(FileValue.key(workspaceRoot)) == null) { + return null; + } + Path workspaceFilePath = workspaceRoot.getRoot().getRelative(workspaceRoot.getRelativePath()); + WorkspaceNameHolder holder = new WorkspaceNameHolder(); + ExternalPackageBuilder builder = new ExternalPackageBuilder(workspaceFilePath); + StoredEventHandler localReporter = new StoredEventHandler(); + BuildFileAST buildFileAST; + ParserInputSource inputSource = null; + + try { + inputSource = ParserInputSource.create(workspaceFilePath); + } catch (IOException e) { + throw new WorkspaceFileFunctionException(e, Transience.TRANSIENT); + } + buildFileAST = BuildFileAST.parseBuildFile(inputSource, localReporter, null, false); + if (buildFileAST.containsErrors()) { + localReporter.handle(Event.error("WORKSPACE file could not be parsed")); + } else { + try { + if (!evaluateWorkspaceFile(buildFileAST, holder, builder)) { + localReporter.handle( + Event.error("Error evaluating WORKSPACE file " + workspaceFilePath)); + } + } catch (EvalException e) { + throw new WorkspaceFileFunctionException(e); + } + } + + builder.addEvents(localReporter.getEvents()); + if (localReporter.hasErrors()) { + builder.setContainsErrors(); + } + return new WorkspaceFileValue(holder.workspaceName, builder.build()); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + + private static Function newWorkspaceNameFunction(final WorkspaceNameHolder holder) { + List<String> params = ImmutableList.of("name"); + return new MixedModeFunction("workspace", params, 1, true) { + @Override + public Object call(Object[] namedArgs, FuncallExpression ast) throws EvalException, + ConversionException, InterruptedException { + String name = Type.STRING.convert(namedArgs[0], "'name' argument"); + String errorMessage = LabelValidator.validateTargetName(name); + if (errorMessage != null) { + throw new EvalException(ast.getLocation(), errorMessage); + } + holder.workspaceName = name; + return NONE; + } + }; + } + + private static Function newBindFunction(final ExternalPackageBuilder builder) { + List<String> params = ImmutableList.of("name", "actual"); + return new MixedModeFunction(BIND, params, 2, true) { + @Override + public Object call(Object[] namedArgs, FuncallExpression ast) + throws EvalException, ConversionException { + String name = Type.STRING.convert(namedArgs[0], "'name' argument"); + String actual = Type.STRING.convert(namedArgs[1], "'actual' argument"); + + Label nameLabel = null; + try { + nameLabel = Label.parseAbsolute("//external:" + name); + builder.addBinding( + nameLabel, new Binding(Label.parseRepositoryLabel(actual), ast.getLocation())); + } catch (SyntaxException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + + return NONE; + } + }; + } + + /** + * Returns a function-value implementing the build rule "ruleClass" (e.g. cc_library) in the + * specified package context. + */ + private static Function newRuleFunction(final RuleFactory ruleFactory, + final ExternalPackageBuilder builder, final String ruleClassName) { + return new AbstractFunction(ruleClassName) { + @Override + public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast, + com.google.devtools.build.lib.syntax.Environment env) + throws EvalException { + if (!args.isEmpty()) { + throw new EvalException(ast.getLocation(), + "build rules do not accept positional parameters"); + } + + try { + RuleClass ruleClass = ruleFactory.getRuleClass(ruleClassName); + builder.createAndAddRepositoryRule(ruleClass, kwargs, ast); + } catch (RuleFactory.InvalidRuleException | NameConflictException | SyntaxException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + return NONE; + } + }; + } + + public boolean evaluateWorkspaceFile(BuildFileAST buildFileAST, WorkspaceNameHolder holder, + ExternalPackageBuilder builder) + throws InterruptedException, EvalException, WorkspaceFileFunctionException { + // Environment is defined in SkyFunction and the syntax package. + com.google.devtools.build.lib.syntax.Environment workspaceEnv = + new com.google.devtools.build.lib.syntax.Environment(); + + RuleFactory ruleFactory = new RuleFactory(packageFactory.getRuleClassProvider()); + for (String ruleClass : ruleFactory.getRuleClassNames()) { + Function ruleFunction = newRuleFunction(ruleFactory, builder, ruleClass); + workspaceEnv.update(ruleClass, ruleFunction); + } + + workspaceEnv.update(BIND, newBindFunction(builder)); + workspaceEnv.update("workspace", newWorkspaceNameFunction(holder)); + + StoredEventHandler eventHandler = new StoredEventHandler(); + if (!buildFileAST.exec(workspaceEnv, eventHandler)) { + return false; + } + try { + builder.resolveBindTargets(packageFactory.getRuleClass(BIND)); + } catch (NoSuchBindingException e) { + throw new WorkspaceFileFunctionException(e); + } + return true; + } + + private static final class WorkspaceNameHolder { + String workspaceName; + } + + private static final class WorkspaceFileFunctionException extends SkyFunctionException { + public WorkspaceFileFunctionException(IOException e, Transience transience) { + super(e, transience); + } + + public WorkspaceFileFunctionException(NoSuchBindingException e) { + super(e, Transience.PERSISTENT); + } + + public WorkspaceFileFunctionException(EvalException e) { + super(e, Transience.PERSISTENT); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileValue.java new file mode 100644 index 0000000..200f23f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileValue.java
@@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.packages.ExternalPackage; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import javax.annotation.Nullable; + +/** + * Holds the contents of a WORKSPACE file as the //external package. + */ +public class WorkspaceFileValue implements SkyValue { + + private final String workspace; + private final ExternalPackage pkg; + + public WorkspaceFileValue(String workspace, ExternalPackage pkg) { + this.workspace = workspace; + this.pkg = pkg; + } + + /** + * Returns the name of this workspace (or null for the default workspace). + */ + @Nullable + public String getWorkspace() { + return workspace; + } + + /** + * Returns the //external package. + */ + public ExternalPackage getPackage() { + return pkg; + } + + /** + * Generates a SkyKey based on the path to the WORKSPACE file. + */ + public static SkyKey key(RootedPath workspacePath) { + return new SkyKey(SkyFunctions.WORKSPACE_FILE, workspacePath); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusFunction.java new file mode 100644 index 0000000..b1c5ef3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusFunction.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** Creates the workspace status artifacts and action. */ +public class WorkspaceStatusFunction implements SkyFunction { + WorkspaceStatusFunction() { + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + Preconditions.checkState( + WorkspaceStatusValue.SKY_KEY.equals(skyKey), WorkspaceStatusValue.SKY_KEY); + + WorkspaceStatusAction action = PrecomputedValue.WORKSPACE_STATUS_KEY.get(env); + if (action == null) { + return null; + } + + return new WorkspaceStatusValue( + action.getStableStatus(), + action.getVolatileStatus(), + action); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusValue.java new file mode 100644 index 0000000..21b9215 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceStatusValue.java
@@ -0,0 +1,62 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; + +/** + * Value that stores the workspace status artifacts and their generating action. There should be + * only one of these values in the graph at any time. + */ +// TODO(bazel-team): This seems to be superfluous now, but it cannot be removed without making +// PrecomputedValue public instead of package-private +public class WorkspaceStatusValue extends ActionLookupValue { + private final Artifact stableArtifact; + private final Artifact volatileArtifact; + + // There should only ever be one BuildInfo value in the graph. + public static final SkyKey SKY_KEY = new SkyKey(SkyFunctions.BUILD_INFO, "BUILD_INFO"); + static final ArtifactOwner ARTIFACT_OWNER = new BuildInfoKey(); + + public WorkspaceStatusValue(Artifact stableArtifact, Artifact volatileArtifact, + WorkspaceStatusAction action) { + super(action); + this.stableArtifact = stableArtifact; + this.volatileArtifact = volatileArtifact; + } + + public Artifact getStableArtifact() { + return stableArtifact; + } + + public Artifact getVolatileArtifact() { + return volatileArtifact; + } + + private static class BuildInfoKey extends ActionLookupKey { + @Override + SkyFunctionName getType() { + throw new UnsupportedOperationException(); + } + + @Override + SkyKey getSkyKey() { + return SKY_KEY; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/standalone/LinuxSandboxedStrategy.java b/src/main/java/com/google/devtools/build/lib/standalone/LinuxSandboxedStrategy.java new file mode 100644 index 0000000..dc32ddc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/standalone/LinuxSandboxedStrategy.java
@@ -0,0 +1,227 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.standalone; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionInputHelper; +import com.google.devtools.build.lib.actions.Actions; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.actions.UserExecException; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.rules.cpp.CppCompileAction; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.unix.FilesystemUtils; +import com.google.devtools.build.lib.util.CommandFailureUtils; +import com.google.devtools.build.lib.util.DependencySet; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeSet; + +/** + * Strategy that uses sandboxing to execute a process. + */ +@ExecutionStrategy(name = {"sandboxed"}, + contextType = SpawnActionContext.class) +public class LinuxSandboxedStrategy implements SpawnActionContext { + private final boolean verboseFailures; + private final BlazeDirectories directories; + + public LinuxSandboxedStrategy(BlazeDirectories blazeDirectories, boolean verboseFailures) { + this.directories = blazeDirectories; + this.verboseFailures = verboseFailures; + } + + /** + * Executes the given {@code spawn}. + */ + @Override + public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext) + throws ExecException { + Executor executor = actionExecutionContext.getExecutor(); + if (executor.reportsSubcommands()) { + executor.reportSubcommand(Label.print(spawn.getOwner().getLabel()), + spawn.asShellCommand(executor.getExecRoot())); + } + boolean processHeaders = spawn.getResourceOwner() instanceof CppCompileAction; + + Path execPath = this.directories.getExecRoot(); + List<String> spawnArguments = new ArrayList<>(); + + for (String arg : spawn.getArguments()) { + if (arg.startsWith(execPath.getPathString())) { + // make all paths relative for the sandbox + spawnArguments.add(arg.substring(execPath.getPathString().length())); + } else { + spawnArguments.add(arg); + } + } + + List<? extends ActionInput> expandedInputs = + ActionInputHelper.expandMiddlemen(spawn.getInputFiles(), + actionExecutionContext.getMiddlemanExpander()); + + String cwd = executor.getExecRoot().getPathString(); + + FileOutErr outErr = actionExecutionContext.getFileOutErr(); + try { + PathFragment includePrefix = null; // null when there's no include mangling to do + List<PathFragment> includeDirectories = ImmutableList.of(); + if (processHeaders) { + CppCompileAction cppAction = (CppCompileAction) spawn.getResourceOwner(); + // headers are mounted in the sandbox in a separate include dir, so their names are mangled + // when running the compilation and will have to be unmangled after it's done in the *.pic.d + includeDirectories = extractIncludeDirs(execPath, cppAction, spawnArguments); + includePrefix = getSandboxIncludeDir(cppAction); + } + + NamespaceSandboxRunner runner = new NamespaceSandboxRunner(directories, spawn, includePrefix, + includeDirectories, spawn.getRunfilesManifests()); + runner.setupSandbox(expandedInputs, spawn.getOutputFiles()); + runner.run(spawnArguments, spawn.getEnvironment(), new File(cwd), outErr); + runner.copyOutputs(spawn.getOutputFiles(), outErr); + if (processHeaders) { + CppCompileAction cppAction = (CppCompileAction) spawn.getResourceOwner(); + unmangleHeaderFiles(cppAction); + } + runner.cleanup(); + } catch (CommandException e) { + String message = CommandFailureUtils.describeCommandFailure(verboseFailures, + spawn.getArguments(), spawn.getEnvironment(), cwd); + throw new UserExecException(String.format("%s: %s", message, e)); + } catch (IOException e) { + throw new UserExecException(e.getMessage()); + } + } + + private void unmangleHeaderFiles(CppCompileAction cppCompileAction) throws IOException { + Path execPath = this.directories.getExecRoot(); + CppCompileAction.DotdFile dotdfile = cppCompileAction.getDotdFile(); + DependencySet depset = new DependencySet(execPath).read(dotdfile.getPath()); + DependencySet unmangled = new DependencySet(execPath); + PathFragment sandboxIncludeDir = getSandboxIncludeDir(cppCompileAction); + PathFragment prefix = sandboxIncludeDir.getRelative(execPath.asFragment().relativeTo("/")); + for (PathFragment dep : depset.getDependencies()) { + if (dep.startsWith(prefix)) { + dep = dep.relativeTo(prefix); + } + unmangled.addDependency(dep); + } + unmangled.write(execPath.getRelative(depset.getOutputFileName()), ".d"); + } + + private PathFragment getSandboxIncludeDir(CppCompileAction cppCompileAction) { + return new PathFragment( + "include-" + Actions.escapedPath(cppCompileAction.getPrimaryOutput().toString())); + } + + private ImmutableList<PathFragment> extractIncludeDirs(Path execPath, + CppCompileAction cppCompileAction, List<String> spawnArguments) throws IOException { + List<PathFragment> includes = new ArrayList<>(); + includes.addAll(cppCompileAction.getQuoteIncludeDirs()); + includes.addAll(cppCompileAction.getIncludeDirs()); + includes.addAll(cppCompileAction.getSystemIncludeDirs()); + + // gcc implicitly includes headers in the same dir as .cc file + PathFragment sourceDirectory = + cppCompileAction.getSourceFile().getPath().getParentDirectory().asFragment(); + includes.add(sourceDirectory); + spawnArguments.add("-iquote"); + spawnArguments.add(sourceDirectory.toString()); + + TreeSet<PathFragment> processedIncludes = new TreeSet<>(); + for (int i = 0; i < includes.size(); i++) { + PathFragment absolutePath; + if (!includes.get(i).isAbsolute()) { + absolutePath = execPath.getRelative(includes.get(i)).asFragment(); + } else { + absolutePath = includes.get(i); + } + // CppCompileAction may provide execPath as one of the include directories. This is a big + // overestimation of what is actually needed and doesn't make for very hermetic sandbox + // (since everything from the workspace will be somehow accessed in the sandbox). To have + // some more hermeticity in this situation we mount all the include dirs in: + // sandbox-directory/include-prefix/actual-include-dir + // (where include-prefix is obtained from this.getSandboxIncludeDir(cppCompileAction)) + // and make so gcc looks there for includes. This should prevent the user from accessing + // files that technically should not be in the sandbox. + // TODO(bazel-team): change CppCompileAction so that include dirs contain only subsets of the + // execPath + if (absolutePath.equals(execPath.asFragment())) { + // we can't mount execPath because it will lead to a circular mount; instead mount its + // subdirs inside (other than the ones containing sandbox) + String[] subdirs = FilesystemUtils.readdir(absolutePath.toString()); + for (String dirName : subdirs) { + if (dirName.equals("_bin") || dirName.equals("bazel-out")) { + continue; + } + PathFragment child = absolutePath.getChild(dirName); + processedIncludes.add(child); + } + } else { + processedIncludes.add(absolutePath); + } + } + + // pseudo random name for include directory inside sandbox, so it won't be accessed by accident + String prefix = getSandboxIncludeDir(cppCompileAction).toString(); + + // change names in the invocation + for (int i = 0; i < spawnArguments.size(); i++) { + if (spawnArguments.get(i).startsWith("-I")) { + String argument = spawnArguments.get(i).substring(2); + spawnArguments.set(i, setIncludeDirSandboxPath(execPath, argument, "-I" + prefix)); + } + if (spawnArguments.get(i).equals("-iquote") || spawnArguments.get(i).equals("-isystem")) { + spawnArguments.set(i + 1, setIncludeDirSandboxPath(execPath, + spawnArguments.get(i + 1), prefix)); + } + } + return ImmutableList.copyOf(processedIncludes); + } + + private String setIncludeDirSandboxPath(Path execPath, String argument, String prefix) { + StringBuilder builder = new StringBuilder(prefix); + if (argument.charAt(0) != '/') { + // relative path + builder.append(execPath); + builder.append('/'); + } + builder.append(argument); + + return builder.toString(); + } + + @Override + public String strategyLocality(String mnemonic, boolean remotable) { + return "linux-sandboxing"; + } + + @Override + public boolean isRemotable(String mnemonic, boolean remotable) { + return false; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/standalone/LocalSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/standalone/LocalSpawnStrategy.java new file mode 100644 index 0000000..fc2387c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/standalone/LocalSpawnStrategy.java
@@ -0,0 +1,111 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.standalone; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.actions.UserExecException; +import com.google.devtools.build.lib.shell.Command; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.CommandFailureUtils; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.OsUtils; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +/** + * Strategy that uses subprocessing to execute a process. + */ +@ExecutionStrategy(name = { "standalone" }, contextType = SpawnActionContext.class) +public class LocalSpawnStrategy implements SpawnActionContext { + private final boolean verboseFailures; + + private final Path processWrapper; + + public LocalSpawnStrategy(Path execRoot, boolean verboseFailures) { + this.verboseFailures = verboseFailures; + this.processWrapper = execRoot.getRelative( + "_bin/process-wrapper" + OsUtils.executableExtension()); + } + + /** + * Executes the given {@code spawn}. + */ + @Override + public void exec(Spawn spawn, + ActionExecutionContext actionExecutionContext) + throws ExecException { + Executor executor = actionExecutionContext.getExecutor(); + if (executor.reportsSubcommands()) { + executor.reportSubcommand(Label.print(spawn.getOwner().getLabel()), + spawn.asShellCommand(executor.getExecRoot())); + } + + // We must wrap the subprocess with process-wrapper to kill the process tree. + // All actions therefore depend on the process-wrapper file. Since it's embedded, + // we don't bother with declaring it as an input. + List<String> args = new ArrayList<>(); + if (OS.getCurrent() != OS.WINDOWS) { + // TODO(bazel-team): process-wrapper seems to work on Windows, but requires + // additional setup as it is an msys2 binary, so it needs msys2 DLLs on %PATH%. + // Disable it for now to make the setup easier and to avoid further PATH hacks. + // Ideally we should have a native implementation of process-wrapper for Windows. + args.add(processWrapper.getPathString()); + args.add("-1"); /* timeout */ + args.add("0"); /* kill delay. */ + + // TODO(bazel-team): use process-wrapper redirection so we don't have to + // pass test logs through the Java heap. + args.add("-"); /* stdout. */ + args.add("-"); /* stderr. */ + } + args.addAll(spawn.getArguments()); + + String cwd = executor.getExecRoot().getPathString(); + Command cmd = new Command(args.toArray(new String[]{}), spawn.getEnvironment(), new File(cwd)); + + FileOutErr outErr = actionExecutionContext.getFileOutErr(); + try { + cmd.execute( + /* stdin */ new byte[]{}, + Command.NO_OBSERVER, + outErr.getOutputStream(), + outErr.getErrorStream(), + /*killSubprocessOnInterrupt*/ true); + } catch (CommandException e) { + String message = CommandFailureUtils.describeCommandFailure( + verboseFailures, spawn.getArguments(), spawn.getEnvironment(), cwd); + throw new UserExecException(String.format("%s: %s", message, e)); + } + } + + @Override + public String strategyLocality(String mnemonic, boolean remotable) { + return "standalone"; + } + + @Override + public boolean isRemotable(String mnemonic, boolean remotable) { + return false; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/standalone/NamespaceSandboxRunner.java b/src/main/java/com/google/devtools/build/lib/standalone/NamespaceSandboxRunner.java new file mode 100644 index 0000000..3c7a8e0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/standalone/NamespaceSandboxRunner.java
@@ -0,0 +1,267 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.standalone; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.shell.Command; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.unix.FilesystemUtils; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map.Entry; + +/** + * Helper class for running the namespace sandbox. This runner prepares environment inside the + * sandbox (copies inputs, creates file structure), handles sandbox output, performs cleanup and + * changes invocation if necessary. + */ +public class NamespaceSandboxRunner { + private final boolean debug = true; + private final PathFragment sandboxDirectory; + private final Path sandboxPath; + private final List<String> mounts; + private final Path embeddedBinaries; + private final Path tools; + private final ImmutableList<PathFragment> includeDirectories; + private final PathFragment includePrefix; + private final ImmutableMap<PathFragment, Artifact> manifests; + private final Path execRoot; + + public NamespaceSandboxRunner(BlazeDirectories directories, Spawn spawn, + PathFragment includePrefix, List<PathFragment> includeDirectories, + ImmutableMap<PathFragment, Artifact> manifests) { + String md5sum = Fingerprint.md5Digest(spawn.getResourceOwner().getPrimaryOutput().toString()); + this.sandboxDirectory = new PathFragment("sandbox-root-" + md5sum); + this.sandboxPath = + directories.getExecRoot().getRelative("sandboxes").getRelative(sandboxDirectory); + this.mounts = new ArrayList<>(); + this.tools = directories.getExecRoot().getChild("tools"); + this.embeddedBinaries = directories.getEmbeddedBinariesRoot(); + this.includePrefix = includePrefix; + this.includeDirectories = ImmutableList.copyOf(includeDirectories); + this.manifests = manifests; + this.execRoot = directories.getExecRoot(); + } + + private void createFileSystem(Collection<? extends ActionInput> outputs) throws IOException { + // create the sandboxes' parent directory if needed + // TODO(bazel-team): create this with rest of the workspace dirs + if (!sandboxPath.getParentDirectory().isDirectory()) { + FilesystemUtils.mkdir(sandboxPath.getParentDirectory().getPathString(), 0755); + } + + FilesystemUtils.mkdir(sandboxPath.getPathString(), 0755); + String[] dirs = { "bin", "etc" }; + for (String dir : dirs) { + FilesystemUtils.mkdir(sandboxPath.getChild(dir).getPathString(), 0755); + mounts.add("/" + dir); + } + + // usr + String[] dirsUsr = { "bin", "include" }; + FilesystemUtils.mkdir(sandboxPath.getChild("usr").getPathString(), 0755); + Path usr = sandboxPath.getChild("usr"); + for (String dir : dirsUsr) { + FilesystemUtils.mkdir(usr.getChild(dir).getPathString(), 0755); + mounts.add("/usr/" + dir); + } + FileSystemUtils.createDirectoryAndParents(usr.getChild("local").getChild("include")); + mounts.add("/usr/local/include"); + + // shared libs + String[] rootDirs = FilesystemUtils.readdir("/"); + for (String entry : rootDirs) { + if (entry.startsWith("lib")) { + FilesystemUtils.mkdir(sandboxPath.getChild(entry).getPathString(), 0755); + mounts.add("/" + entry); + } + } + + String[] usrDirs = FilesystemUtils.readdir("/usr/"); + for (String entry : usrDirs) { + if (entry.startsWith("lib")) { + String lib = usr.getChild(entry).getPathString(); + FilesystemUtils.mkdir(lib, 0755); + mounts.add("/usr/" + entry); + } + } + + if (this.includePrefix != null) { + FilesystemUtils.mkdir(sandboxPath.getRelative(includePrefix).getPathString(), 0755); + + for (PathFragment fullPath : includeDirectories) { + // includeDirectories should be absolute paths like /usr/include/foo.h. we want to combine + // them into something like sandbox/include-prefix/usr/include/foo.h - for that we remove + // the leading '/' from the path string and concatenate with sandbox/include/prefix + FileSystemUtils.createDirectoryAndParents(sandboxPath.getRelative(includePrefix) + .getRelative(fullPath.getPathString().substring(1))); + } + } + + // output directories + for (ActionInput output : outputs) { + PathFragment parentDirectory = + new PathFragment(output.getExecPathString()).getParentDirectory(); + FileSystemUtils.createDirectoryAndParents(sandboxPath.getRelative(parentDirectory)); + } + } + + public void setupSandbox(List<? extends ActionInput> inputs, + Collection<? extends ActionInput> outputs) throws IOException { + createFileSystem(outputs); + setupBlazeUtils(); + includeManifests(); + copyInputs(inputs); + } + + private void copyInputs(List<? extends ActionInput> inputs) throws IOException { + for (ActionInput input : inputs) { + if (input.getExecPathString().contains("internal/_middlemen/")) { + continue; + } + // entire tools will be mounted in the sandbox, so don't copy parts of it + if (input.getExecPathString().startsWith("tools/")) { + continue; + } + Path target = sandboxPath.getRelative(input.getExecPathString()); + Path source = execRoot.getRelative(input.getExecPathString()); + FileSystemUtils.createDirectoryAndParents(target.getParentDirectory()); + File targetFile = new File(target.getPathString()); + // TODO(bazel-team): mount inputs inside sandbox instead of copying + Files.copy(new File(source.getPathString()), targetFile); + FilesystemUtils.chmod(targetFile, 0755); + } + } + + private void includeManifests() throws IOException { + for (Entry<PathFragment, Artifact> manifest : this.manifests.entrySet()) { + String path = manifest.getValue().getPath().getPathString(); + for (String line : Files.readLines(new File(path), Charset.defaultCharset())) { + String[] fields = line.split(" "); + String targetPath = sandboxPath.getPathString() + PathFragment.SEPARATOR_CHAR + fields[0]; + String sourcePath = fields[1]; + File source = new File(sourcePath); + File target = new File(targetPath); + Files.createParentDirs(target); + Files.copy(source, target); + } + } + } + + private void setupBlazeUtils() throws IOException { + Path bin = this.sandboxPath.getChild("_bin"); + if (!bin.isDirectory()) { + FilesystemUtils.mkdir(bin.getPathString(), 0755); + } + Files.copy(new File(this.embeddedBinaries.getChild("build-runfiles").getPathString()), + new File(bin.getChild("build-runfiles").getPathString())); + FilesystemUtils.chmod(bin.getChild("build-runfiles").getPathString(), 0755); + // TODO(bazel-team) filter tools out of input files instead + // some of the tools could be in inputs; we will mount entire tools anyway so it's just + // easier to remove them and remount inside sandbox + FilesystemUtils.rmTree(sandboxPath.getChild("tools").getPathString()); + } + + + /** + * Runs given + * + * @param spawnArguments - arguments of spawn to run inside the sandbox + * @param env - environment to run sandbox in + * @param cwd - current working directory + * @param outErr - error output to capture sandbox's and command's stderr + * @throws CommandException + */ + public void run(List<String> spawnArguments, ImmutableMap<String, String> env, File cwd, + FileOutErr outErr) throws CommandException { + List<String> args = new ArrayList<>(); + args.add(execRoot.getRelative("_bin/namespace-sandbox").getPathString()); + + // Only for c++ compilation + if (includePrefix != null) { + for (PathFragment include : includeDirectories) { + args.add("-n"); + args.add(include.getPathString()); + } + + args.add("-N"); + args.add(includePrefix.getPathString()); + } + + if (debug) { + args.add("-D"); + } + args.add("-t"); + args.add(tools.getPathString()); + + args.add("-S"); + args.add(sandboxPath.getPathString()); + for (String mount : mounts) { + args.add("-m"); + args.add(mount); + } + + args.add("-C"); + args.addAll(spawnArguments); + Command cmd = new Command(args.toArray(new String[] {}), env, cwd); + + cmd.execute( + /* stdin */new byte[] {}, + Command.NO_OBSERVER, + outErr.getOutputStream(), + outErr.getErrorStream(), + /* killSubprocessOnInterrupt */true); + } + + + public void cleanup() throws IOException { + FilesystemUtils.rmTree(sandboxPath.getPathString()); + } + + + public void copyOutputs(Collection<? extends ActionInput> outputs, FileOutErr outErr) + throws IOException { + for (ActionInput output : outputs) { + Path source = this.sandboxPath.getRelative(output.getExecPathString()); + Path target = this.execRoot.getRelative(output.getExecPathString()); + FileSystemUtils.createDirectoryAndParents(target.getParentDirectory()); + // TODO(bazel-team): eliminate cases when there are excessive outputs in spawns + // (java compilation expects "srclist" file in its outputs which is sometimes not produced) + if (source.isFile()) { + Files.move(new File(source.getPathString()), new File(target.getPathString())); + } else { + outErr.getErrorStream().write(("Output wasn't created by action: " + output + "\n") + .getBytes(StandardCharsets.UTF_8)); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextConsumer.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextConsumer.java new file mode 100644 index 0000000..2011327 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextConsumer.java
@@ -0,0 +1,57 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.standalone; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import com.google.devtools.build.lib.actions.ActionContextConsumer; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.analysis.actions.FileWriteActionContext; +import com.google.devtools.build.lib.rules.cpp.CppCompileActionContext; +import com.google.devtools.build.lib.rules.cpp.IncludeScanningContext; +import com.google.devtools.build.lib.rules.cpp.LinkStrategy; +import com.google.devtools.build.lib.rules.test.TestStrategy; + +import java.util.Map; + +/** + * {@link ActionContextConsumer} that requests the action contexts necessary for standalone + * execution. + */ +public class StandaloneContextConsumer implements ActionContextConsumer { + + @Override + public Map<String, String> getSpawnActionContexts() { + return ImmutableMap.of(); + } + + @Override + public Map<Class<? extends ActionContext>, String> getActionContexts() { + Builder<Class<? extends ActionContext>, String> actionContexts = + new ImmutableMap.Builder<Class<? extends ActionContext>, String>(); + + actionContexts.put(SpawnActionContext.class, "standalone"); + + // C++. + actionContexts.put(LinkStrategy.class, ""); + actionContexts.put(IncludeScanningContext.class, ""); + actionContexts.put(CppCompileActionContext.class, ""); + actionContexts.put(TestStrategy.class, ""); + actionContexts.put(FileWriteActionContext.class, ""); + + return actionContexts.build(); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextProvider.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextProvider.java new file mode 100644 index 0000000..dbfac2e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextProvider.java
@@ -0,0 +1,126 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.standalone; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.devtools.build.lib.actions.ActionContextProvider; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.ActionMetadata; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ExecutorInitException; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.exec.FileWriteStrategy; +import com.google.devtools.build.lib.rules.cpp.IncludeScanningContext; +import com.google.devtools.build.lib.rules.cpp.LocalGccStrategy; +import com.google.devtools.build.lib.rules.cpp.LocalLinkStrategy; +import com.google.devtools.build.lib.rules.test.ExclusiveTestStrategy; +import com.google.devtools.build.lib.rules.test.StandaloneTestStrategy; +import com.google.devtools.build.lib.rules.test.TestActionContext; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.FileSystemUtils; + +import java.io.IOException; + +/** + * Provide a standalone, local execution context. + */ +public class StandaloneContextProvider implements ActionContextProvider { + + /** + * a IncludeScanningContext that does nothing. Since local execution does not need to + * discover inclusion in advance, we do not need include scanning. + */ + @ExecutionStrategy(contextType = IncludeScanningContext.class) + class DummyIncludeScanningContext implements IncludeScanningContext { + @Override + public void extractIncludes(ActionExecutionContext actionExecutionContext, + ActionMetadata resourceOwner, Artifact primaryInput, Artifact primaryOutput) + throws IOException, InterruptedException { + FileSystemUtils.writeContent(primaryOutput.getPath(), new byte[]{}); + } + + @Override + public ArtifactResolver getArtifactResolver() { + return runtime.getView().getArtifactFactory(); + } + } + + @SuppressWarnings("unchecked") + private final ActionContext localSpawnStrategy; + private final ImmutableList<ActionContext> strategies; + private final BlazeRuntime runtime; + + public StandaloneContextProvider( + BlazeRuntime runtime, BuildRequest buildRequest) { + boolean verboseFailures = buildRequest.getOptions(ExecutionOptions.class).verboseFailures; + + localSpawnStrategy = new LocalSpawnStrategy( + runtime.getDirectories().getExecRoot(), verboseFailures); + this.runtime = runtime; + + TestActionContext testStrategy = new StandaloneTestStrategy(buildRequest, + runtime.getStartupOptionsProvider(), runtime.getBinTools(), runtime.getRunfilesPrefix()); + Builder<ActionContext> strategiesBuilder = ImmutableList.builder(); + // order of strategies passed to builder is significant - when there are many strategies that + // could potentially be used and a spawnActionContext doesn't specify which one it wants, the + // last one from strategies list will be used + + // put sandboxed strategy first, as we don't want it by default + if (OS.getCurrent() == OS.LINUX) { + LinuxSandboxedStrategy sandboxedLinuxStrategy = + new LinuxSandboxedStrategy(runtime.getDirectories(), verboseFailures); + strategiesBuilder.add(sandboxedLinuxStrategy); + } + strategiesBuilder.add( + localSpawnStrategy, + new DummyIncludeScanningContext(), + new LocalLinkStrategy(), + testStrategy, + new ExclusiveTestStrategy(testStrategy), + new LocalGccStrategy(buildRequest), + new FileWriteStrategy()); + + + this.strategies = strategiesBuilder.build(); + } + + @Override + public Iterable<ActionContext> getActionContexts() { + return strategies; + } + + @Override + public void executorCreated(Iterable<ActionContext> usedContexts) throws ExecutorInitException { + } + + @Override + public void executionPhaseStarting( + ActionInputFileCache actionInputFileCache, + ActionGraph actionGraph, + Iterable<Artifact> topLevelArtifacts) throws ExecutorInitException { + } + + @Override + public void executionPhaseEnding() {} +} + +
diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java new file mode 100644 index 0000000..06bffd0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java
@@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.standalone; + +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.actions.ActionContextConsumer; +import com.google.devtools.build.lib.actions.ActionContextProvider; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; + +/** + * StandaloneModule provides pluggable functionality for blaze. + */ +public class StandaloneModule extends BlazeModule { + private final ActionContextConsumer actionContextConsumer = new StandaloneContextConsumer(); + private BuildRequest buildRequest; + private BlazeRuntime runtime; + + /** + * Returns the action context provider the module contributes to Blaze, if any. + */ + @Override + public ActionContextProvider getActionContextProvider() { + return new StandaloneContextProvider(runtime, buildRequest); + } + + /** + * Returns the action context consumer the module contributes to Blaze, if any. + */ + @Override + public ActionContextConsumer getActionContextConsumer() { + return actionContextConsumer; + } + + @Override + public void beforeCommand(BlazeRuntime runtime, Command command) { + this.runtime = runtime; + runtime.getEventBus().register(this); + } + + @Subscribe + public void buildStarting(BuildStartingEvent event) { + buildRequest = event.getRequest(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ASTNode.java b/src/main/java/com/google/devtools/build/lib/syntax/ASTNode.java new file mode 100644 index 0000000..81ca584 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/ASTNode.java
@@ -0,0 +1,65 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.lib.events.Location; + +import java.io.Serializable; + +/** + * Root class for nodes in the Abstract Syntax Tree of the Build language. + */ +public abstract class ASTNode implements Serializable { + + private Location location; + + protected ASTNode() {} + + @VisibleForTesting // productionVisibility = Visibility.PACKAGE_PRIVATE + public void setLocation(Location location) { + this.location = location; + } + + public Location getLocation() { + return location; + } + + /** + * Print the syntax node in a form useful for debugging. The output is not + * precisely specified, and should not be used by pretty-printing routines. + */ + @Override + public abstract String toString(); + + @Override + public int hashCode() { + throw new UnsupportedOperationException(); // avoid nondeterminism + } + + @Override + public boolean equals(Object that) { + throw new UnsupportedOperationException(); + } + + /** + * Implements the double dispatch by calling into the node specific + * <code>visit</code> method of the {@link SyntaxTreeVisitor} + * + * @param visitor the {@link SyntaxTreeVisitor} instance to dispatch to. + */ + public abstract void accept(SyntaxTreeVisitor visitor); + +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/AbstractFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/AbstractFunction.java new file mode 100644 index 0000000..f444c23 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/AbstractFunction.java
@@ -0,0 +1,64 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import java.util.List; +import java.util.Map; + +/** + * Partial implementation of Function interface. + */ +public abstract class AbstractFunction implements Function { + + private final String name; + + protected AbstractFunction(String name) { + this.name = name; + } + + /** + * Returns the name of this function. + */ + @Override + public String getName() { + return name; + } + + @Override + public Class<?> getObjectType() { + return null; + } + + /** + * Abstract implementation of Function that accepts no parameters. + */ + public abstract static class NoArgFunction extends AbstractFunction { + + public NoArgFunction(String name) { + super(name); + } + + @Override + public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast, + Environment env) throws EvalException, InterruptedException { + if (args.size() != 1 || kwargs.size() != 0) { + throw new EvalException(ast.getLocation(), "Invalid number of arguments (expected 0)"); + } + return call(args.get(0), ast, env); + } + + public abstract Object call(Object self, FuncallExpression ast, Environment env) + throws EvalException, InterruptedException; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Argument.java b/src/main/java/com/google/devtools/build/lib/syntax/Argument.java new file mode 100644 index 0000000..0706dee --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/Argument.java
@@ -0,0 +1,122 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +/** + * Syntax node for a function argument. This can be a key/value pair such as + * appears as a keyword argument to a function call or just an expression that + * is used as a positional argument. It also can be used for function definitions + * to identify the name (and optionally the default value) of the argument. + */ +public final class Argument extends ASTNode { + + private final Ident name; + + private final Expression value; + + private final boolean kwargs; + + /** + * Create a new argument. + * At call site: name is optional, value is mandatory. kwargs is true for ** arguments. + * At definition site: name is mandatory, (default) value is optional. + */ + public Argument(Ident name, Expression value, boolean kwargs) { + this.name = name; + this.value = value; + this.kwargs = kwargs; + } + + public Argument(Ident name, Expression value) { + this.name = name; + this.value = value; + this.kwargs = false; + } + + /** + * Creates an Argument with null as name. It can be used as positional arguments + * of function calls. + */ + public Argument(Expression value) { + this(null, value); + } + + /** + * Creates an Argument with null as value. It can be used as a mandatory keyword argument + * of a function definition. + */ + public Argument(Ident name) { + this(name, null); + } + + /** + * Returns the name of this keyword argument or null if this argument is + * positional. + */ + public Ident getName() { + return name; + } + + /** + * Returns the String value of the Ident of this argument. Shortcut for arg.getName().getName(). + */ + public String getArgName() { + return name.getName(); + } + + /** + * Returns the syntax of this argument expression. + */ + public Expression getValue() { + return value; + } + + /** + * Returns true if this argument is positional. + */ + public boolean isPositional() { + return name == null && !kwargs; + } + + /** + * Returns true if this argument is a keyword argument. + */ + public boolean isNamed() { + return name != null; + } + + /** + * Returns true if this argument is a **kwargs argument. + */ + public boolean isKwargs() { + return kwargs; + } + + /** + * Returns true if this argument has value. + */ + public boolean hasValue() { + return value != null; + } + + @Override + public String toString() { + return isNamed() ? name + "=" + value : String.valueOf(value); + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/AssignmentStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/AssignmentStatement.java new file mode 100644 index 0000000..619e841 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/AssignmentStatement.java
@@ -0,0 +1,108 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Preconditions; + +/** + * Syntax node for an assignment statement. + */ +public final class AssignmentStatement extends Statement { + + private final Expression lvalue; + + private final Expression expression; + + /** + * Constructs an assignment: "lvalue := value". + */ + AssignmentStatement(Expression lvalue, Expression expression) { + this.lvalue = lvalue; + this.expression = expression; + } + + /** + * Returns the LHS of the assignment. + */ + public Expression getLValue() { + return lvalue; + } + + /** + * Returns the RHS of the assignment. + */ + public Expression getExpression() { + return expression; + } + + @Override + public String toString() { + return lvalue + " = " + expression + '\n'; + } + + @Override + void exec(Environment env) throws EvalException, InterruptedException { + if (!(lvalue instanceof Ident)) { + throw new EvalException(getLocation(), + "can only assign to variables, not to '" + lvalue + "'"); + } + + Ident ident = (Ident) lvalue; + Object result = expression.eval(env); + Preconditions.checkNotNull(result, "result of " + expression + " is null"); + + if (env.isSkylarkEnabled()) { + // The variable may have been referenced successfully if a global variable + // with the same name exists. In this case an Exception needs to be thrown. + SkylarkEnvironment skylarkEnv = (SkylarkEnvironment) env; + if (skylarkEnv.hasBeenReadGlobalVariable(ident.getName())) { + throw new EvalException(getLocation(), "Variable '" + ident.getName() + + "' is referenced before assignment." + + "The variable is defined in the global scope."); + } + Class<?> variableType = skylarkEnv.getVariableType(ident.getName()); + Class<?> resultType = EvalUtils.getSkylarkType(result.getClass()); + if (variableType != null && !variableType.equals(resultType) + && !resultType.equals(Environment.NoneType.class) + && !variableType.equals(Environment.NoneType.class)) { + throw new EvalException(getLocation(), String.format("Incompatible variable types, " + + "trying to assign %s (type of %s) to variable %s which is already %s", + EvalUtils.prettyPrintValue(result), + EvalUtils.getDatatypeName(result), + ident.getName(), + EvalUtils.getDataTypeNameFromClass(variableType))); + } + } + env.update(ident.getName(), result); + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + void validate(ValidationEnvironment env) throws EvalException { + // TODO(bazel-team): Implement other validations. + if (lvalue instanceof Ident) { + Ident ident = (Ident) lvalue; + SkylarkType resultType = expression.validate(env); + env.update(ident.getName(), resultType, getLocation()); + } else { + throw new EvalException(getLocation(), + "can only assign to variables, not to '" + lvalue + "'"); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java new file mode 100644 index 0000000..f53538f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java
@@ -0,0 +1,412 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject; + +import java.util.Collection; +import java.util.Collections; +import java.util.IllegalFormatException; +import java.util.List; +import java.util.Map; + +/** + * Syntax node for a binary operator expression. + */ +public final class BinaryOperatorExpression extends Expression { + + private final Expression lhs; + + private final Expression rhs; + + private final Operator operator; + + public BinaryOperatorExpression(Operator operator, + Expression lhs, + Expression rhs) { + this.lhs = lhs; + this.rhs = rhs; + this.operator = operator; + } + + public Expression getLhs() { + return lhs; + } + + public Expression getRhs() { + return rhs; + } + + /** + * Returns the operator kind for this binary operation. + */ + public Operator getOperator() { + return operator; + } + + @Override + public String toString() { + return lhs + " " + operator + " " + rhs; + } + + private int compare(Object lval, Object rval) throws EvalException { + if (!(lval instanceof Comparable)) { + throw new EvalException(getLocation(), lval + " is not comparable"); + } + try { + return ((Comparable) lval).compareTo(rval); + } catch (ClassCastException e) { + throw new EvalException(getLocation(), "Cannot compare " + EvalUtils.getDatatypeName(lval) + + " with " + EvalUtils.getDatatypeName(rval)); + } + } + + @Override + Object eval(Environment env) throws EvalException, InterruptedException { + Object lval = lhs.eval(env); + + // Short-circuit operators + if (operator == Operator.AND) { + if (EvalUtils.toBoolean(lval)) { + return rhs.eval(env); + } else { + return lval; + } + } + + if (operator == Operator.OR) { + if (EvalUtils.toBoolean(lval)) { + return lval; + } else { + return rhs.eval(env); + } + } + + Object rval = rhs.eval(env); + + switch (operator) { + case PLUS: { + // int + int + if (lval instanceof Integer && rval instanceof Integer) { + return ((Integer) lval).intValue() + ((Integer) rval).intValue(); + } + + // string + string + if (lval instanceof String && rval instanceof String) { + return (String) lval + (String) rval; + } + + // list + list, tuple + tuple (list + tuple, tuple + list => error) + if (lval instanceof List<?> && rval instanceof List<?>) { + List<?> llist = (List<?>) lval; + List<?> rlist = (List<?>) rval; + if (EvalUtils.isImmutable(llist) != EvalUtils.isImmutable(rlist)) { + throw new EvalException(getLocation(), "can only concatenate " + + EvalUtils.getDatatypeName(rlist) + " (not \"" + + EvalUtils.getDatatypeName(llist) + "\") to " + + EvalUtils.getDatatypeName(rlist)); + } + if (llist instanceof GlobList<?> || rlist instanceof GlobList<?>) { + return GlobList.concat(llist, rlist); + } else { + List<Object> result = Lists.newArrayListWithCapacity(llist.size() + rlist.size()); + result.addAll(llist); + result.addAll(rlist); + return EvalUtils.makeSequence(result, EvalUtils.isImmutable(llist)); + } + } + + if (lval instanceof SkylarkList && rval instanceof SkylarkList) { + return SkylarkList.concat((SkylarkList) lval, (SkylarkList) rval, getLocation()); + } + + if (env.isSkylarkEnabled() && lval instanceof Map<?, ?> && rval instanceof Map<?, ?>) { + Map<?, ?> ldict = (Map<?, ?>) lval; + Map<?, ?> rdict = (Map<?, ?>) rval; + Map<Object, Object> result = Maps.newHashMapWithExpectedSize(ldict.size() + rdict.size()); + result.putAll(ldict); + result.putAll(rdict); + return result; + } + + if (env.isSkylarkEnabled() + && lval instanceof SkylarkClassObject && rval instanceof SkylarkClassObject) { + return SkylarkClassObject.concat( + (SkylarkClassObject) lval, (SkylarkClassObject) rval, getLocation()); + } + + if (env.isSkylarkEnabled() && lval instanceof SkylarkNestedSet) { + return new SkylarkNestedSet((SkylarkNestedSet) lval, rval, getLocation()); + } + break; + } + + case MINUS: { + if (lval instanceof Integer && rval instanceof Integer) { + return ((Integer) lval).intValue() - ((Integer) rval).intValue(); + } + break; + } + + case MULT: { + // int * int + if (lval instanceof Integer && rval instanceof Integer) { + return ((Integer) lval).intValue() * ((Integer) rval).intValue(); + } + + // string * int + if (lval instanceof String && rval instanceof Integer) { + return Strings.repeat((String) lval, ((Integer) rval).intValue()); + } + + // int * string + if (lval instanceof Integer && rval instanceof String) { + return Strings.repeat((String) rval, ((Integer) lval).intValue()); + } + break; + } + + case PERCENT: { + // int % int + if (lval instanceof Integer && rval instanceof Integer) { + return ((Integer) lval).intValue() % ((Integer) rval).intValue(); + } + + // string % tuple, string % dict, string % anything-else + if (lval instanceof String) { + try { + String pattern = (String) lval; + if (rval instanceof List<?>) { + List<?> rlist = (List<?>) rval; + if (EvalUtils.isTuple(rlist)) { + return EvalUtils.formatString(pattern, rlist); + } + /* string % list: fall thru */ + } + if (rval instanceof SkylarkList) { + SkylarkList rlist = (SkylarkList) rval; + if (rlist.isTuple()) { + return EvalUtils.formatString(pattern, rlist.toList()); + } + } + + return EvalUtils.formatString(pattern, + Collections.singletonList(rval)); + } catch (IllegalFormatException e) { + throw new EvalException(getLocation(), e.getMessage()); + } + } + break; + } + + case EQUALS_EQUALS: { + return lval.equals(rval); + } + + case NOT_EQUALS: { + return !lval.equals(rval); + } + + case LESS: { + return compare(lval, rval) < 0; + } + + case LESS_EQUALS: { + return compare(lval, rval) <= 0; + } + + case GREATER: { + return compare(lval, rval) > 0; + } + + case GREATER_EQUALS: { + return compare(lval, rval) >= 0; + } + + case IN: { + if (rval instanceof SkylarkList) { + for (Object obj : (SkylarkList) rval) { + if (obj.equals(lval)) { + return true; + } + } + return false; + } else if (rval instanceof Collection<?>) { + return ((Collection<?>) rval).contains(lval); + } else if (rval instanceof Map<?, ?>) { + return ((Map<?, ?>) rval).containsKey(lval); + } else if (rval instanceof String) { + if (lval instanceof String) { + return ((String) rval).contains((String) lval); + } else { + throw new EvalException(getLocation(), + "in operator only works on strings if the left operand is also a string"); + } + } else { + throw new EvalException(getLocation(), + "in operator only works on lists, tuples, dictionaries and strings"); + } + } + + default: { + throw new AssertionError("Unsupported binary operator: " + operator); + } + } // endswitch + + throw new EvalException(getLocation(), + "unsupported operand types for '" + operator + "': '" + + EvalUtils.getDatatypeName(lval) + "' and '" + + EvalUtils.getDatatypeName(rval) + "'"); + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + SkylarkType validate(ValidationEnvironment env) throws EvalException { + SkylarkType ltype = lhs.validate(env); + SkylarkType rtype = rhs.validate(env); + String lname = EvalUtils.getDataTypeNameFromClass(ltype.getType()); + String rname = EvalUtils.getDataTypeNameFromClass(rtype.getType()); + + switch (operator) { + case AND: { + return ltype.infer(rtype, "and operator", rhs.getLocation(), lhs.getLocation()); + } + + case OR: { + return ltype.infer(rtype, "or operator", rhs.getLocation(), lhs.getLocation()); + } + + case PLUS: { + // int + int + if (ltype == SkylarkType.INT && rtype == SkylarkType.INT) { + return SkylarkType.INT; + } + + // string + string + if (ltype == SkylarkType.STRING && rtype == SkylarkType.STRING) { + return SkylarkType.STRING; + } + + // list + list + if (ltype.isList() && rtype.isList()) { + return ltype.infer(rtype, "list concatenation", rhs.getLocation(), lhs.getLocation()); + } + + // dict + dict + if (ltype.isDict() && rtype.isDict()) { + return ltype.infer(rtype, "dict concatenation", rhs.getLocation(), lhs.getLocation()); + } + + // struct + struct + if (ltype.isStruct() && rtype.isStruct()) { + return SkylarkType.of(ClassObject.class); + } + + if (ltype.isNset()) { + if (rtype.isNset()) { + return ltype.infer(rtype, "nested set", rhs.getLocation(), lhs.getLocation()); + } else if (rtype.isList()) { + return ltype.infer(SkylarkType.of(SkylarkNestedSet.class, rtype.getGenericType1()), + "nested set", rhs.getLocation(), lhs.getLocation()); + } + if (rtype != SkylarkType.UNKNOWN) { + throw new EvalException(getLocation(), String.format("can only concatenate nested sets " + + "with other nested sets or list of items, not '" + rname + "'")); + } + } + + break; + } + + case MULT: { + // int * int + if (ltype == SkylarkType.INT && rtype == SkylarkType.INT) { + return SkylarkType.INT; + } + + // string * int + if (ltype == SkylarkType.STRING && rtype == SkylarkType.INT) { + return SkylarkType.STRING; + } + + // int * string + if (ltype == SkylarkType.INT && rtype == SkylarkType.STRING) { + return SkylarkType.STRING; + } + break; + } + + case MINUS: { + if (ltype == SkylarkType.INT && rtype == SkylarkType.INT) { + return SkylarkType.INT; + } + break; + } + + case PERCENT: { + // int % int + if (ltype == SkylarkType.INT && rtype == SkylarkType.INT) { + return SkylarkType.INT; + } + + // string % tuple, string % dict, string % anything-else + if (ltype == SkylarkType.STRING) { + return SkylarkType.STRING; + } + break; + } + + case EQUALS_EQUALS: + case NOT_EQUALS: + case LESS: + case LESS_EQUALS: + case GREATER: + case GREATER_EQUALS: { + if (ltype != SkylarkType.UNKNOWN && !(Comparable.class.isAssignableFrom(ltype.getType()))) { + throw new EvalException(getLocation(), lname + " is not comparable"); + } + ltype.infer(rtype, "comparison", lhs.getLocation(), rhs.getLocation()); + return SkylarkType.BOOL; + } + + case IN: { + if (rtype.isList() + || rtype.isSet() + || rtype.isDict() + || rtype == SkylarkType.STRING) { + return SkylarkType.BOOL; + } else { + if (rtype != SkylarkType.UNKNOWN) { + throw new EvalException(getLocation(), String.format("operand 'in' only works on " + + "strings, dictionaries, lists, sets or tuples, not on a(n) %s", + EvalUtils.getDataTypeNameFromClass(rtype.getType()))); + } + } + } + } // endswitch + + if (ltype != SkylarkType.UNKNOWN && rtype != SkylarkType.UNKNOWN) { + throw new EvalException(getLocation(), + "unsupported operand types for '" + operator + "': '" + lname + "' and '" + rname + "'"); + } + return SkylarkType.UNKNOWN; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java b/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java new file mode 100644 index 0000000..6c85ab1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java
@@ -0,0 +1,244 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.CachingPackageLocator; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Abstract syntax node for an entire BUILD file. + */ +public class BuildFileAST extends ASTNode { + + private final ImmutableList<Statement> stmts; + + private final ImmutableList<Comment> comments; + + private final ImmutableSet<PathFragment> imports; + + /** + * Whether any errors were encountered during scanning or parsing. + */ + private final boolean containsErrors; + + private final String contentHashCode; + + private BuildFileAST(Lexer lexer, List<Statement> preludeStatements, Parser.ParseResult result) { + this(lexer, preludeStatements, result, null); + } + + private BuildFileAST(Lexer lexer, List<Statement> preludeStatements, + Parser.ParseResult result, String contentHashCode) { + this.stmts = ImmutableList.<Statement>builder() + .addAll(preludeStatements) + .addAll(result.statements) + .build(); + this.comments = ImmutableList.copyOf(result.comments); + this.containsErrors = result.containsErrors; + this.contentHashCode = contentHashCode; + this.imports = fetchImports(this.stmts); + if (result.statements.size() > 0) { + setLocation(lexer.createLocation( + result.statements.get(0).getLocation().getStartOffset(), + result.statements.get(result.statements.size() - 1).getLocation().getEndOffset())); + } else { + setLocation(Location.fromFile(lexer.getFilename())); + } + } + + private ImmutableSet<PathFragment> fetchImports(List<Statement> stmts) { + Set<PathFragment> imports = new HashSet<>(); + for (Statement stmt : stmts) { + if (stmt instanceof LoadStatement) { + LoadStatement imp = (LoadStatement) stmt; + imports.add(imp.getImportPath()); + } + } + return ImmutableSet.copyOf(imports); + } + + /** + * Returns true if any errors were encountered during scanning or parsing. If + * set, clients should not rely on the correctness of the AST for builds or + * BUILD-file editing. + */ + public boolean containsErrors() { + return containsErrors; + } + + /** + * Returns an (immutable, ordered) list of statements in this BUILD file. + */ + public ImmutableList<Statement> getStatements() { + return stmts; + } + + /** + * Returns an (immutable, ordered) list of comments in this BUILD file. + */ + public ImmutableList<Comment> getComments() { + return comments; + } + + /** + * Returns an (immutable) set of imports in this BUILD file. + */ + public ImmutableCollection<PathFragment> getImports() { + return imports; + } + + /** + * Executes this build file in a given Environment. + * + * <p>If, for any reason, execution of a statement cannot be completed, an + * {@link EvalException} is thrown by {@link Statement#exec(Environment)}. + * This exception is caught here and reported through reporter and execution + * continues on the next statement. In effect, there is a "try/except" block + * around every top level statement. Such exceptions are not ignored, though: + * they are visible via the return value. Rules declared in a package + * containing any error (including loading-phase semantical errors that + * cannot be checked here) must also be considered "in error". + * + * <p>Note that this method will not affect the value of {@link + * #containsErrors()}; that refers only to lexer/parser errors. + * + * @return true if no error occurred during execution. + */ + public boolean exec(Environment env, EventHandler eventHandler) throws InterruptedException { + boolean ok = true; + for (Statement stmt : stmts) { + try { + stmt.exec(env); + } catch (EvalException e) { + ok = false; + // Do not report errors caused by a previous parsing error, as it has already been + // reported. + if (e.isDueToIncompleteAST()) { + continue; + } + // When the exception is raised from another file, report first the location in the + // BUILD file (as it is the most probable cause for the error). + Location exnLoc = e.getLocation(); + Location nodeLoc = stmt.getLocation(); + if (exnLoc == null || !nodeLoc.getPath().equals(exnLoc.getPath())) { + eventHandler.handle(Event.error(nodeLoc, + e.getMessage() + " (raised from " + exnLoc + ")")); + } else { + eventHandler.handle(Event.error(exnLoc, e.getMessage())); + } + } + } + return ok; + } + + @Override + public String toString() { + return "BuildFileAST" + getStatements(); + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + /** + * Parse the specified build file, returning its AST. All errors during + * scanning or parsing will be reported to the reporter. + * + * @throws IOException if the file cannot not be read. + */ + public static BuildFileAST parseBuildFile(Path buildFile, EventHandler eventHandler, + CachingPackageLocator locator, boolean parsePython) + throws IOException { + ParserInputSource inputSource = ParserInputSource.create(buildFile); + return parseBuildFile(inputSource, eventHandler, locator, parsePython); + } + + /** + * Parse the specified build file, returning its AST. All errors during + * scanning or parsing will be reported to the reporter. + */ + public static BuildFileAST parseBuildFile(ParserInputSource input, + List<Statement> preludeStatements, + EventHandler eventHandler, + CachingPackageLocator locator, + boolean parsePython) { + Lexer lexer = new Lexer(input, eventHandler, parsePython); + Parser.ParseResult result = Parser.parseFile(lexer, eventHandler, locator, parsePython); + return new BuildFileAST(lexer, preludeStatements, result); + } + + public static BuildFileAST parseBuildFile(ParserInputSource input, EventHandler eventHandler, + CachingPackageLocator locator, boolean parsePython) { + Lexer lexer = new Lexer(input, eventHandler, parsePython); + Parser.ParseResult result = Parser.parseFile(lexer, eventHandler, locator, parsePython); + return new BuildFileAST(lexer, ImmutableList.<Statement>of(), result); + } + + /** + * Parse the specified build file, returning its AST. All errors during + * scanning or parsing will be reported to the reporter. + */ + public static BuildFileAST parseBuildFile(Lexer lexer, EventHandler eventHandler) { + Parser.ParseResult result = Parser.parseFile(lexer, eventHandler, null, false); + return new BuildFileAST(lexer, ImmutableList.<Statement>of(), result); + } + + /** + * Parse the specified Skylark file, returning its AST. All errors during + * scanning or parsing will be reported to the reporter. + * + * @throws IOException if the file cannot not be read. + */ + public static BuildFileAST parseSkylarkFile(Path file, EventHandler eventHandler, + CachingPackageLocator locator, ValidationEnvironment validationEnvironment) + throws IOException { + ParserInputSource input = ParserInputSource.create(file); + Lexer lexer = new Lexer(input, eventHandler, false); + Parser.ParseResult result = + Parser.parseFileForSkylark(lexer, eventHandler, locator, validationEnvironment); + return new BuildFileAST(lexer, ImmutableList.<Statement>of(), result, input.contentHashCode()); + } + + /** + * Parse the specified build file, without building the AST. + * + * @return true if the input file is syntactically valid + */ + public static boolean checkSyntax(ParserInputSource input, + EventHandler eventHandler, boolean parsePython) { + return !parseBuildFile(input, eventHandler, null, parsePython).containsErrors(); + } + + /** + * Returns a hash code calculated from the string content of the source file of this AST. + */ + @Nullable public String getContentHashCode() { + return contentHashCode; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ClassObject.java b/src/main/java/com/google/devtools/build/lib/syntax/ClassObject.java new file mode 100644 index 0000000..3b1cccf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/ClassObject.java
@@ -0,0 +1,113 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import com.google.common.collect.Sets.SetView; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.Location; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * An interface for objects behaving like Skylark structs. + */ +// TODO(bazel-team): type checks +public interface ClassObject { + + /** + * Returns the value associated with the name field in this struct, + * or null if the field does not exist. + */ + @Nullable + Object getValue(String name); + + /** + * Returns the fields of this struct. + */ + ImmutableCollection<String> getKeys(); + + /** + * Returns a customized error message to print if the name is not a valid struct field + * of this struct, or returns null to use the default error message. + */ + @Nullable String errorMessage(String name); + + /** + * An implementation class of ClassObject for structs created in Skylark code. + */ + @Immutable + @SkylarkModule(name = "struct", + doc = "A special language element to support structs (i.e. simple value objects). " + + "See the global <code>struct</code> method for more details.") + public class SkylarkClassObject implements ClassObject { + + private final ImmutableMap<String, Object> values; + private final Location creationLoc; + private final String errorMessage; + + /** + * Creates a built-in struct (i.e. without creation loc). The errorMessage has to have + * exactly one '%s' parameter to substitute the struct field name. + */ + public SkylarkClassObject(Map<String, Object> values, String errorMessage) { + this.values = ImmutableMap.copyOf(values); + this.creationLoc = null; + this.errorMessage = errorMessage; + } + + public SkylarkClassObject(Map<String, Object> values, Location creationLoc) { + this.values = ImmutableMap.copyOf(values); + this.creationLoc = Preconditions.checkNotNull(creationLoc); + this.errorMessage = null; + } + + @Override + public Object getValue(String name) { + return values.get(name); + } + + @Override + public ImmutableCollection<String> getKeys() { + return values.keySet(); + } + + public Location getCreationLoc() { + return Preconditions.checkNotNull(creationLoc, + "This struct was not created in a Skylark code"); + } + + static SkylarkClassObject concat( + SkylarkClassObject lval, SkylarkClassObject rval, Location loc) throws EvalException { + SetView<String> commonFields = Sets.intersection(lval.values.keySet(), rval.values.keySet()); + if (!commonFields.isEmpty()) { + throw new EvalException(loc, "Cannot concat structs with common field(s): " + + Joiner.on(",").join(commonFields)); + } + return new SkylarkClassObject(ImmutableMap.<String, Object>builder() + .putAll(lval.values).putAll(rval.values).build(), loc); + } + + @Override + public String errorMessage(String name) { + return errorMessage != null ? String.format(errorMessage, name) : null; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/CommaSeparatedPackageNameListConverter.java b/src/main/java/com/google/devtools/build/lib/syntax/CommaSeparatedPackageNameListConverter.java new file mode 100644 index 0000000..070e928 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/CommaSeparatedPackageNameListConverter.java
@@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.cmdline.LabelValidator; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.List; + +/** + * A converter from strings containing comma-separated names of packages to lists of strings. + */ +public class CommaSeparatedPackageNameListConverter + implements Converter<List<String>> { + + private static final Splitter SPACE_SPLITTER = Splitter.on(','); + + @Override + public List<String> convert(String input) throws OptionsParsingException { + if (Strings.isNullOrEmpty(input)) { + return ImmutableList.of(); + } + ImmutableList.Builder<String> list = ImmutableList.builder(); + for (String s : SPACE_SPLITTER.split(input)) { + String errorMessage = LabelValidator.validatePackageName(s); + if (errorMessage != null) { + throw new OptionsParsingException(errorMessage); + } + list.add(s); + } + return list.build(); + } + + @Override + public String getTypeDescription() { + return "comma-separated list of package names"; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Comment.java b/src/main/java/com/google/devtools/build/lib/syntax/Comment.java new file mode 100644 index 0000000..29d9474 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/Comment.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +/** + * Syntax node for comments. + */ +public final class Comment extends ASTNode { + + protected final String value; + + public Comment(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + public String toString() { + return value; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/DictComprehension.java b/src/main/java/com/google/devtools/build/lib/syntax/DictComprehension.java new file mode 100644 index 0000000..a69605e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/DictComprehension.java
@@ -0,0 +1,102 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.collect.ImmutableMap; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Syntax node for dictionary comprehension expressions. + */ +public class DictComprehension extends Expression { + + private final Expression keyExpression; + private final Expression valueExpression; + private final Ident loopVar; + private final Expression listExpression; + + public DictComprehension(Expression keyExpression, Expression valueExpression, Ident loopVar, + Expression listExpression) { + this.keyExpression = keyExpression; + this.valueExpression = valueExpression; + this.loopVar = loopVar; + this.listExpression = listExpression; + } + + Expression getKeyExpression() { + return keyExpression; + } + + Expression getValueExpression() { + return valueExpression; + } + + Ident getLoopVar() { + return loopVar; + } + + Expression getListExpression() { + return listExpression; + } + + @Override + Object eval(Environment env) throws EvalException, InterruptedException { + // We want to keep the iteration order + LinkedHashMap<Object, Object> map = new LinkedHashMap<>(); + Iterable<?> elements = EvalUtils.toIterable(listExpression.eval(env), getLocation()); + for (Object element : elements) { + env.update(loopVar.getName(), element); + Object key = keyExpression.eval(env); + map.put(key, valueExpression.eval(env)); + } + return ImmutableMap.copyOf(map); + } + + @Override + SkylarkType validate(ValidationEnvironment env) throws EvalException { + SkylarkType elementsType = listExpression.validate(env); + // TODO(bazel-team): GenericType1 should be a SkylarkType. + Class<?> listElementType = elementsType.getGenericType1(); + SkylarkType listElementSkylarkType = listElementType.equals(Object.class) + ? SkylarkType.UNKNOWN : SkylarkType.of(listElementType); + env.update(loopVar.getName(), listElementSkylarkType, getLocation()); + SkylarkType keyType = keyExpression.validate(env); + if (!keyType.isSimple()) { + // TODO(bazel-team): this is most probably dead code but it's better to have it here + // in case we enable e.g. list of lists or we validate function calls on Java objects + throw new EvalException(getLocation(), "Dict comprehension key must be of a simple type"); + } + valueExpression.validate(env); + if (elementsType != SkylarkType.UNKNOWN && !elementsType.isList()) { + throw new EvalException(getLocation(), "Dict comprehension elements must be a list"); + } + return SkylarkType.of(Map.class, keyType.getType()); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append('{').append(keyExpression).append(": ").append(valueExpression); + sb.append(" for ").append(loopVar).append(" in ").append(listExpression); + sb.append('}'); + return sb.toString(); + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.accept(this); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/DictionaryLiteral.java b/src/main/java/com/google/devtools/build/lib/syntax/DictionaryLiteral.java new file mode 100644 index 0000000..8f79739 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/DictionaryLiteral.java
@@ -0,0 +1,117 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.collect.ImmutableList; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Syntax node for dictionary literals. + */ +public class DictionaryLiteral extends Expression { + + static final class DictionaryEntryLiteral extends ASTNode { + + private final Expression key; + private final Expression value; + + public DictionaryEntryLiteral(Expression key, Expression value) { + this.key = key; + this.value = value; + } + + Expression getKey() { + return key; + } + + Expression getValue() { + return value; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(key); + sb.append(": "); + sb.append(value); + return sb.toString(); + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + } + + private final ImmutableList<DictionaryEntryLiteral> entries; + + public DictionaryLiteral(List<DictionaryEntryLiteral> exprs) { + this.entries = ImmutableList.copyOf(exprs); + } + + @Override + Object eval(Environment env) throws EvalException, InterruptedException { + // We need LinkedHashMap to maintain the order during iteration (e.g. for loops) + Map<Object, Object> map = new LinkedHashMap<>(); + for (DictionaryEntryLiteral entry : entries) { + if (entry == null) { + throw new EvalException(getLocation(), "null expression in " + this); + } + map.put(entry.key.eval(env), entry.value.eval(env)); + + } + return map; + } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("{"); + String sep = ""; + for (DictionaryEntryLiteral e : entries) { + sb.append(sep); + sb.append(e); + sep = ", "; + } + sb.append("}"); + return sb.toString(); + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + public ImmutableList<DictionaryEntryLiteral> getEntries() { + return entries; + } + + @Override + SkylarkType validate(ValidationEnvironment env) throws EvalException { + SkylarkType type = SkylarkType.UNKNOWN; + for (DictionaryEntryLiteral entry : entries) { + SkylarkType nextType = entry.key.validate(env); + entry.value.validate(env); + if (!nextType.isSimple()) { + throw new EvalException(getLocation(), + String.format("Dict cannot contain composite type '%s' as key", nextType)); + } + type = type.infer(nextType, "dict literal", entry.getLocation(), getLocation()); + } + return SkylarkType.of(Map.class, type.getType()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/DotExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/DotExpression.java new file mode 100644 index 0000000..b0ae5a9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/DotExpression.java
@@ -0,0 +1,110 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.FuncallExpression.MethodDescriptor; + +import java.lang.reflect.InvocationTargetException; +import java.util.List; +import java.util.concurrent.ExecutionException; + +/** + * Syntax node for a dot expression. + * e.g. obj.field, but not obj.method() + */ +public final class DotExpression extends Expression { + + private final Expression obj; + + private final Ident field; + + public DotExpression(Expression obj, Ident field) { + this.obj = obj; + this.field = field; + } + + public Expression getObj() { + return obj; + } + + public Ident getField() { + return field; + } + + @Override + public String toString() { + return obj + "." + field; + } + + @Override + Object eval(Environment env) throws EvalException, InterruptedException { + Object objValue = obj.eval(env); + String name = field.getName(); + Object result = eval(objValue, name, getLocation()); + if (result == null) { + if (objValue instanceof ClassObject) { + String customErrorMessage = ((ClassObject) objValue).errorMessage(name); + if (customErrorMessage != null) { + throw new EvalException(getLocation(), customErrorMessage); + } + } + throw new EvalException(getLocation(), "Object of type '" + + EvalUtils.getDatatypeName(objValue) + "' has no field '" + name + "'"); + } + return result; + } + + /** + * Returns the field of the given name of the struct objValue, or null if no such field exists. + */ + public static Object eval(Object objValue, String name, Location loc) throws EvalException { + Object result = null; + if (objValue instanceof ClassObject) { + result = ((ClassObject) objValue).getValue(name); + result = SkylarkType.convertToSkylark(result, loc); + // If we access NestedSets using ClassObject.getValue() we won't know the generic type, + // so we have to disable it. This should not happen. + SkylarkType.checkTypeAllowedInSkylark(result, loc); + } else { + try { + List<MethodDescriptor> methods = FuncallExpression.getMethods(objValue.getClass(), name, 0); + if (methods != null && methods.size() > 0) { + MethodDescriptor method = Iterables.getOnlyElement(methods); + if (method.getAnnotation().structField()) { + result = FuncallExpression.callMethod( + method, name, objValue, new Object[] {}, loc); + } + } + } catch (ExecutionException | IllegalAccessException | InvocationTargetException e) { + throw new EvalException(loc, "Method invocation failed: " + e); + } + } + + return result; + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + SkylarkType validate(ValidationEnvironment env) throws EvalException { + obj.validate(env); + // TODO(bazel-team): check existance of field + return SkylarkType.UNKNOWN; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Environment.java b/src/main/java/com/google/devtools/build/lib/syntax/Environment.java new file mode 100644 index 0000000..a148a70 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/Environment.java
@@ -0,0 +1,345 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * The BUILD environment. + */ +public class Environment { + + @SkylarkBuiltin(name = "True", returnType = Boolean.class, doc = "Literal for the boolean true.") + private static final Boolean TRUE = true; + + @SkylarkBuiltin(name = "False", returnType = Boolean.class, + doc = "Literal for the boolean false.") + private static final Boolean FALSE = false; + + @SkylarkBuiltin(name = "PACKAGE_NAME", returnType = String.class, + doc = "The name of the package the rule or build extension is called from. " + + "This variable is special, because its value comes from outside of the extension " + + "module (it comes from the BUILD file), so it can only be accessed in functions " + + "(transitively) called from BUILD files. For example:<br>" + + "<pre class=language-python>def extension():\n" + + " return PACKAGE_NAME</pre>" + + "In this case calling <code>extension()</code> works from the BUILD file (if the " + + "function is loaded), but not as a top level function call in the extension module.") + public static final String PKG_NAME = "PACKAGE_NAME"; + + /** + * There should be only one instance of this type to allow "== None" tests. + */ + @Immutable + public static final class NoneType { + @Override + public String toString() { return "None"; } + private NoneType() {} + } + + @SkylarkBuiltin(name = "None", returnType = NoneType.class, doc = "Literal for the None value.") + public static final NoneType NONE = new NoneType(); + + protected final Map<String, Object> env = new HashMap<>(); + + // Functions with namespaces. Works only in the global environment. + protected final Map<Class<?>, Map<String, Function>> functions = new HashMap<>(); + + /** + * The parent environment. For Skylark it's the global environment, + * used for global read only variable lookup. + */ + protected final Environment parent; + + /** + * Map from a Skylark extension to an environment, which contains all symbols defined in the + * extension. + */ + protected Map<PathFragment, SkylarkEnvironment> importedExtensions; + + /** + * A set of disable variables propagating through function calling. This is needed because + * UserDefinedFunctions lock the definition Environment which should be immutable. + */ + protected Set<String> disabledVariables = new HashSet<>(); + + /** + * A set of disable namespaces propagating through function calling. See disabledVariables. + */ + protected Set<Class<?>> disabledNameSpaces = new HashSet<>(); + + /** + * A set of variables propagating through function calling. It's only used to call + * native rules from Skylark build extensions. + */ + protected Set<String> propagatingVariables = new HashSet<>(); + + /** + * An EventHandler for errors and warnings. This is not used in the BUILD language, + * however it might be used in Skylark code called from the BUILD language. + */ + @Nullable protected EventHandler eventHandler; + + /** + * Constructs an empty root non-Skylark environment. + * The root environment is also the global environment. + */ + public Environment() { + this.parent = null; + this.importedExtensions = new HashMap<>(); + setupGlobal(); + } + + /** + * Constructs an empty child environment. + */ + public Environment(Environment parent) { + Preconditions.checkNotNull(parent); + this.parent = parent; + this.importedExtensions = new HashMap<>(); + } + + /** + * Constructs an empty child environment with an EventHandler. + */ + public Environment(Environment parent, EventHandler eventHandler) { + this(parent); + this.eventHandler = Preconditions.checkNotNull(eventHandler); + } + + // Sets up the global environment + private void setupGlobal() { + // In Python 2.x, True and False are global values and can be redefined by the user. + // In Python 3.x, they are keywords. We implement them as values, for the sake of + // simplicity. We define them as Boolean objects. + env.put("False", FALSE); + env.put("True", TRUE); + env.put("None", NONE); + } + + public boolean isSkylarkEnabled() { + return false; + } + + protected boolean hasVariable(String varname) { + return env.containsKey(varname); + } + + /** + * @return the value from the environment whose name is "varname". + * @throws NoSuchVariableException if the variable is not defined in the Environment. + * + */ + public Object lookup(String varname) throws NoSuchVariableException { + if (disabledVariables.contains(varname)) { + throw new NoSuchVariableException(varname); + } + Object value = env.get(varname); + if (value == null) { + if (parent != null) { + return parent.lookup(varname); + } + throw new NoSuchVariableException(varname); + } + return value; + } + + /** + * Like <code>lookup(String)</code>, but instead of throwing an exception in + * the case where "varname" is not defined, "defaultValue" is returned instead. + * + */ + public Object lookup(String varname, Object defaultValue) { + Object value = env.get(varname); + if (value == null) { + if (parent != null) { + return parent.lookup(varname, defaultValue); + } + return defaultValue; + } + return value; + } + + /** + * Updates the value of variable "varname" in the environment, corresponding + * to an {@link AssignmentStatement}. + */ + public void update(String varname, Object value) { + Preconditions.checkNotNull(value, "update(value == null)"); + env.put(varname, value); + } + + /** + * Same as {@link #update}, but also marks the variable propagating, meaning it will + * be present in the execution environment of a UserDefinedFunction called from this + * Environment. Using this method is discouraged. + */ + public void updateAndPropagate(String varname, Object value) { + update(varname, value); + propagatingVariables.add(varname); + } + + /** + * Remove the variable from the environment, returning + * any previous mapping (null if there was none). + */ + public Object remove(String varname) { + return env.remove(varname); + } + + /** + * Returns the (immutable) set of names of all variables defined in this + * environment. Exposed for testing; not very efficient! + */ + @VisibleForTesting + public Set<String> getVariableNames() { + if (parent == null) { + return env.keySet(); + } else { + Set<String> vars = new HashSet<>(); + vars.addAll(env.keySet()); + vars.addAll(parent.getVariableNames()); + return vars; + } + } + + @Override + public int hashCode() { + throw new UnsupportedOperationException(); // avoid nondeterminism + } + + @Override + public boolean equals(Object that) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + StringBuilder out = new StringBuilder(); + out.append("Environment{"); + List<String> keys = new ArrayList<>(env.keySet()); + Collections.sort(keys); + for (String key: keys) { + out.append(key).append(" -> ").append(env.get(key)).append(", "); + } + out.append("}"); + if (parent != null) { + out.append("=>"); + out.append(parent.toString()); + } + return out.toString(); + } + + /** + * An exception thrown when an attempt is made to lookup a non-existent + * variable in the environment. + */ + public static class NoSuchVariableException extends Exception { + NoSuchVariableException(String variable) { + super("no such variable: " + variable); + } + } + + /** + * An exception thrown when an attempt is made to import a symbol from a file + * that was not properly loaded. + */ + public static class LoadFailedException extends Exception { + LoadFailedException(String file) { + super("file '" + file + "' was not correctly loaded. Make sure the 'load' statement appears " + + "in the global scope, in the BUILD file"); + } + } + + public void setImportedExtensions(Map<PathFragment, SkylarkEnvironment> importedExtensions) { + this.importedExtensions = importedExtensions; + } + + public void importSymbol(PathFragment extension, String symbol) + throws NoSuchVariableException, LoadFailedException { + if (!importedExtensions.containsKey(extension)) { + throw new LoadFailedException(extension.toString()); + } + Object value = importedExtensions.get(extension).lookup(symbol); + if (!isSkylarkEnabled()) { + value = SkylarkType.convertFromSkylark(value); + } + update(symbol, value); + } + + /** + * Registers a function with namespace to this global environment. + */ + public void registerFunction(Class<?> nameSpace, String name, Function function) { + Preconditions.checkArgument(parent == null); + if (!functions.containsKey(nameSpace)) { + functions.put(nameSpace, new HashMap<String, Function>()); + } + functions.get(nameSpace).put(name, function); + } + + private Map<String, Function> getNamespaceFunctions(Class<?> nameSpace) { + if (disabledNameSpaces.contains(nameSpace) + || (parent != null && parent.disabledNameSpaces.contains(nameSpace))) { + return null; + } + Environment topLevel = this; + while (topLevel.parent != null) { + topLevel = topLevel.parent; + } + return topLevel.functions.get(nameSpace); + } + + /** + * Returns the function of the namespace of the given name or null of it does not exists. + */ + public Function getFunction(Class<?> nameSpace, String name) { + Map<String, Function> nameSpaceFunctions = getNamespaceFunctions(nameSpace); + return nameSpaceFunctions != null ? nameSpaceFunctions.get(name) : null; + } + + /** + * Returns the function names registered with the namespace. + */ + public Set<String> getFunctionNames(Class<?> nameSpace) { + Map<String, Function> nameSpaceFunctions = getNamespaceFunctions(nameSpace); + return nameSpaceFunctions != null ? nameSpaceFunctions.keySet() : ImmutableSet.<String>of(); + } + + /** + * Return the current stack trace (list of function names). + */ + public ImmutableList<String> getStackTrace() { + // Empty list, since this environment does not allow function definition + // (see SkylarkEnvironment) + return ImmutableList.of(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/EvalException.java b/src/main/java/com/google/devtools/build/lib/syntax/EvalException.java new file mode 100644 index 0000000..27aba0f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/EvalException.java
@@ -0,0 +1,105 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.events.Location; + +/** + * Exceptions thrown during evaluation of BUILD ASTs or Skylark extensions. + * + * <p>This exception must always correspond to a repeatable, permanent error, i.e. evaluating the + * same package again must yield the same exception. Notably, do not use this for reporting I/O + * errors. + * + * <p>This requirement is in place so that we can cache packages where an error is reported by way + * of {@link EvalException}. + */ +public class EvalException extends Exception { + + private final Location location; + private final String message; + private final boolean dueToIncompleteAST; + + /** + * @param location the location where evaluation/execution failed. + * @param message the error message. + */ + public EvalException(Location location, String message) { + this.location = location; + this.message = Preconditions.checkNotNull(message); + this.dueToIncompleteAST = false; + } + + /** + * @param location the location where evaluation/execution failed. + * @param message the error message. + * @param dueToIncompleteAST if the error is caused by a previous error, such as parsing. + */ + public EvalException(Location location, String message, boolean dueToIncompleteAST) { + this.location = location; + this.message = Preconditions.checkNotNull(message); + this.dueToIncompleteAST = dueToIncompleteAST; + } + + private EvalException(Location location, Throwable cause) { + super(cause); + this.location = location; + // This is only used from Skylark, it's useful for debugging. Note that this only happens + // when the Precondition below kills the execution anyway. + if (cause.getMessage() == null) { + cause.printStackTrace(); + } + this.message = Preconditions.checkNotNull(cause.getMessage()); + this.dueToIncompleteAST = false; + } + + /** + * Returns the error message. + */ + @Override + public String getMessage() { + return message; + } + + /** + * Returns the location of the evaluation error. + */ + public Location getLocation() { + return location; + } + + public boolean isDueToIncompleteAST() { + return dueToIncompleteAST; + } + + /** + * A class to support a special case of EvalException when the cause of the error is an + * Exception during a direct Java call. + */ + public static final class EvalExceptionWithJavaCause extends EvalException { + + public EvalExceptionWithJavaCause(Location location, Throwable cause) { + super(location, cause); + } + } + + /** + * Returns the error message with location info if exists. + */ + public String print() { + return getLocation() == null ? getMessage() : getLocation().print() + ": " + getMessage(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/EvalUtils.java b/src/main/java/com/google/devtools/build/lib/syntax/EvalUtils.java new file mode 100644 index 0000000..70d89bc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/EvalUtils.java
@@ -0,0 +1,590 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Formattable; +import java.util.Formatter; +import java.util.IllegalFormatException; +import java.util.List; +import java.util.Map; +import java.util.MissingFormatWidthException; +import java.util.Set; + +/** + * Utilities used by the evaluator. + */ +public abstract class EvalUtils { + + // TODO(bazel-team): Yet an other hack committed in the name of Skylark. One problem is that the + // syntax package cannot depend on actions so we have to have this until Actions are immutable. + // The other is that BuildConfigurations are technically not immutable but they cannot be modified + // from Skylark. + private static final ImmutableSet<Class<?>> quasiImmutableClasses; + static { + try { + ImmutableSet.Builder<Class<?>> builder = ImmutableSet.builder(); + builder.add(Class.forName("com.google.devtools.build.lib.actions.Action")); + builder.add(Class.forName("com.google.devtools.build.lib.analysis.config.BuildConfiguration")); + builder.add(Class.forName("com.google.devtools.build.lib.actions.Root")); + quasiImmutableClasses = builder.build(); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + private EvalUtils() { + } + + /** + * @return true if the specified sequence is a tuple; false if it's a modifiable list. + */ + public static boolean isTuple(List<?> l) { + return isTuple(l.getClass()); + } + + public static boolean isTuple(Class<?> c) { + Preconditions.checkState(List.class.isAssignableFrom(c)); + if (ImmutableList.class.isAssignableFrom(c)) { + return true; + } else { + return false; + } + } + + /** + * @return true if the specified value is immutable (suitable for use as a + * dictionary key) according to the rules of the Build language. + */ + public static boolean isImmutable(Object o) { + if (o instanceof Map<?, ?> || o instanceof Function + || o instanceof FilesetEntry || o instanceof GlobList<?>) { + return false; + } else if (o instanceof List<?>) { + return isTuple((List<?>) o); // tuples are immutable, lists are not. + } else { + return true; // string/int + } + } + + /** + * Returns true if the type is immutable in the skylark language. + */ + public static boolean isSkylarkImmutable(Class<?> c) { + if (c.isAnnotationPresent(Immutable.class)) { + return true; + } else if (c.equals(String.class) || c.equals(Integer.class) || c.equals(Boolean.class) + || SkylarkList.class.isAssignableFrom(c) || ImmutableMap.class.isAssignableFrom(c) + || NestedSet.class.isAssignableFrom(c)) { + return true; + } else { + for (Class<?> classObject : quasiImmutableClasses) { + if (classObject.isAssignableFrom(c)) { + return true; + } + } + } + return false; + } + + /** + * Returns a transitive superclass or interface implemented by c which is annotated + * with SkylarkModule. Returns null if no such class or interface exists. + */ + @VisibleForTesting + static Class<?> getParentWithSkylarkModule(Class<?> c) { + if (c == null) { + return null; + } + if (c.isAnnotationPresent(SkylarkModule.class)) { + return c; + } + Class<?> parent = getParentWithSkylarkModule(c.getSuperclass()); + if (parent != null) { + return parent; + } + for (Class<?> ifparent : c.getInterfaces()) { + ifparent = getParentWithSkylarkModule(ifparent); + if (ifparent != null) { + return ifparent; + } + } + return null; + } + + /** + * Returns the Skylark equivalent type of the parameter. Note that the Skylark + * language doesn't have inheritance. + */ + public static Class<?> getSkylarkType(Class<?> c) { + if (ImmutableList.class.isAssignableFrom(c)) { + return ImmutableList.class; + } else if (List.class.isAssignableFrom(c)) { + return List.class; + } else if (SkylarkList.class.isAssignableFrom(c)) { + return SkylarkList.class; + } else if (Map.class.isAssignableFrom(c)) { + return Map.class; + } else if (NestedSet.class.isAssignableFrom(c)) { + // This could be removed probably + return NestedSet.class; + } else if (Set.class.isAssignableFrom(c)) { + return Set.class; + } else { + // Check if one of the superclasses or implemented interfaces has the SkylarkModule + // annotation. If yes return that class. + Class<?> parent = getParentWithSkylarkModule(c); + if (parent != null) { + return parent; + } + } + return c; + } + + /** + * Returns a pretty name for the datatype of object 'o' in the Build language. + */ + public static String getDatatypeName(Object o) { + Preconditions.checkNotNull(o); + if (o instanceof SkylarkList) { + return ((SkylarkList) o).isTuple() ? "tuple" : "list"; + } + return getDataTypeNameFromClass(o.getClass()); + } + + /** + * Returns a pretty name for the datatype equivalent of class 'c' in the Build language. + */ + public static String getDataTypeNameFromClass(Class<?> c) { + if (c.equals(Object.class)) { + return "unknown"; + } else if (c.equals(String.class)) { + return "string"; + } else if (c.equals(Integer.class)) { + return "int"; + } else if (c.equals(Boolean.class)) { + return "bool"; + } else if (c.equals(Void.TYPE) || c.equals(Environment.NoneType.class)) { + return "None"; + } else if (List.class.isAssignableFrom(c)) { + return isTuple(c) ? "tuple" : "list"; + } else if (GlobList.class.isAssignableFrom(c)) { + return "list"; + } else if (Map.class.isAssignableFrom(c)) { + return "dict"; + } else if (Function.class.isAssignableFrom(c)) { + return "function"; + } else if (c.equals(FilesetEntry.class)) { + return "FilesetEntry"; + } else if (NestedSet.class.isAssignableFrom(c) || SkylarkNestedSet.class.isAssignableFrom(c)) { + return "set"; + } else if (SkylarkClassObject.class.isAssignableFrom(c)) { + return "struct"; + } else if (SkylarkList.class.isAssignableFrom(c)) { + // TODO(bazel-team): this is not entirely correct, it can also be a tuple. + return "list"; + } else if (c.isAnnotationPresent(SkylarkModule.class)) { + SkylarkModule module = c.getAnnotation(SkylarkModule.class); + return c.getAnnotation(SkylarkModule.class).name() + + (module.namespace() ? " (a language module)" : ""); + } else { + if (c.getSimpleName().isEmpty()) { + return c.getName(); + } else { + return c.getSimpleName(); + } + } + } + + /** + * Returns a sequence of the appropriate list/tuple datatype for 'seq', based on 'isTuple'. + */ + public static List<?> makeSequence(List<?> seq, boolean isTuple) { + return isTuple ? ImmutableList.copyOf(seq) : seq; + } + + /** + * Print build-language value 'o' in display format into the specified buffer. + */ + public static void printValue(Object o, Appendable buffer) { + // Exception-swallowing wrapper due to annoying Appendable interface. + try { + printValueX(o, buffer); + } catch (IOException e) { + throw new AssertionError(e); // can't happen + } + } + + private static void printValueX(Object o, Appendable buffer) + throws IOException { + if (o == null) { + throw new NullPointerException(); // None is not a build language value. + } else if (o instanceof String || + o instanceof Integer || + o instanceof Double) { + buffer.append(o.toString()); + + } else if (o == Environment.NONE) { + buffer.append("None"); + + } else if (o == Boolean.TRUE) { + buffer.append("True"); + + } else if (o == Boolean.FALSE) { + buffer.append("False"); + + } else if (o instanceof List<?>) { + List<?> seq = (List<?>) o; + boolean isTuple = isImmutable(seq); + String sep = ""; + buffer.append(isTuple ? '(' : '['); + for (int ii = 0, len = seq.size(); ii < len; ++ii) { + buffer.append(sep); + prettyPrintValue(seq.get(ii), buffer); + sep = ", "; + } + buffer.append(isTuple ? ')' : ']'); + + } else if (o instanceof Map<?, ?>) { + Map<?, ?> dict = (Map<?, ?>) o; + buffer.append('{'); + String sep = ""; + for (Map.Entry<?, ?> entry : dict.entrySet()) { + buffer.append(sep); + prettyPrintValue(entry.getKey(), buffer); + buffer.append(": "); + prettyPrintValue(entry.getValue(), buffer); + sep = ", "; + } + buffer.append('}'); + + } else if (o instanceof Function) { + Function func = (Function) o; + buffer.append("<function " + func.getName() + ">"); + + } else if (o instanceof FilesetEntry) { + FilesetEntry entry = (FilesetEntry) o; + buffer.append("FilesetEntry(srcdir = "); + prettyPrintValue(entry.getSrcLabel().toString(), buffer); + buffer.append(", files = "); + prettyPrintValue(makeStringList(entry.getFiles()), buffer); + buffer.append(", excludes = "); + prettyPrintValue(makeList(entry.getExcludes()), buffer); + buffer.append(", destdir = "); + prettyPrintValue(entry.getDestDir().getPathString(), buffer); + buffer.append(", strip_prefix = "); + prettyPrintValue(entry.getStripPrefix(), buffer); + buffer.append(", symlinks = \""); + buffer.append(entry.getSymlinkBehavior().toString()); + buffer.append("\")"); + } else if (o instanceof PathFragment) { + buffer.append(((PathFragment) o).getPathString()); + } else { + buffer.append(o.toString()); + } + } + + private static List<?> makeList(Collection<?> list) { + return list == null ? Lists.newArrayList() : Lists.newArrayList(list); + } + + private static List<String> makeStringList(List<Label> labels) { + if (labels == null) { return Collections.emptyList(); } + List<String> strings = Lists.newArrayListWithCapacity(labels.size()); + for (Label label : labels) { + strings.add(label.toString()); + } + return strings; + } + + /** + * Print build-language value 'o' in parseable format into the specified + * buffer. (Only differs from printValueX in treatment of strings at toplevel, + * i.e. not within a sequence or dict) + */ + public static void prettyPrintValue(Object o, Appendable buffer) { + // Exception-swallowing wrapper due to annoying Appendable interface. + try { + prettyPrintValueX(o, buffer); + } catch (IOException e) { + throw new AssertionError(e); // can't happen + } + } + + private static void prettyPrintValueX(Object o, Appendable buffer) + throws IOException { + if (o instanceof Label) { + o = o.toString(); // Pretty-print a label like a string + } + if (o instanceof String) { + String s = (String) o; + buffer.append('"'); + for (int ii = 0, len = s.length(); ii < len; ++ii) { + char c = s.charAt(ii); + switch (c) { + case '\r': + buffer.append('\\').append('r'); + break; + case '\n': + buffer.append('\\').append('n'); + break; + case '\t': + buffer.append('\\').append('t'); + break; + case '\"': + buffer.append('\\').append('"'); + break; + default: + if (c < 32) { + buffer.append(String.format("\\x%02x", (int) c)); + } else { + buffer.append(c); // no need to support UTF-8 + } + } // endswitch + } + buffer.append('\"'); + } else { + printValueX(o, buffer); + } + } + + /** + * Pretty-print value 'o' to a string. Convenience overloading of + * prettyPrintValue(Object, Appendable). + */ + public static String prettyPrintValue(Object o) { + StringBuffer buffer = new StringBuffer(); + prettyPrintValue(o, buffer); + return buffer.toString(); + } + + /** + * Pretty-print values of 'o' separated by the separator. + */ + public static String prettyPrintValues(String separator, Iterable<Object> o) { + return Joiner.on(separator).join(Iterables.transform(o, + new com.google.common.base.Function<Object, String>() { + @Override + public String apply(Object input) { + return prettyPrintValue(input); + } + })); + } + + /** + * Print value 'o' to a string. Convenience overloading of printValue(Object, Appendable). + */ + public static String printValue(Object o) { + StringBuffer buffer = new StringBuffer(); + printValue(o, buffer); + return buffer.toString(); + } + + public static Object checkNotNull(Expression expr, Object obj) throws EvalException { + if (obj == null) { + throw new EvalException(expr.getLocation(), + "Unexpected null value, please send a bug report. " + + "This was generated by '" + expr + "'"); + } + return obj; + } + + /** + * Convert BUILD language objects to Formattable so JDK can render them correctly. + * Don't do this for numeric or string types because we want %d, %x, %s to work. + */ + private static Object makeFormattable(final Object o) { + if (o instanceof Integer || o instanceof Double || o instanceof String) { + return o; + } else { + return new Formattable() { + @Override + public String toString() { + return "Formattable[" + o + "]"; + } + + @Override + public void formatTo(Formatter formatter, int flags, int width, + int precision) { + printValue(o, formatter.out()); + } + }; + } + } + + private static final Object[] EMPTY = new Object[0]; + + /* + * N.B. MissingFormatWidthException is the only kind of IllegalFormatException + * whose constructor can take and display arbitrary error message, hence its use below. + */ + + /** + * Perform Python-style string formatting. Implemented by delegation to Java's + * own string formatting routine to avoid reinventing the wheel. In more + * obscure cases, semantics follow JDK (not Python) rules. + * + * @param pattern a format string. + * @param tuple a tuple containing positional arguments + */ + public static String formatString(String pattern, List<?> tuple) + throws IllegalFormatException { + int count = countPlaceholders(pattern); + if (count < tuple.size()) { + throw new MissingFormatWidthException( + "not all arguments converted during string formatting"); + } + + List<Object> args = new ArrayList<>(); + + for (Object o : tuple) { + args.add(makeFormattable(o)); + } + + try { + return String.format(pattern, args.toArray(EMPTY)); + } catch (IllegalFormatException e) { + throw new MissingFormatWidthException( + "invalid arguments for format string"); + } + } + + private static int countPlaceholders(String pattern) { + int length = pattern.length(); + boolean afterPercent = false; + int i = 0; + int count = 0; + while (i < length) { + switch (pattern.charAt(i)) { + case 's': + case 'd': + if (afterPercent) { + count++; + afterPercent = false; + } + break; + + case '%': + afterPercent = !afterPercent; + break; + + default: + if (afterPercent) { + throw new MissingFormatWidthException("invalid arguments for format string"); + } + afterPercent = false; + break; + } + i++; + } + + return count; + } + + /** + * @return the truth value of an object, according to Python rules. + * http://docs.python.org/2/library/stdtypes.html#truth-value-testing + */ + public static boolean toBoolean(Object o) { + if (o == null || o == Environment.NONE) { + return false; + } else if (o instanceof Boolean) { + return (Boolean) o; + } else if (o instanceof String) { + return !((String) o).isEmpty(); + } else if (o instanceof Integer) { + return (Integer) o != 0; + } else if (o instanceof Collection<?>) { + return !((Collection<?>) o).isEmpty(); + } else if (o instanceof Map<?, ?>) { + return !((Map<?, ?>) o).isEmpty(); + } else if (o instanceof NestedSet<?>) { + return !((NestedSet<?>) o).isEmpty(); + } else if (o instanceof SkylarkNestedSet) { + return !((SkylarkNestedSet) o).isEmpty(); + } else if (o instanceof Iterable<?>) { + return !(Iterables.isEmpty((Iterable<?>) o)); + } else { + return true; + } + } + + @SuppressWarnings("unchecked") + public static Collection<Object> toCollection(Object o, Location loc) throws EvalException { + if (o instanceof Collection) { + return (Collection<Object>) o; + } else if (o instanceof Map<?, ?>) { + // For dictionaries we iterate through the keys only + return ((Map<Object, Object>) o).keySet(); + } else if (o instanceof SkylarkNestedSet) { + return ((SkylarkNestedSet) o).toCollection(); + } else { + throw new EvalException(loc, + "type '" + EvalUtils.getDatatypeName(o) + "' is not a collection"); + } + } + + @SuppressWarnings("unchecked") + public static Iterable<Object> toIterable(Object o, Location loc) throws EvalException { + if (o instanceof String) { + // This is not as efficient as special casing String in for and dict and list comprehension + // statements. However this is a more unified way. + // The regex matches every character in the string until the end of the string, + // so "abc" will be split into ["a", "b", "c"]. + return ImmutableList.<Object>copyOf(((String) o).split("(?!^)")); + } else if (o instanceof Iterable) { + return (Iterable<Object>) o; + } else if (o instanceof Map<?, ?>) { + // For dictionaries we iterate through the keys only + return ((Map<Object, Object>) o).keySet(); + } else { + throw new EvalException(loc, + "type '" + EvalUtils.getDatatypeName(o) + "' is not an iterable"); + } + } + + /** + * Returns the size of the Skylark object or -1 in case the object doesn't have a size. + */ + public static int size(Object arg) { + if (arg instanceof String) { + return ((String) arg).length(); + } else if (arg instanceof Map) { + return ((Map<?, ?>) arg).size(); + } else if (arg instanceof SkylarkList) { + return ((SkylarkList) arg).size(); + } else if (arg instanceof Iterable) { + // Iterables.size() checks if arg is a Collection so it's efficient in that sense. + return Iterables.size((Iterable<?>) arg); + } + return -1; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Expression.java b/src/main/java/com/google/devtools/build/lib/syntax/Expression.java new file mode 100644 index 0000000..1659eb0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/Expression.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +/** + * Base class for all expression nodes in the AST. + */ +public abstract class Expression extends ASTNode { + + /** + * Returns the result of evaluating this build-language expression in the + * specified environment. All BUILD language datatypes are mapped onto the + * corresponding Java types as follows: + * + * <pre> + * int -> Integer + * float -> Double (currently not generated by the grammar) + * str -> String + * [...] -> List<Object> (mutable) + * (...) -> List<Object> (immutable) + * {...} -> Map<Object, Object> + * func -> Function + * </pre> + * + * @return the result of evaluting the expression: a Java object corresponding + * to a datatype in the BUILD language. + * @throws EvalException if the expression could not be evaluated. + */ + abstract Object eval(Environment env) throws EvalException, InterruptedException; + + /** + * Returns the inferred type of the result of the Expression. + * + * <p>Checks the semantics of the Expression using the SkylarkEnvironment according to + * the rules of the Skylark language, throws EvalException in case of a semantical error. + * + * @see Statement + */ + abstract SkylarkType validate(ValidationEnvironment env) throws EvalException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ExpressionStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/ExpressionStatement.java new file mode 100644 index 0000000..f742d40 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/ExpressionStatement.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +/** + * Syntax node for a function call statement. Used for build rules. + */ +public final class ExpressionStatement extends Statement { + + private final Expression expr; + + public ExpressionStatement(Expression expr) { + this.expr = expr; + } + + public Expression getExpression() { + return expr; + } + + @Override + public String toString() { + return expr.toString() + '\n'; + } + + @Override + void exec(Environment env) throws EvalException, InterruptedException { + expr.eval(env); + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + void validate(ValidationEnvironment env) throws EvalException { + expr.validate(env); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/FilesetEntry.java b/src/main/java/com/google/devtools/build/lib/syntax/FilesetEntry.java new file mode 100644 index 0000000..4586b64 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/FilesetEntry.java
@@ -0,0 +1,175 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * FilesetEntry is a value object used to represent a "FilesetEntry" inside a "Fileset" BUILD rule. + */ +public final class FilesetEntry { + /** SymlinkBehavior decides what to do when a source file of a FilesetEntry is a symlink. */ + public enum SymlinkBehavior { + /** Just copies the symlink as-is. May result in dangling links. */ + COPY, + /** Follow the link and make the destination point to the absolute path of the final target. */ + DEREFERENCE; + + public static SymlinkBehavior parse(String value) throws IllegalArgumentException { + return valueOf(value.toUpperCase()); + } + + @Override + public String toString() { + return super.toString().toLowerCase(); + } + } + + private final Label srcLabel; + @Nullable private final ImmutableList<Label> files; + @Nullable private final ImmutableSet<String> excludes; + private final PathFragment destDir; + private final SymlinkBehavior symlinkBehavior; + private final String stripPrefix; + + /** + * Constructs a FilesetEntry with the given values. + * + * @param srcLabel the label of the source directory. Must be non-null. + * @param files The explicit files to include. May be null. + * @param excludes The files to exclude. Man be null. May only be non-null if files is null. + * @param destDir The target-relative output directory. + * @param symlinkBehavior how to treat symlinks on the input. See + * {@link FilesetEntry.SymlinkBehavior}. + * @param stripPrefix the prefix to strip from the package-relative path. If ".", keep only the + * basename. + */ + public FilesetEntry(Label srcLabel, + @Nullable List<Label> files, + @Nullable List<String> excludes, + String destDir, + SymlinkBehavior symlinkBehavior, + String stripPrefix) { + this.srcLabel = checkNotNull(srcLabel); + this.destDir = new PathFragment((destDir == null) ? "" : destDir); + this.files = files == null ? null : ImmutableList.copyOf(files); + this.excludes = (excludes == null || excludes.isEmpty()) ? null : ImmutableSet.copyOf(excludes); + this.symlinkBehavior = symlinkBehavior; + this.stripPrefix = stripPrefix; + } + + /** + * @return the source label. + */ + public Label getSrcLabel() { + return srcLabel; + } + + /** + * @return the destDir. Non null. + */ + public PathFragment getDestDir() { + return destDir; + } + + /** + * @return how symlinks should be handled. + */ + public SymlinkBehavior getSymlinkBehavior() { + return symlinkBehavior; + } + + /** + * @return an immutable list of excludes. Null if none specified. + */ + @Nullable + public ImmutableSet<String> getExcludes() { + return excludes; + } + + /** + * @return an immutable list of file labels. Null if none specified. + */ + @Nullable + public ImmutableList<Label> getFiles() { + return files; + } + + /** + * @return true if this Fileset should get files from the source directory. + */ + public boolean isSourceFileset() { + return "BUILD".equals(srcLabel.getName()); + } + + /** + * @return all prerequisite labels in the FilesetEntry. + */ + public Collection<Label> getLabels() { + Set<Label> labels = new LinkedHashSet<>(); + if (files != null) { + labels.addAll(files); + } else { + labels.add(srcLabel); + } + return labels; + } + + /** + * @return the prefix that should be stripped from package-relative path names. + */ + public String getStripPrefix() { + return stripPrefix; + } + + /** + * @return null if the entry is valid, and a human-readable error message otherwise. + */ + @Nullable + public String validate() { + if (excludes != null && files != null) { + return "Cannot specify both 'files' and 'excludes' in a FilesetEntry"; + } else if (files != null && !isSourceFileset()) { + return "Cannot specify files with Fileset label '" + srcLabel + "'"; + } else if (destDir.isAbsolute()) { + return "Cannot specify absolute destdir '" + destDir + "'"; + } else if (!stripPrefix.equals(".") && files == null) { + return "If the strip prefix is not '.', files must be specified"; + } else if (new PathFragment(stripPrefix).containsUplevelReferences()) { + return "Strip prefix must not contain uplevel references"; + } else { + return null; + } + } + + @Override + public String toString() { + return String.format("FilesetEntry(srcdir=%s, destdir=%s, strip_prefix=%s, symlinks=%s, " + + "%d file(s) and %d excluded)", srcLabel, destDir, stripPrefix, symlinkBehavior, + files != null ? files.size() : 0, + excludes != null ? excludes.size() : 0); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ForStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/ForStatement.java new file mode 100644 index 0000000..34a4eea --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/ForStatement.java
@@ -0,0 +1,97 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * Syntax node for a for loop statement. + */ +public final class ForStatement extends Statement { + + private final Ident variable; + private final Expression collection; + private final ImmutableList<Statement> block; + + /** + * Constructs a for loop statement. + */ + ForStatement(Ident variable, Expression collection, List<Statement> block) { + this.variable = Preconditions.checkNotNull(variable); + this.collection = Preconditions.checkNotNull(collection); + this.block = ImmutableList.copyOf(block); + } + + public Ident getVariable() { + return variable; + } + + public Expression getCollection() { + return collection; + } + + public ImmutableList<Statement> block() { + return block; + } + + @Override + public String toString() { + // TODO(bazel-team): if we want to print the complete statement, the function + // needs an extra argument to specify indentation level. + return "for " + variable + " in " + collection + ": ...\n"; + } + + @Override + void exec(Environment env) throws EvalException, InterruptedException { + Object o = collection.eval(env); + Iterable<?> col = EvalUtils.toIterable(o, getLocation()); + + int i = 0; + for (Object it : ImmutableList.copyOf(col)) { + env.update(variable.getName(), it); + for (Statement stmt : block) { + stmt.exec(env); + } + i++; + } + // TODO(bazel-team): This should not happen if every collection is immutable. + if (i != EvalUtils.size(col)) { + throw new EvalException(getLocation(), + String.format("Cannot modify '%s' during during iteration.", collection.toString())); + } + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + void validate(ValidationEnvironment env) throws EvalException { + if (env.isTopLevel()) { + throw new EvalException(getLocation(), + "'For' is not allowed as a the top level statement"); + } + // TODO(bazel-team): validate variable. Maybe make it temporarily readonly. + SkylarkType type = collection.validate(env); + env.checkIterable(type, getLocation()); + env.update(variable.getName(), SkylarkType.UNKNOWN, getLocation()); + for (Statement stmt : block) { + stmt.validate(env); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/FuncallExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/FuncallExpression.java new file mode 100644 index 0000000..e24d97f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/FuncallExpression.java
@@ -0,0 +1,550 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.EvalException.EvalExceptionWithJavaCause; +import com.google.devtools.build.lib.util.StringUtilities; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +/** + * Syntax node for a function call expression. + */ +public final class FuncallExpression extends Expression { + + private static enum ArgConversion { + FROM_SKYLARK, + TO_SKYLARK, + NO_CONVERSION + } + + /** + * A value class to store Methods with their corresponding SkylarkCallable annotations. + * This is needed because the annotation is sometimes in a superclass. + */ + public static final class MethodDescriptor { + private final Method method; + private final SkylarkCallable annotation; + + private MethodDescriptor(Method method, SkylarkCallable annotation) { + this.method = method; + this.annotation = annotation; + } + + Method getMethod() { + return method; + } + + /** + * Returns the SkylarkCallable annotation corresponding to this method. + */ + public SkylarkCallable getAnnotation() { + return annotation; + } + } + + private static final LoadingCache<Class<?>, Map<String, List<MethodDescriptor>>> methodCache = + CacheBuilder.newBuilder() + .initialCapacity(10) + .maximumSize(100) + .build(new CacheLoader<Class<?>, Map<String, List<MethodDescriptor>>>() { + + @Override + public Map<String, List<MethodDescriptor>> load(Class<?> key) throws Exception { + Map<String, List<MethodDescriptor>> methodMap = new HashMap<>(); + for (Method method : key.getMethods()) { + // Synthetic methods lead to false multiple matches + if (method.isSynthetic()) { + continue; + } + SkylarkCallable callable = getAnnotationFromParentClass( + method.getDeclaringClass(), method); + if (callable == null) { + continue; + } + String name = callable.name(); + if (name.isEmpty()) { + name = StringUtilities.toPythonStyleFunctionName(method.getName()); + } + String signature = name + "#" + method.getParameterTypes().length; + if (methodMap.containsKey(signature)) { + methodMap.get(signature).add(new MethodDescriptor(method, callable)); + } else { + methodMap.put(signature, Lists.newArrayList(new MethodDescriptor(method, callable))); + } + } + return ImmutableMap.copyOf(methodMap); + } + }); + + /** + * Returns a map of methods and corresponding SkylarkCallable annotations + * of the methods of the classObj class reachable from Skylark. + */ + public static ImmutableMap<Method, SkylarkCallable> collectSkylarkMethodsWithAnnotation( + Class<?> classObj) { + ImmutableMap.Builder<Method, SkylarkCallable> methodMap = ImmutableMap.builder(); + for (Method method : classObj.getMethods()) { + // Synthetic methods lead to false multiple matches + if (!method.isSynthetic()) { + SkylarkCallable annotation = getAnnotationFromParentClass(classObj, method); + if (annotation != null) { + methodMap.put(method, annotation); + } + } + } + return methodMap.build(); + } + + private static SkylarkCallable getAnnotationFromParentClass(Class<?> classObj, Method method) { + boolean keepLooking = false; + try { + Method superMethod = classObj.getMethod(method.getName(), method.getParameterTypes()); + if (classObj.isAnnotationPresent(SkylarkModule.class) + && superMethod.isAnnotationPresent(SkylarkCallable.class)) { + return superMethod.getAnnotation(SkylarkCallable.class); + } else { + keepLooking = true; + } + } catch (NoSuchMethodException e) { + // The class might not have the specified method, so an exceptions is OK. + keepLooking = true; + } + if (keepLooking) { + if (classObj.getSuperclass() != null) { + SkylarkCallable annotation = getAnnotationFromParentClass(classObj.getSuperclass(), method); + if (annotation != null) { + return annotation; + } + } + for (Class<?> interfaceObj : classObj.getInterfaces()) { + SkylarkCallable annotation = getAnnotationFromParentClass(interfaceObj, method); + if (annotation != null) { + return annotation; + } + } + } + return null; + } + + /** + * An exception class to handle exceptions in direct Java API calls. + */ + public static final class FuncallException extends Exception { + + public FuncallException(String msg) { + super(msg); + } + } + + private final Expression obj; + + private final Ident func; + + private final List<Argument> args; + + private final int numPositionalArgs; + + /** + * Note: the grammar definition restricts the function value in a function + * call expression to be a global identifier; however, the representation of + * values in the interpreter is flexible enough to allow functions to be + * arbitrary expressions. In any case, the "func" expression is always + * evaluated, so functions and variables share a common namespace. + */ + public FuncallExpression(Expression obj, Ident func, + List<Argument> args) { + for (Argument arg : args) { + Preconditions.checkArgument(arg.hasValue()); + } + this.obj = obj; + this.func = func; + this.args = args; + this.numPositionalArgs = countPositionalArguments(); + } + + /** + * Note: the grammar definition restricts the function value in a function + * call expression to be a global identifier; however, the representation of + * values in the interpreter is flexible enough to allow functions to be + * arbitrary expressions. In any case, the "func" expression is always + * evaluated, so functions and variables share a common namespace. + */ + public FuncallExpression(Ident func, List<Argument> args) { + this(null, func, args); + } + + /** + * Returns the number of positional arguments. + */ + private int countPositionalArguments() { + int num = 0; + for (Argument arg : args) { + if (arg.isPositional()) { + num++; + } + } + return num; + } + + /** + * Returns the function expression. + */ + public Ident getFunction() { + return func; + } + + /** + * Returns the object the function called on. + * It's null if the function is not called on an object. + */ + public Expression getObject() { + return obj; + } + + /** + * Returns an (immutable, ordered) list of function arguments. The first n are + * positional and the remaining ones are keyword args, where n = + * getNumPositionalArguments(). + */ + public List<Argument> getArguments() { + return Collections.unmodifiableList(args); + } + + /** + * Returns the number of arguments which are positional; the remainder are + * keyword arguments. + */ + public int getNumPositionalArguments() { + return numPositionalArgs; + } + + @Override + public String toString() { + if (func.getName().equals("$substring")) { + return obj + "[" + args.get(0) + ":" + args.get(1) + "]"; + } + if (func.getName().equals("$index")) { + return obj + "[" + args.get(0) + "]"; + } + if (obj != null) { + return obj + "." + func + "(" + args + ")"; + } + return func + "(" + args + ")"; + } + + /** + * Returns the list of Skylark callable Methods of objClass with the given name + * and argument number. + */ + public static List<MethodDescriptor> getMethods(Class<?> objClass, String methodName, int argNum) + throws ExecutionException { + return methodCache.get(objClass).get(methodName + "#" + argNum); + } + + /** + * Returns the list of the Skylark name of all Skylark callable methods. + */ + public static List<String> getMethodNames(Class<?> objClass) + throws ExecutionException { + List<String> names = new ArrayList<>(); + for (List<MethodDescriptor> methods : methodCache.get(objClass).values()) { + for (MethodDescriptor method : methods) { + // TODO(bazel-team): store the Skylark name in the MethodDescriptor. + String name = method.annotation.name(); + if (name.isEmpty()) { + name = StringUtilities.toPythonStyleFunctionName(method.method.getName()); + } + names.add(name); + } + } + return names; + } + + static Object callMethod(MethodDescriptor methodDescriptor, String methodName, Object obj, + Object[] args, Location loc) throws EvalException, IllegalAccessException, + IllegalArgumentException, InvocationTargetException { + Method method = methodDescriptor.getMethod(); + if (obj == null && !Modifier.isStatic(method.getModifiers())) { + throw new EvalException(loc, "Method '" + methodName + "' is not static"); + } + // This happens when the interface is public but the implementation classes + // have reduced visibility. + method.setAccessible(true); + Object result = method.invoke(obj, args); + if (method.getReturnType().equals(Void.TYPE)) { + return Environment.NONE; + } + if (result == null) { + if (methodDescriptor.getAnnotation().allowReturnNones()) { + return Environment.NONE; + } else { + throw new EvalException(loc, + "Method invocation returned None, please contact Skylark developers: " + methodName + + "(" + EvalUtils.prettyPrintValues(", ", ImmutableList.copyOf(args)) + ")"); + } + } + result = SkylarkType.convertToSkylark(result, method); + if (result != null && !EvalUtils.isSkylarkImmutable(result.getClass())) { + throw new EvalException(loc, "Method '" + methodName + + "' returns a mutable object (type of " + EvalUtils.getDatatypeName(result) + ")"); + } + return result; + } + + // TODO(bazel-team): If there's exactly one usable method, this works. If there are multiple + // matching methods, it still can be a problem. Figure out how the Java compiler does it + // exactly and copy that behaviour. + // TODO(bazel-team): check if this and SkylarkBuiltInFunctions.createObject can be merged. + private Object invokeJavaMethod( + Object obj, Class<?> objClass, String methodName, List<Object> args) throws EvalException { + try { + MethodDescriptor matchingMethod = null; + List<MethodDescriptor> methods = getMethods(objClass, methodName, args.size()); + if (methods != null) { + for (MethodDescriptor method : methods) { + Class<?>[] params = method.getMethod().getParameterTypes(); + int i = 0; + boolean matching = true; + for (Class<?> param : params) { + if (!param.isAssignableFrom(args.get(i).getClass())) { + matching = false; + break; + } + i++; + } + if (matching) { + if (matchingMethod == null) { + matchingMethod = method; + } else { + throw new EvalException(func.getLocation(), + "Multiple matching methods for " + formatMethod(methodName, args) + + " in " + EvalUtils.getDataTypeNameFromClass(objClass)); + } + } + } + } + if (matchingMethod != null && !matchingMethod.getAnnotation().structField()) { + return callMethod(matchingMethod, methodName, obj, args.toArray(), getLocation()); + } else { + throw new EvalException(getLocation(), "No matching method found for " + + formatMethod(methodName, args) + " in " + + EvalUtils.getDataTypeNameFromClass(objClass)); + } + } catch (IllegalAccessException e) { + // TODO(bazel-team): Print a nice error message. Maybe the method exists + // and an argument is missing or has the wrong type. + throw new EvalException(getLocation(), "Method invocation failed: " + e); + } catch (InvocationTargetException e) { + if (e.getCause() instanceof FuncallException) { + throw new EvalException(getLocation(), e.getCause().getMessage()); + } else if (e.getCause() != null) { + throw new EvalExceptionWithJavaCause(getLocation(), e.getCause()); + } else { + // This is unlikely to happen + throw new EvalException(getLocation(), "Method invocation failed: " + e); + } + } catch (ExecutionException e) { + throw new EvalException(getLocation(), "Method invocation failed: " + e); + } + } + + private String formatMethod(String methodName, List<Object> args) { + StringBuilder sb = new StringBuilder(); + sb.append(methodName).append("("); + boolean first = true; + for (Object obj : args) { + if (!first) { + sb.append(", "); + } + sb.append(EvalUtils.getDatatypeName(obj)); + first = false; + } + return sb.append(")").toString(); + } + + /** + * Add one argument to the keyword map, raising an exception when names conflict. + */ + private void addKeywordArg(Map<String, Object> kwargs, String name, Object value) + throws EvalException { + if (kwargs.put(name, value) != null) { + throw new EvalException(getLocation(), + "duplicate keyword '" + name + "' in call to '" + func + "'"); + } + } + + /** + * Add multiple arguments to the keyword map (**kwargs). + */ + private void addKeywordArgs(Map<String, Object> kwargs, Object items) + throws EvalException { + if (!(items instanceof Map<?, ?>)) { + throw new EvalException(getLocation(), + "Argument after ** must be a dictionary, not " + EvalUtils.getDatatypeName(items)); + } + for (Map.Entry<?, ?> entry : ((Map<?, ?>) items).entrySet()) { + if (!(entry.getKey() instanceof String)) { + throw new EvalException(getLocation(), + "Keywords must be strings, not " + EvalUtils.getDatatypeName(entry.getKey())); + } + addKeywordArg(kwargs, (String) entry.getKey(), entry.getValue()); + } + } + + private void evalArguments(List<Object> posargs, Map<String, Object> kwargs, + Environment env, Function function) + throws EvalException, InterruptedException { + ArgConversion conversion = getArgConversion(function); + for (Argument arg : args) { + Object value = arg.getValue().eval(env); + if (conversion == ArgConversion.FROM_SKYLARK) { + value = SkylarkType.convertFromSkylark(value); + } else if (conversion == ArgConversion.TO_SKYLARK) { + // We try to auto convert the type if we can. + value = SkylarkType.convertToSkylark(value, getLocation()); + // We call into Skylark so we need to be sure that the caller uses the appropriate types. + SkylarkType.checkTypeAllowedInSkylark(value, getLocation()); + } + if (arg.isPositional()) { + posargs.add(value); + } else if (arg.isKwargs()) { // expand the kwargs + addKeywordArgs(kwargs, value); + } else { + addKeywordArg(kwargs, arg.getArgName(), value); + } + } + if (function instanceof UserDefinedFunction) { + // Adding the default values for a UserDefinedFunction if needed. + UserDefinedFunction func = (UserDefinedFunction) function; + if (args.size() < func.getArgs().size()) { + for (Map.Entry<String, Object> entry : func.getDefaultValues().entrySet()) { + String key = entry.getKey(); + if (func.getArgIndex(key) >= numPositionalArgs && !kwargs.containsKey(key)) { + kwargs.put(key, entry.getValue()); + } + } + } + } + } + + static boolean isNamespace(Class<?> classObject) { + return classObject.isAnnotationPresent(SkylarkModule.class) + && classObject.getAnnotation(SkylarkModule.class).namespace(); + } + + @Override + Object eval(Environment env) throws EvalException, InterruptedException { + List<Object> posargs = new ArrayList<>(); + Map<String, Object> kwargs = new LinkedHashMap<>(); + + if (obj != null) { + Object objValue = obj.eval(env); + // Strings, lists and dictionaries (maps) have functions that we want to use in MethodLibrary. + // For other classes, we can call the Java methods. + Function function = + env.getFunction(EvalUtils.getSkylarkType(objValue.getClass()), func.getName()); + if (function != null) { + if (!isNamespace(objValue.getClass())) { + posargs.add(objValue); + } + evalArguments(posargs, kwargs, env, function); + return EvalUtils.checkNotNull(this, function.call(posargs, kwargs, this, env)); + } else if (env.isSkylarkEnabled()) { + + // When calling a Java method, the name is not in the Environment, so + // evaluating 'func' would fail. For arguments we don't need to consider the default + // arguments since the Java function doesn't have any. + + evalArguments(posargs, kwargs, env, null); + if (!kwargs.isEmpty()) { + throw new EvalException(func.getLocation(), + "Keyword arguments are not allowed when calling a java method"); + } + if (objValue instanceof Class<?>) { + // Static Java method call + return invokeJavaMethod(null, (Class<?>) objValue, func.getName(), posargs); + } else { + return invokeJavaMethod(objValue, objValue.getClass(), func.getName(), posargs); + } + } else { + throw new EvalException(getLocation(), String.format( + "function '%s' is not defined on '%s'", func.getName(), + EvalUtils.getDatatypeName(objValue))); + } + } + + Object funcValue = func.eval(env); + if (!(funcValue instanceof Function)) { + throw new EvalException(getLocation(), + "'" + EvalUtils.getDatatypeName(funcValue) + + "' object is not callable"); + } + Function function = (Function) funcValue; + evalArguments(posargs, kwargs, env, function); + return EvalUtils.checkNotNull(this, function.call(posargs, kwargs, this, env)); + } + + private ArgConversion getArgConversion(Function function) { + if (function == null) { + // It means we try to call a Java function. + return ArgConversion.FROM_SKYLARK; + } + // If we call a UserDefinedFunction we call into Skylark. If we call from Skylark + // the argument conversion is invariant, but if we call from the BUILD language + // we might need an auto conversion. + return function instanceof UserDefinedFunction + ? ArgConversion.TO_SKYLARK : ArgConversion.NO_CONVERSION; + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + SkylarkType validate(ValidationEnvironment env) throws EvalException { + // TODO(bazel-team): implement semantical check. + + if (obj != null) { + // TODO(bazel-team): validate function calls on objects too. + return env.getReturnType(obj.validate(env), func.getName(), getLocation()); + } else { + // TODO(bazel-team): Imported functions are not validated properly. + if (!env.hasSymbolInEnvironment(func.getName())) { + throw new EvalException(getLocation(), + String.format("function '%s' does not exist", func.getName())); + } + return env.getReturnType(func.getName(), getLocation()); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Function.java b/src/main/java/com/google/devtools/build/lib/syntax/Function.java new file mode 100644 index 0000000..5636a95 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/Function.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import java.util.List; +import java.util.Map; + +/** + * Function values in the BUILD language. + * + * <p>Each implementation of this interface defines a function in the BUILD language. + * + */ +public interface Function { + + /** + * Implements the behavior of a call to a function with positional arguments + * "args" and keyword arguments "kwargs". The "ast" argument is provided to + * allow construction of EvalExceptions containing source information. + */ + Object call(List<Object> args, + Map<String, Object> kwargs, + FuncallExpression ast, + Environment env) + throws EvalException, InterruptedException; + + /** + * Returns the name of the function. + */ + String getName(); + + // TODO(bazel-team): implement this for MethodLibrary functions as well. + /** + * Returns the type of the object on which this function is defined or null + * if this is a global function. + */ + Class<?> getObjectType(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/FunctionDefStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/FunctionDefStatement.java new file mode 100644 index 0000000..29ed579 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/FunctionDefStatement.java
@@ -0,0 +1,97 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType; + +import java.util.Collection; + +/** + * Syntax node for a function definition. + */ +public class FunctionDefStatement extends Statement { + + private final Ident ident; + private final ImmutableList<Argument> args; + private final ImmutableList<Statement> statements; + + public FunctionDefStatement(Ident ident, Collection<Argument> args, + Collection<Statement> statements) { + for (Argument arg : args) { + Preconditions.checkArgument(arg.isNamed()); + } + this.ident = ident; + this.args = ImmutableList.copyOf(args); + this.statements = ImmutableList.copyOf(statements); + } + + @Override + void exec(Environment env) throws EvalException, InterruptedException { + ImmutableMap.Builder<String, Object> defaultValues = ImmutableMap.builder(); + for (Argument arg : args) { + if (arg.hasValue()) { + defaultValues.put(arg.getArgName(), arg.getValue().eval(env)); + } + } + env.update(ident.getName(), new UserDefinedFunction( + ident, args, defaultValues.build(), statements, (SkylarkEnvironment) env)); + } + + @Override + public String toString() { + return "def " + ident + "(" + args + "):\n"; + } + + public Ident getIdent() { + return ident; + } + + public ImmutableList<Statement> getStatements() { + return statements; + } + + public ImmutableList<Argument> getArgs() { + return args; + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + void validate(ValidationEnvironment env) throws EvalException { + SkylarkFunctionType type = SkylarkFunctionType.of(ident.getName()); + ValidationEnvironment localEnv = new ValidationEnvironment(env, type); + for (Argument i : args) { + SkylarkType argType = SkylarkType.UNKNOWN; + if (i.hasValue()) { + argType = i.getValue().validate(env); + if (argType.equals(SkylarkType.NONE)) { + argType = SkylarkType.UNKNOWN; + } + } + localEnv.update(i.getArgName(), argType, getLocation()); + } + for (Statement stmts : statements) { + stmts.validate(localEnv); + } + env.updateFunction(ident.getName(), type, getLocation()); + // Register a dummy return value with an incompatible type if there was no return statement. + type.setReturnType(SkylarkType.NONE, getLocation()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/GlobCriteria.java b/src/main/java/com/google/devtools/build/lib/syntax/GlobCriteria.java new file mode 100644 index 0000000..577bd4a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/GlobCriteria.java
@@ -0,0 +1,214 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Functions; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.Iterables; + +import java.util.List; + +import javax.annotation.Nullable; + +/** + * Either the arguments to a glob call (the include and exclude lists) or the + * contents of a fixed list that was appended to a list of glob results. + * (The latter need to be stored by {@link GlobList} in order to fully + * reproduce the inputs that created the output list.) + * + * <p>For example, the expression + * <code>glob(['*.java']) + ['x.properties']</code> + * will result in two GlobCriteria: one has include = ['*.java'], glob = true + * and the other, include = ['x.properties'], glob = false. + */ +public class GlobCriteria { + + /** + * A list of names or patterns that are included by this glob. They should + * consist of characters that are valid in labels in the BUILD language. + */ + private final ImmutableList<String> include; + + /** + * A list of names or patterns that are excluded by this glob. They should + * consist of characters that are valid in labels in the BUILD language. + */ + private final ImmutableList<String> exclude; + + /** True if the includes list was passed to glob(), false if not. */ + private final boolean glob; + + /** + * Parses criteria from its {@link #toExpression} form. + * Package-private for use by tests and GlobList. + * @throws IllegalArgumentException if the expression cannot be parsed + */ + public static GlobCriteria parse(String text) { + if (text.startsWith("glob([") && text.endsWith("])")) { + int excludeIndex = text.indexOf("], exclude=["); + if (excludeIndex == -1) { + String listText = text.substring(6, text.length() - 2); + return new GlobCriteria(parseList(listText), ImmutableList.<String>of(), true); + } else { + String listText = text.substring(6, excludeIndex); + String excludeText = text.substring(excludeIndex + 12, text.length() - 2); + return new GlobCriteria(parseList(listText), parseList(excludeText), true); + } + } else if (text.startsWith("[") && text.endsWith("]")) { + String listText = text.substring(1, text.length() - 1); + return new GlobCriteria(parseList(listText), ImmutableList.<String>of(), false); + } else { + throw new IllegalArgumentException( + "unrecognized format (not from toExpression?): " + text); + } + } + + /** + * Constructs a copy of a given glob critera object, with additional exclude patterns added. + * + * @param base a glob criteria object to copy. Must be an actual glob + * @param excludes a list of pattern strings indicating new excludes to provide + * @return a new glob criteria object which contains the same parameters as {@code base}, with + * the additional patterns in {@code excludes} added. + * @throws IllegalArgumentException if {@code base} is not a glob + */ + public static GlobCriteria createWithAdditionalExcludes(GlobCriteria base, + List<String> excludes) { + Preconditions.checkArgument(base.isGlob()); + return fromGlobCall(base.include, + ImmutableList.copyOf(Iterables.concat(base.exclude, excludes))); + } + + /** + * Constructs a copy of a fixed list, converted to Strings. + */ + public static GlobCriteria fromList(Iterable<?> list) { + Iterable<String> strings = Iterables.transform(list, Functions.toStringFunction()); + return new GlobCriteria(ImmutableList.copyOf(strings), ImmutableList.<String>of(), false); + } + + /** + * Constructs a glob call with include and exclude list. + * + * @param include list of included patterns + * @param exclude list of excluded patterns + */ + public static GlobCriteria fromGlobCall( + ImmutableList<String> include, ImmutableList<String> exclude) { + return new GlobCriteria(include, exclude, true); + } + + /** + * Constructs a glob call with include and exclude list. + */ + private GlobCriteria(ImmutableList<String> include, ImmutableList<String> exclude, boolean glob) { + this.include = include; + this.exclude = exclude; + this.glob = glob; + } + + /** + * Returns the patterns that were included in this {@code glob()} call. + */ + public ImmutableList<String> getIncludePatterns() { + return include; + } + + /** + * Returns the patterns that were excluded in this {@code glob()} call. + */ + public ImmutableList<String> getExcludePatterns() { + return exclude; + } + + /** + * Returns true if the include list was passed to {@code glob()}, false + * if it was a fixed list. If this returns false, the exclude list will + * always be empty. + */ + public boolean isGlob() { + return glob; + } + + /** + * Returns a String that represents this glob as a BUILD expression. + * For example, <code>glob(['abc', 'def'], exclude=['uvw', 'xyz'])</code> + * or <code>['foo', 'bar', 'baz']</code>. + */ + public String toExpression() { + StringBuilder sb = new StringBuilder(); + if (glob) { + sb.append("glob("); + } + sb.append('['); + appendList(sb, include); + if (!exclude.isEmpty()) { + sb.append("], exclude=["); + appendList(sb, exclude); + } + sb.append(']'); + if (glob) { + sb.append(')'); + } + return sb.toString(); + } + + @Override + public String toString() { + return toExpression(); + } + + /** + * Takes a list of Strings, quotes them in single quotes, and appends them to + * a StringBuilder separated by a comma and space. This can be parsed back + * out by {@link #parseList}. + */ + private static void appendList(StringBuilder sb, List<String> list) { + boolean first = true; + for (String content : list) { + if (!first) { + sb.append(", "); + } + sb.append('\'').append(content).append('\''); + first = false; + } + } + + /** + * Takes a String in the format created by {@link #appendList} and returns + * the original Strings. A null String (which may be returned when Pattern + * does not find a match) or the String "" (which will be captured in "[]") + * will result in an empty list. + */ + private static ImmutableList<String> parseList(@Nullable String text) { + if (text == null) { + return ImmutableList.of(); + } + Iterable<String> split = Splitter.on(", ").split(text); + Builder<String> listBuilder = ImmutableList.builder(); + for (String element : split) { + if (!element.isEmpty()) { + if ((element.length() < 2) || !element.startsWith("'") || !element.endsWith("'")) { + throw new IllegalArgumentException("expected a filename or pattern in quotes: " + text); + } + listBuilder.add(element.substring(1, element.length() - 1)); + } + } + return listBuilder.build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/GlobList.java b/src/main/java/com/google/devtools/build/lib/syntax/GlobList.java new file mode 100644 index 0000000..82afd01 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/GlobList.java
@@ -0,0 +1,122 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.collect.ForwardingList; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.Iterables; + +import java.util.ArrayList; +import java.util.List; + +/** + * Glob matches and information about glob patterns, which are useful to + * ide_build_info. Its implementation of the List interface is as an immutable + * list of the matching files. Glob criteria can be retrieved through + * {@link #getCriteria}. + * + * @param <E> the element this List contains (generally either String or Label) + */ +public class GlobList<E> extends ForwardingList<E> { + + /** Include/exclude criteria. */ + private final ImmutableList<GlobCriteria> criteria; + + /** Matching files (usually either String or Label). */ + private final ImmutableList<E> matches; + + /** + * Constructs a list with {@code glob()} call results. + * + * @param includes the patterns that the glob includes + * @param excludes the patterns that the glob excludes + * @param matches the filenames that matched the includes/excludes criteria + */ + public static <T> GlobList<T> captureResults(List<String> includes, + List<String> excludes, List<T> matches) { + GlobCriteria criteria = GlobCriteria.fromGlobCall( + ImmutableList.copyOf(includes), ImmutableList.copyOf(excludes)); + return new GlobList<>(ImmutableList.of(criteria), matches); + } + + /** + * Parses a GlobInfo from its {@link #toExpression} representation. + */ + public static GlobList<String> parse(String text) { + List<GlobCriteria> criteria = new ArrayList<>(); + Iterable<String> globs = Splitter.on(" + ").split(text); + for (String glob : globs) { + criteria.add(GlobCriteria.parse(glob)); + } + return new GlobList<>(criteria, ImmutableList.<String>of()); + } + + /** + * Concatenates two lists into a new GlobList. If either of the lists is a + * GlobList, its GlobCriteria are preserved. Otherwise a simple GlobCriteria + * is created to represent the fixed list. + */ + public static <T> GlobList<T> concat( + List<? extends T> list1, List<? extends T> list2) { + // we add the list to both includes and matches, preserving order + Builder<GlobCriteria> criteriaBuilder = ImmutableList.<GlobCriteria>builder(); + if (list1 instanceof GlobList<?>) { + criteriaBuilder.addAll(((GlobList<?>) list1).criteria); + } else { + criteriaBuilder.add(GlobCriteria.fromList(list1)); + } + if (list2 instanceof GlobList<?>) { + criteriaBuilder.addAll(((GlobList<?>) list2).criteria); + } else { + criteriaBuilder.add(GlobCriteria.fromList(list2)); + } + List<T> matches = ImmutableList.copyOf(Iterables.concat(list1, list2)); + return new GlobList<>(criteriaBuilder.build(), matches); + } + + /** + * Constructs a list with given criteria and matches. + */ + public GlobList(List<GlobCriteria> criteria, List<E> matches) { + Preconditions.checkNotNull(criteria); + Preconditions.checkNotNull(matches); + this.criteria = ImmutableList.copyOf(criteria); + this.matches = ImmutableList.copyOf(matches); + } + + /** + * Returns the criteria used to create this list, from which the + * includes/excludes can be retrieved. + */ + public ImmutableList<GlobCriteria> getCriteria() { + return criteria; + } + + /** + * Returns a String that represents this glob list as a BUILD expression. + */ + public String toExpression() { + return Joiner.on(" + ").join(criteria); + } + + @Override + protected ImmutableList<E> delegate() { + return matches; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Ident.java b/src/main/java/com/google/devtools/build/lib/syntax/Ident.java new file mode 100644 index 0000000..86bd458 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/Ident.java
@@ -0,0 +1,66 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +/** + * Syntax node for an identifier. + */ +public final class Ident extends Expression { + + private final String name; + + public Ident(String name) { + this.name = name; + } + + /** + * Returns the name of the Ident. + */ + public String getName() { + return name; + } + + @Override + public String toString() { + return name; + } + + @Override + Object eval(Environment env) throws EvalException { + try { + return env.lookup(name); + } catch (Environment.NoSuchVariableException e) { + if (name.equals("$error$")) { + throw new EvalException(getLocation(), "contains syntax error(s)", true); + } else { + throw new EvalException(getLocation(), "name '" + name + "' is not defined"); + } + } + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + SkylarkType validate(ValidationEnvironment env) throws EvalException { + if (env.hasSymbolInEnvironment(name)) { + return env.getVartype(name); + } else { + throw new EvalException(getLocation(), "name '" + name + "' is not defined"); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/IfStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/IfStatement.java new file mode 100644 index 0000000..3877a9c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/IfStatement.java
@@ -0,0 +1,138 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +// TODO(bazel-team): maybe we should get rid of the ConditionalStatements and +// create a chain of if-else statements for elif-s. +/** + * Syntax node for an if/else statement. + */ +public final class IfStatement extends Statement { + + /** + * Syntax node for an [el]if statement. + */ + static final class ConditionalStatements extends Statement { + + private final Expression condition; + private final ImmutableList<Statement> stmts; + + public ConditionalStatements(Expression condition, List<Statement> stmts) { + this.condition = Preconditions.checkNotNull(condition); + this.stmts = ImmutableList.copyOf(stmts); + } + + @Override + void exec(Environment env) throws EvalException, InterruptedException { + for (Statement stmt : stmts) { + stmt.exec(env); + } + } + + @Override + public String toString() { + // TODO(bazel-team): see TODO in the outer class + return "[el]if " + condition + ": ...\n"; + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + Expression getCondition() { + return condition; + } + + ImmutableList<Statement> getStmts() { + return stmts; + } + + @Override + void validate(ValidationEnvironment env) throws EvalException { + // EvalUtils.toBoolean() evaluates everything so we don't need type check here. + condition.validate(env); + validateStmts(env, stmts); + } + } + + private final ImmutableList<ConditionalStatements> thenBlocks; + private final ImmutableList<Statement> elseBlock; + + /** + * Constructs a if-elif-else statement. The else part is mandatory, but the list may be empty. + * ThenBlocks has to have at least one element. + */ + IfStatement(List<ConditionalStatements> thenBlocks, List<Statement> elseBlock) { + Preconditions.checkArgument(thenBlocks.size() > 0); + this.thenBlocks = ImmutableList.copyOf(thenBlocks); + this.elseBlock = ImmutableList.copyOf(elseBlock); + } + + public ImmutableList<ConditionalStatements> getThenBlocks() { + return thenBlocks; + } + + public ImmutableList<Statement> getElseBlock() { + return elseBlock; + } + + @Override + public String toString() { + // TODO(bazel-team): if we want to print the complete statement, the function + // needs an extra argument to specify indentation level. + return "if : ...\n"; + } + + @Override + void exec(Environment env) throws EvalException, InterruptedException { + for (ConditionalStatements stmt : thenBlocks) { + if (EvalUtils.toBoolean(stmt.getCondition().eval(env))) { + stmt.exec(env); + return; + } + } + for (Statement stmt : elseBlock) { + stmt.exec(env); + } + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + void validate(ValidationEnvironment env) throws EvalException { + env.startTemporarilyDisableReadonlyCheckSession(); + for (ConditionalStatements stmts : thenBlocks) { + stmts.validate(env); + } + validateStmts(env, elseBlock); + env.finishTemporarilyDisableReadonlyCheckSession(); + } + + private static void validateStmts(ValidationEnvironment env, List<Statement> stmts) + throws EvalException { + for (Statement stmt : stmts) { + stmt.validate(env); + } + env.finishTemporarilyDisableReadonlyCheckBranch(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/IntegerLiteral.java b/src/main/java/com/google/devtools/build/lib/syntax/IntegerLiteral.java new file mode 100644 index 0000000..e6852e6b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/IntegerLiteral.java
@@ -0,0 +1,34 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +/** + * Syntax node for an integer literal. + */ +public final class IntegerLiteral extends Literal<Integer> { + + public IntegerLiteral(Integer value) { + super(value); + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + SkylarkType validate(ValidationEnvironment env) throws EvalException { + return SkylarkType.INT; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Label.java b/src/main/java/com/google/devtools/build/lib/syntax/Label.java new file mode 100644 index 0000000..89db4de --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/Label.java
@@ -0,0 +1,412 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ComparisonChain; +import com.google.devtools.build.lib.cmdline.LabelValidator; +import com.google.devtools.build.lib.cmdline.LabelValidator.BadLabelException; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.util.StringCanonicalizer; +import com.google.devtools.build.lib.util.StringUtilities; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.InvalidObjectException; +import java.io.ObjectInputStream; +import java.io.Serializable; + +/** + * A class to identify a BUILD target. All targets belong to exactly one package. + * The name of a target is called its label. A typical label looks like this: + * //dir1/dir2:target_name where 'dir1/dir2' identifies the package containing a BUILD file, + * and 'target_name' identifies the target within the package. + * + * <p>Parsing is robust against bad input, for example, from the command line. + */ +@SkylarkModule(name = "Label", doc = "A BUILD target identifier.") +@Immutable @ThreadSafe +public final class Label implements Comparable<Label>, Serializable { + + /** + * Thrown by the parsing methods to indicate a bad label. + */ + public static class SyntaxException extends Exception { + public SyntaxException(String message) { + super(message); + } + } + + /** + * Factory for Labels from absolute string form, possibly including a repository name prefix. For + * example: + * <pre> + * //foo/bar + * {@literal @}foo//bar + * {@literal @}foo//bar:baz + * </pre> + */ + public static Label parseRepositoryLabel(String absName) throws SyntaxException { + String repo = PackageIdentifier.DEFAULT_REPOSITORY; + int packageStartPos = absName.indexOf("//"); + if (packageStartPos > 0) { + repo = absName.substring(0, packageStartPos); + absName = absName.substring(packageStartPos); + } + try { + LabelValidator.PackageAndTarget labelParts = LabelValidator.parseAbsoluteLabel(absName); + return new Label(new PackageIdentifier(repo, new PathFragment(labelParts.getPackageName())), + labelParts.getTargetName()); + } catch (BadLabelException e) { + throw new SyntaxException(e.getMessage()); + } + } + + /** + * Factory for Labels from absolute string form. e.g. + * <pre> + * //foo/bar + * //foo/bar:quux + * </pre> + */ + public static Label parseAbsolute(String absName) throws SyntaxException { + try { + LabelValidator.PackageAndTarget labelParts = LabelValidator.parseAbsoluteLabel(absName); + return create(labelParts.getPackageName(), labelParts.getTargetName()); + } catch (BadLabelException e) { + throw new SyntaxException(e.getMessage()); + } + } + + /** + * Alternate factory method for Labels from absolute strings. This is a convenience method for + * cases when a Label needs to be initialized statically, so the declared exception is + * inconvenient. + * + * <p>Do not use this when the argument is not hard-wired. + */ + public static Label parseAbsoluteUnchecked(String absName) { + try { + return parseAbsolute(absName); + } catch (SyntaxException e) { + throw new IllegalArgumentException(e); + } + } + + /** + * Factory for Labels from separate components. + * + * @param packageName The name of the package. The package name does + * <b>not</b> include {@code //}. Must be valid according to + * {@link LabelValidator#validatePackageName}. + * @param targetName The name of the target within the package. Must be + * valid according to {@link LabelValidator#validateTargetName}. + * @throws SyntaxException if either of the arguments was invalid. + */ + public static Label create(String packageName, String targetName) throws SyntaxException { + return new Label(packageName, targetName); + } + + /** + * Similar factory to above, but takes a package identifier to allow external repository labels + * to be created. + */ + public static Label create(PackageIdentifier packageId, String targetName) + throws SyntaxException { + return new Label(packageId, targetName); + } + + /** + * Resolves a relative label using a workspace-relative path to the current working directory. The + * method handles these cases: + * <ul> + * <li>The label is absolute. + * <li>The label starts with a colon. + * <li>The label consists of a relative path, a colon, and a local part. + * <li>The label consists only of a local part. + * </ul> + * + * <p>Note that this method does not support any of the special syntactic constructs otherwise + * supported on the command line, like ":all", "/...", and so on. + * + * <p>It would be cleaner to use the TargetPatternEvaluator for this resolution, but that is not + * possible, because it is sometimes necessary to resolve a relative label before the package path + * is setup; in particular, before the tools/defaults package is created. + * + * @throws SyntaxException if the resulting label is not valid + */ + public static Label parseCommandLineLabel(String label, PathFragment workspaceRelativePath) + throws SyntaxException { + Preconditions.checkArgument(!workspaceRelativePath.isAbsolute()); + if (label.startsWith("//")) { + return parseAbsolute(label); + } + int index = label.indexOf(':'); + if (index < 0) { + index = 0; + label = ":" + label; + } + PathFragment path = workspaceRelativePath.getRelative(label.substring(0, index)); + // Use the String, String constructor, to make sure that the package name goes through the + // validity check. + return new Label(path.getPathString(), label.substring(index + 1)); + } + + /** + * Validates the given target name and returns a canonical String instance if it is valid. + * Otherwise it throws a SyntaxException. + */ + private static String canonicalizeTargetName(String name) throws SyntaxException { + String error = LabelValidator.validateTargetName(name); + if (error != null) { + error = "invalid target name '" + StringUtilities.sanitizeControlChars(name) + "': " + error; + throw new SyntaxException(error); + } + + // TODO(bazel-team): This should be an error, but we can't make it one for legacy reasons. + if (name.endsWith("/.")) { + name = name.substring(0, name.length() - 2); + } + + return StringCanonicalizer.intern(name); + } + + /** + * Validates the given package name and returns a canonical PathFragment instance if it is valid. + * Otherwise it throws a SyntaxException. + */ + private static PathFragment validate(String packageName, String name) throws SyntaxException { + String error = LabelValidator.validatePackageName(packageName); + if (error != null) { + error = "invalid package name '" + packageName + "': " + error; + // This check is just for a more helpful error message + // i.e. valid target name, invalid package name, colon-free label form + // used => probably they meant "//foo:bar.c" not "//foo/bar.c". + if (packageName.endsWith("/" + name)) { + error += " (perhaps you meant \":" + name + "\"?)"; + } + throw new SyntaxException(error); + } + return new PathFragment(packageName); + } + + /** The name and repository of the package. */ + private final PackageIdentifier packageIdentifier; + + /** The name of the target within the package. Canonical. */ + private final String name; + + /** + * Constructor from a package name, target name. Both are checked for validity + * and a SyntaxException is thrown if either is invalid. + * TODO(bazel-team): move the validation to {@link PackageIdentifier}. Unfortunately, there are a + * bazillion tests that use invalid package names (taking advantage of the fact that calling + * Label(PathFragment, String) doesn't validate the package name). + */ + private Label(String packageName, String name) throws SyntaxException { + this(validate(packageName, name), name); + } + + /** + * Constructor from canonical valid package name and a target name. The target + * name is checked for validity and a SyntaxException is throw if it isn't. + */ + private Label(PathFragment packageName, String name) throws SyntaxException { + this(PackageIdentifier.createInDefaultRepo(packageName), name); + } + + private Label(PackageIdentifier packageIdentifier, String name) + throws SyntaxException { + Preconditions.checkNotNull(packageIdentifier); + Preconditions.checkNotNull(name); + + try { + this.packageIdentifier = packageIdentifier; + this.name = canonicalizeTargetName(name); + } catch (SyntaxException e) { + // This check is just for a more helpful error message + // i.e. valid target name, invalid package name, colon-free label form + // used => probably they meant "//foo:bar.c" not "//foo/bar.c". + if (packageIdentifier.getPackageFragment().getPathString().endsWith("/" + name)) { + throw new SyntaxException(e.getMessage() + " (perhaps you meant \":" + name + "\"?)"); + } + throw e; + } + } + + private Object writeReplace() { + return new LabelSerializationProxy(toString()); + } + + private void readObject(ObjectInputStream stream) throws InvalidObjectException { + throw new InvalidObjectException("Serialization is allowed only by proxy"); + } + + public PackageIdentifier getPackageIdentifier() { + return packageIdentifier; + } + + /** + * Returns the name of the package in which this rule was declared (e.g. {@code + * //file/base:fileutils_test} returns {@code file/base}). + */ + @SkylarkCallable(name = "package", structField = true, + doc = "The package part of this label. " + + "For instance:<br>" + + "<pre class=language-python>label(\"//pkg/foo:abc\").package == \"pkg/foo\"</pre>") + public String getPackageName() { + return packageIdentifier.getPackageFragment().getPathString(); + } + + /** + * Returns the path fragment of the package in which this rule was declared (e.g. {@code + * //file/base:fileutils_test} returns {@code file/base}). + */ + public PathFragment getPackageFragment() { + return packageIdentifier.getPackageFragment(); + } + + public static final com.google.common.base.Function<Label, PathFragment> PACKAGE_FRAGMENT = + new com.google.common.base.Function<Label, PathFragment>() { + @Override + public PathFragment apply(Label label) { + return label.getPackageFragment(); + } + }; + + /** + * Returns the label as a path fragment, using the package and the label name. + */ + public PathFragment toPathFragment() { + return packageIdentifier.getPackageFragment().getRelative(name); + } + + /** + * Returns the name by which this rule was declared (e.g. {@code //foo/bar:baz} + * returns {@code baz}). + */ + @SkylarkCallable(name = "name", structField = true, + doc = "The name of this label within the package. " + + "For instance:<br>" + + "<pre class=language-python>label(\"//pkg/foo:abc\").name == \"abc\"</pre>") + public String getName() { + return name; + } + + /** + * Renders this label in canonical form. + * + * <p>invariant: {@code parseAbsolute(x.toString()).equals(x)} + */ + @Override + public String toString() { + return packageIdentifier.getRepository() + "//" + packageIdentifier.getPackageFragment() + + ":" + name; + } + + /** + * Renders this label in shorthand form. + * + * <p>Labels with canonical form {@code //foo/bar:bar} have the shorthand form {@code //foo/bar}. + * All other labels have identical shorthand and canonical forms. + */ + public String toShorthandString() { + return packageIdentifier.getRepository() + (getPackageFragment().getBaseName().equals(name) + ? "//" + getPackageFragment() + : toString()); + } + + /** + * Returns a label in the same package as this label with the given target name. + * + * @throws SyntaxException if {@code targetName} is not a valid target name + */ + public Label getLocalTargetLabel(String targetName) throws SyntaxException { + return new Label(packageIdentifier, targetName); + } + + /** + * Resolves a relative or absolute label name. If given name is absolute, then this method calls + * {@link #parseAbsolute}. Otherwise, it calls {@link #getLocalTargetLabel}. + * + * <p>For example: + * {@code :quux} relative to {@code //foo/bar:baz} is {@code //foo/bar:quux}; + * {@code //wiz:quux} relative to {@code //foo/bar:baz} is {@code //wiz:quux}. + * + * @param relName the relative label name; must be non-empty. + */ + @SkylarkCallable(name = "relative", doc = + "Resolves a relative or absolute label name.<br>" + + "For example:<br><ul>" + + "<li><code>:quux</code> relative to <code>//foo/bar:baz</code> is " + + "<code>//foo/bar:quux</code></li>" + + "<li><code>//wiz:quux</code> relative to <code>//foo/bar:baz</code> is " + + "<code>//wiz:quux</code></li></ul>") + public Label getRelative(String relName) throws SyntaxException { + if (relName.length() == 0) { + throw new SyntaxException("empty package-relative label"); + } + if (relName.startsWith("//")) { + return parseAbsolute(relName); + } else if (relName.equals(":")) { + throw new SyntaxException("':' is not a valid package-relative label"); + } else if (relName.charAt(0) == ':') { + return getLocalTargetLabel(relName.substring(1)); + } else { + return getLocalTargetLabel(relName); + } + } + + @Override + public int hashCode() { + return name.hashCode() ^ packageIdentifier.hashCode(); + } + + /** + * Two labels are equal iff both their name and their package name are equal. + */ + @Override + public boolean equals(Object other) { + if (!(other instanceof Label)) { + return false; + } + Label otherLabel = (Label) other; + return name.equals(otherLabel.name) // least likely one first + && packageIdentifier.equals(otherLabel.packageIdentifier); + } + + /** + * Defines the order between labels. + * + * <p>Labels are ordered primarily by package name and secondarily by target name. Both components + * are ordered lexicographically. Thus {@code //a:b/c} comes before {@code //a/b:a}, i.e. the + * position of the colon is significant to the order. + */ + @Override + public int compareTo(Label other) { + return ComparisonChain.start() + .compare(packageIdentifier, other.packageIdentifier) + .compare(name, other.name) + .result(); + } + + /** + * Returns a suitable string for the user-friendly representation of the Label. Works even if the + * argument is null. + */ + public static String print(Label label) { + return label == null ? "(unknown)" : label.toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/LabelSerializationProxy.java b/src/main/java/com/google/devtools/build/lib/syntax/LabelSerializationProxy.java new file mode 100644 index 0000000..5b4556a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/LabelSerializationProxy.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectOutput; + +/** + * A serialization proxy for {@code Label}. + */ +public class LabelSerializationProxy implements Externalizable { + + private String labelString; + + public LabelSerializationProxy(String labelString) { + this.labelString = labelString; + } + + // For deserialization machinery. + public LabelSerializationProxy() { + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + // Manual serialization gives us about a 30% reduction in size. + out.writeUTF(labelString); + } + + @Override + public void readExternal(java.io.ObjectInput in) throws IOException { + this.labelString = in.readUTF(); + } + + private Object readResolve() { + return Label.parseAbsoluteUnchecked(labelString); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java new file mode 100644 index 0000000..fc12c66 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
@@ -0,0 +1,803 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Stack; + +/** + * A tokenizer for the BUILD language. + * <p> + * See: <a href="https://docs.python.org/2/reference/lexical_analysis.html"/> + * for some details. + * <p> + * Since BUILD files are small, we just tokenize the entire file a-priori + * instead of interleaving scanning with parsing. + */ +public final class Lexer { + + private final EventHandler eventHandler; + + // Input buffer and position + private char[] buffer; + private int pos; + + /** + * The part of the location information that is common to all LexerLocation + * instances created by this Lexer. Factored into a separate object so that + * many Locations instances can share the same information as compactly as + * possible, without closing over a Lexer instance. + */ + private static class LocationInfo { + final LineNumberTable lineNumberTable; + final Path filename; + LocationInfo(Path filename, LineNumberTable lineNumberTable) { + this.filename = filename; + this.lineNumberTable = lineNumberTable; + } + } + + private final LocationInfo locationInfo; + + // The stack of enclosing indentation levels; always contains '0' at the + // bottom. + private final Stack<Integer> indentStack = new Stack<>(); + + private final List<Token> tokens = new ArrayList<>(); + + // The number of unclosed open-parens ("(", '{', '[') at the current point in + // the stream. Whitespace is handled differently when this is nonzero. + private int openParenStackDepth = 0; + + private boolean containsErrors; + + private boolean parsePython; + + /** + * Constructs a lexer which tokenizes the contents of the specified + * InputBuffer. Any errors during lexing are reported on "handler". + */ + public Lexer(ParserInputSource input, EventHandler eventHandler, boolean parsePython) { + this.buffer = input.getContent(); + this.pos = 0; + this.parsePython = parsePython; + this.eventHandler = eventHandler; + this.locationInfo = new LocationInfo(input.getPath(), + LineNumberTable.create(buffer, input.getPath())); + + indentStack.push(0); + tokenize(); + } + + public Lexer(ParserInputSource input, EventHandler eventHandler) { + this(input, eventHandler, false); + } + + /** + * Returns the filename from which the lexer's input came. Returns a dummy + * value if the input came from a string. + */ + public Path getFilename() { + return locationInfo.filename; + } + + /** + * Returns true if there were errors during scanning of this input file or + * string. The Lexer may attempt to recover from errors, but clients should + * not rely on the results of scanning if this flag is set. + */ + public boolean containsErrors() { + return containsErrors; + } + + /** + * Returns the (mutable) list of tokens generated by the Lexer. + */ + public List<Token> getTokens() { + return tokens; + } + + private void popParen() { + if (openParenStackDepth == 0) { + error("indentation error"); + } else { + openParenStackDepth--; + } + } + + private void error(String message) { + error(message, pos - 1, pos - 1); + } + + private void error(String message, int start, int end) { + this.containsErrors = true; + eventHandler.handle(Event.error(createLocation(start, end), message)); + } + + Location createLocation(int start, int end) { + return new LexerLocation(locationInfo, start, end); + } + + // Don't use an inner class as we don't want to close over the Lexer, only + // the LocationInfo. + @Immutable + private static final class LexerLocation extends Location { + + private final LineNumberTable lineNumberTable; + + LexerLocation(LocationInfo locationInfo, int start, int end) { + super(start, end); + this.lineNumberTable = locationInfo.lineNumberTable; + } + + @Override + public PathFragment getPath() { + return lineNumberTable.getPath(getStartOffset()).asFragment(); + } + + @Override + public LineAndColumn getStartLineAndColumn() { + return lineNumberTable.getLineAndColumn(getStartOffset()); + } + + @Override + public LineAndColumn getEndLineAndColumn() { + return lineNumberTable.getLineAndColumn(getEndOffset()); + } + } + + /** invariant: symbol positions are half-open intervals. */ + private void addToken(Token s) { + tokens.add(s); + } + + /** + * Parses an end-of-line sequence, handling statement indentation correctly. + * + * UNIX newlines are assumed (LF). Carriage returns are always ignored. + * + * ON ENTRY: 'pos' is the index of the char after '\n'. + * ON EXIT: 'pos' is the index of the next non-space char after '\n'. + */ + private void newline() { + if (openParenStackDepth > 0) { + newlineInsideExpression(); // in an expression: ignore space + } else { + newlineOutsideExpression(); // generate NEWLINE/INDENT/OUTDENT tokens + } + } + + private void newlineInsideExpression() { + while (pos < buffer.length) { + switch (buffer[pos]) { + case ' ': case '\t': case '\r': + pos++; + break; + default: + return; + } + } + } + + private void newlineOutsideExpression() { + if (pos > 1) { // skip over newline at start of file + addToken(new Token(TokenKind.NEWLINE, pos - 1, pos)); + } + + // we're in a stmt: suck up space at beginning of next line + int indentLen = 0; + while (pos < buffer.length) { + char c = buffer[pos]; + if (c == ' ') { + indentLen++; + pos++; + } else if (c == '\t') { + indentLen += 8 - indentLen % 8; + pos++; + } else if (c == '\n') { // entirely blank line: discard + indentLen = 0; + pos++; + } else if (c == '#') { // line containing only indented comment + int oldPos = pos; + while (pos < buffer.length && c != '\n') { + c = buffer[pos++]; + } + addToken(new Token(TokenKind.COMMENT, oldPos, pos - 1, bufferSlice(oldPos, pos - 1))); + indentLen = 0; + } else { // printing character + break; + } + } + + if (pos == buffer.length) { + indentLen = 0; + } // trailing space on last line + + int peekedIndent = indentStack.peek(); + if (peekedIndent < indentLen) { // push a level + indentStack.push(indentLen); + addToken(new Token(TokenKind.INDENT, pos - 1, pos)); + + } else if (peekedIndent > indentLen) { // pop one or more levels + while (peekedIndent > indentLen) { + indentStack.pop(); + addToken(new Token(TokenKind.OUTDENT, pos - 1, pos)); + peekedIndent = indentStack.peek(); + } + + if (peekedIndent < indentLen) { + error("indentation error"); + } + } + } + + /** + * Returns true if current position is in the middle of a triple quote + * delimiter (3 x quot), and advances 'pos' by two if so. + */ + private boolean skipTripleQuote(char quot) { + if (pos + 1 < buffer.length && buffer[pos] == quot && buffer[pos + 1] == quot) { + pos += 2; + return true; + } else { + return false; + } + } + + /** + * Scans a string literal delimited by 'quot', containing escape sequences. + * + * ON ENTRY: 'pos' is 1 + the index of the first delimiter + * ON EXIT: 'pos' is 1 + the index of the last delimiter. + * + * @return the string-literal token. + */ + private Token escapedStringLiteral(char quot) { + boolean inTriplequote = skipTripleQuote(quot); + + int oldPos = pos - 1; + // more expensive second choice that expands escaped into a buffer + StringBuilder literal = new StringBuilder(); + while (pos < buffer.length) { + char c = buffer[pos]; + pos++; + switch (c) { + case '\n': + if (inTriplequote) { + literal.append(c); + break; + } else { + error("unterminated string literal at eol", oldPos, pos); + newline(); + return new Token(TokenKind.STRING, oldPos, pos, literal.toString()); + } + case '\\': + if (pos == buffer.length) { + error("unterminated string literal at eof", oldPos, pos); + return new Token(TokenKind.STRING, oldPos, pos, literal.toString()); + } + c = buffer[pos]; + pos++; + switch (c) { + case '\n': + // ignore end of line character + break; + case 'n': + literal.append('\n'); + break; + case 'r': + literal.append('\r'); + break; + case 't': + literal.append('\t'); + break; + case '\\': + literal.append('\\'); + break; + case '\'': + literal.append('\''); + break; + case '"': + literal.append('"'); + break; + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': { // octal escape + int octal = c - '0'; + if (pos < buffer.length) { + c = buffer[pos]; + if (c >= '0' && c <= '7') { + pos++; + octal = (octal << 3) | (c - '0'); + if (pos < buffer.length) { + c = buffer[pos]; + if (c >= '0' && c <= '7') { + pos++; + octal = (octal << 3) | (c - '0'); + } + } + } + } + literal.append((char) (octal & 0xff)); + break; + } + case 'a': case 'b': case 'f': case 'N': case 'u': case 'U': case 'v': case 'x': + // exists in Python but not implemented in Blaze => error + error("escape sequence not implemented: \\" + c, oldPos, pos); + break; + default: + // unknown char escape => "\literal" + literal.append('\\'); + literal.append(c); + break; + } + break; + case '\'': + case '"': + if (c != quot + || (inTriplequote && !skipTripleQuote(quot))) { + // Non-matching quote, treat it like a regular char. + literal.append(c); + } else { + // Matching close-delimiter, all done. + return new Token(TokenKind.STRING, oldPos, pos, literal.toString()); + } + break; + default: + literal.append(c); + break; + } + } + error("unterminated string literal at eof", oldPos, pos); + return new Token(TokenKind.STRING, oldPos, pos, literal.toString()); + } + + /** + * Scans a string literal delimited by 'quot'. + * + * <ul> + * <li> ON ENTRY: 'pos' is 1 + the index of the first delimiter + * <li> ON EXIT: 'pos' is 1 + the index of the last delimiter. + * </ul> + * + * @param isRaw if true, do not escape the string. + * @return the string-literal token. + */ + private Token stringLiteral(char quot, boolean isRaw) { + int oldPos = pos - 1; + + // Don't even attempt to parse triple-quotes here. + if (skipTripleQuote(quot)) { + pos -= 2; + return escapedStringLiteral(quot); + } + + // first quick optimistic scan for a simple non-escaped string + while (pos < buffer.length) { + char c = buffer[pos++]; + switch (c) { + case '\n': + error("unterminated string literal at eol", oldPos, pos); + Token t = new Token(TokenKind.STRING, oldPos, pos, + bufferSlice(oldPos + 1, pos - 1)); + newline(); + return t; + case '\\': + if (isRaw) { + // skip the next character + pos++; + break; + } else { + // oops, hit an escape, need to start over & build a new string buffer + pos = oldPos + 1; + return escapedStringLiteral(quot); + } + case '\'': + case '"': + if (c == quot) { + // close-quote, all done. + return new Token(TokenKind.STRING, oldPos, pos, + bufferSlice(oldPos + 1, pos - 1)); + } + } + } + + error("unterminated string literal at eof", oldPos, pos); + return new Token(TokenKind.STRING, oldPos, pos, + bufferSlice(oldPos + 1, pos)); + } + + private static final Map<String, TokenKind> keywordMap = new HashMap<>(); + + static { + keywordMap.put("and", TokenKind.AND); + keywordMap.put("as", TokenKind.AS); + keywordMap.put("class", TokenKind.CLASS); // reserved for future expansion + keywordMap.put("def", TokenKind.DEF); + keywordMap.put("elif", TokenKind.ELIF); + keywordMap.put("else", TokenKind.ELSE); + keywordMap.put("except", TokenKind.EXCEPT); + keywordMap.put("finally", TokenKind.FINALLY); + keywordMap.put("for", TokenKind.FOR); + keywordMap.put("from", TokenKind.FROM); + keywordMap.put("if", TokenKind.IF); + keywordMap.put("import", TokenKind.IMPORT); + keywordMap.put("in", TokenKind.IN); + keywordMap.put("not", TokenKind.NOT); + keywordMap.put("or", TokenKind.OR); + keywordMap.put("return", TokenKind.RETURN); + keywordMap.put("try", TokenKind.TRY); + } + + private TokenKind getTokenKindForIdentfier(String id) { + TokenKind kind = keywordMap.get(id); + return kind == null ? TokenKind.IDENTIFIER : kind; + } + + private String scanIdentifier() { + int oldPos = pos - 1; + while (pos < buffer.length) { + switch (buffer[pos]) { + case '_': + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': + case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': + case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': + case 's': case 't': case 'u': case 'v': case 'w': case 'x': + case 'y': case 'z': + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': + case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': + case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': + case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': + case 'Y': case 'Z': + case '0': case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': + pos++; + break; + default: + return bufferSlice(oldPos, pos); + } + } + return bufferSlice(oldPos, pos); + } + + /** + * Scans an identifier or keyword. + * + * ON ENTRY: 'pos' is 1 + the index of the first char in the identifier. + * ON EXIT: 'pos' is 1 + the index of the last char in the identifier. + * + * @return the identifier or keyword token. + */ + private Token identifierOrKeyword() { + int oldPos = pos - 1; + String id = scanIdentifier(); + TokenKind kind = getTokenKindForIdentfier(id); + return new Token(kind, oldPos, pos, + (kind == TokenKind.IDENTIFIER) ? id : null); + } + + private String scanInteger() { + int oldPos = pos - 1; + while (pos < buffer.length) { + char c = buffer[pos]; + switch (c) { + case 'X': case 'x': + case 'a': case 'A': + case 'b': case 'B': + case 'c': case 'C': + case 'd': case 'D': + case 'e': case 'E': + case 'f': case 'F': + case '0': case '1': + case '2': case '3': + case '4': case '5': + case '6': case '7': + case '8': case '9': + pos++; + break; + default: + return bufferSlice(oldPos, pos); + } + } + // TODO(bazel-team): (2009) to do roundtripping when we evaluate the integer + // constants, we must save the actual text of the tokens, not just their + // integer value. + + return bufferSlice(oldPos, pos); + } + + /** + * Scans an integer literal. + * + * ON ENTRY: 'pos' is 1 + the index of the first char in the literal. + * ON EXIT: 'pos' is 1 + the index of the last char in the literal. + * + * @return the integer token. + */ + private Token integer() { + int oldPos = pos - 1; + String literal = scanInteger(); + + final String substring; + final int radix; + if (literal.startsWith("0x") || literal.startsWith("0X")) { + radix = 16; + substring = literal.substring(2); + } else if (literal.startsWith("0") && literal.length() > 1) { + radix = 8; + substring = literal.substring(1); + } else { + radix = 10; + substring = literal; + } + + int value = 0; + try { + value = Integer.parseInt(substring, radix); + } catch (NumberFormatException e) { + error("invalid base-" + radix + " integer constant: " + literal); + } + + return new Token(TokenKind.INT, oldPos, pos, value); + } + + /** + * Tokenizes a two-char operator. + * @return true if it tokenized an operator + */ + private boolean tokenizeTwoChars() { + if (pos + 2 >= buffer.length) { + return false; + } + char c1 = buffer[pos]; + char c2 = buffer[pos + 1]; + if (c2 == '=') { + switch (c1) { + case '=': { + addToken(new Token(TokenKind.EQUALS_EQUALS, pos, pos + 2)); + return true; + } + case '!': { + addToken(new Token(TokenKind.NOT_EQUALS, pos, pos + 2)); + return true; + } + case '>': { + addToken(new Token(TokenKind.GREATER_EQUALS, pos, pos + 2)); + return true; + } + case '<': { + addToken(new Token(TokenKind.LESS_EQUALS, pos, pos + 2)); + return true; + } + case '+': { + addToken(new Token(TokenKind.PLUS_EQUALS, pos, pos + 2)); + return true; + } + } + } + return false; + } + + /** + * Performs tokenization of the character buffer of file contents provided to + * the constructor. + */ + private void tokenize() { + while (pos < buffer.length) { + if (tokenizeTwoChars()) { + pos += 2; + continue; + } + char c = buffer[pos]; + pos++; + switch (c) { + case '{': { + addToken(new Token(TokenKind.LBRACE, pos - 1, pos)); + openParenStackDepth++; + break; + } + case '}': { + addToken(new Token(TokenKind.RBRACE, pos - 1, pos)); + popParen(); + break; + } + case '(': { + addToken(new Token(TokenKind.LPAREN, pos - 1, pos)); + openParenStackDepth++; + break; + } + case ')': { + addToken(new Token(TokenKind.RPAREN, pos - 1, pos)); + popParen(); + break; + } + case '[': { + addToken(new Token(TokenKind.LBRACKET, pos - 1, pos)); + openParenStackDepth++; + break; + } + case ']': { + addToken(new Token(TokenKind.RBRACKET, pos - 1, pos)); + popParen(); + break; + } + case '>': { + addToken(new Token(TokenKind.GREATER, pos - 1, pos)); + break; + } + case '<': { + addToken(new Token(TokenKind.LESS, pos - 1, pos)); + break; + } + case ':': { + addToken(new Token(TokenKind.COLON, pos - 1, pos)); + break; + } + case ',': { + addToken(new Token(TokenKind.COMMA, pos - 1, pos)); + break; + } + case '+': { + addToken(new Token(TokenKind.PLUS, pos - 1, pos)); + break; + } + case '-': { + addToken(new Token(TokenKind.MINUS, pos - 1, pos)); + break; + } + case '=': { + addToken(new Token(TokenKind.EQUALS, pos - 1, pos)); + break; + } + case '%': { + addToken(new Token(TokenKind.PERCENT, pos - 1, pos)); + break; + } + case ';': { + addToken(new Token(TokenKind.SEMI, pos - 1, pos)); + break; + } + case '.': { + addToken(new Token(TokenKind.DOT, pos - 1, pos)); + break; + } + case '*': { + addToken(new Token(TokenKind.STAR, pos - 1, pos)); + break; + } + case ' ': + case '\t': + case '\r': { + /* ignore */ + break; + } + case '\\': { + // Backslash character is valid only at the end of a line (or in a string) + if (pos + 1 < buffer.length && buffer[pos] == '\n') { + pos++; // skip the end of line character + } else { + addToken(new Token(TokenKind.ILLEGAL, pos - 1, pos, Character.toString(c))); + } + break; + } + case '\n': { + newline(); + break; + } + case '#': { + int oldPos = pos - 1; + while (pos < buffer.length) { + c = buffer[pos]; + if (c == '\n') { + break; + } else { + pos++; + } + } + addToken(new Token(TokenKind.COMMENT, oldPos, pos, bufferSlice(oldPos, pos))); + break; + } + case '\'': + case '\"': { + addToken(stringLiteral(c, false)); + break; + } + default: { + // detect raw strings, e.g. r"str" + if (c == 'r' && pos < buffer.length + && (buffer[pos] == '\'' || buffer[pos] == '\"')) { + c = buffer[pos]; + pos++; + addToken(stringLiteral(c, true)); + break; + } + + if (Character.isDigit(c)) { + addToken(integer()); + } else if (Character.isJavaIdentifierStart(c) && c != '$') { + addToken(identifierOrKeyword()); + } else { + // Some characters in Python are not recognized in Blaze syntax (e.g. '!') + if (parsePython) { + addToken(new Token(TokenKind.ILLEGAL, pos - 1, pos, Character.toString(c))); + } else { + error("invalid character: '" + c + "'"); + } + } + break; + } // default + } // switch + } // while + + if (indentStack.size() > 1) { // top of stack is always zero + addToken(new Token(TokenKind.NEWLINE, pos - 1, pos)); + while (indentStack.size() > 1) { + indentStack.pop(); + addToken(new Token(TokenKind.OUTDENT, pos - 1, pos)); + } + } + + // Like Python, always end with a NEWLINE token, even if no '\n' in input: + if (tokens.size() == 0 + || tokens.get(tokens.size() - 1).kind != TokenKind.NEWLINE) { + addToken(new Token(TokenKind.NEWLINE, pos - 1, pos)); + } + + addToken(new Token(TokenKind.EOF, pos, pos)); + } + + /** + * Returns the character in the input buffer at the given position. + * + * @param at the position to get the character at. + * @return the character at the given position. + */ + public char charAt(int at) { + return buffer[at]; + } + + /** + * Returns the string at the current line, minus the new line. + * + * @param line the line from which to retrieve the String, 1-based + * @return the text of the line + */ + public String stringAtLine(int line) { + Pair<Integer, Integer> offsets = locationInfo.lineNumberTable.getOffsetsForLine(line); + return bufferSlice(offsets.first, offsets.second); + } + + /** + * Returns parts of the source buffer based on offsets + * + * @param start the beginning offset for the slice + * @param end the offset immediately following the slice + * @return the text at offset start with length end - start + */ + private String bufferSlice(int start, int end) { + return new String(this.buffer, start, end - start); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/LineNumberTable.java b/src/main/java/com/google/devtools/build/lib/syntax/LineNumberTable.java new file mode 100644 index 0000000..4842a16 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/LineNumberTable.java
@@ -0,0 +1,235 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.Location.LineAndColumn; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.StringUtilities; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.Serializable; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A table to keep track of line numbers in source files. The client creates a LineNumberTable for + * their buffer using {@link #create}. The client can then ask for the line and column given a + * position using ({@link #getLineAndColumn(int)}). + */ +abstract class LineNumberTable implements Serializable { + + /** + * Returns the (line, column) pair for the specified offset. + */ + abstract LineAndColumn getLineAndColumn(int offset); + + /** + * Returns the (start, end) offset pair for a specified line, not including + * newline chars. + */ + abstract Pair<Integer, Integer> getOffsetsForLine(int line); + + /** + * Returns the path corresponding to the given offset. + */ + abstract Path getPath(int offset); + + static LineNumberTable create(char[] buffer, Path path) { + // If #line appears within a BUILD file, we assume it has been preprocessed + // by gconfig2blaze. We ignore all actual newlines and compute the logical + // LNT based only on the presence of #line markers. + return StringUtilities.containsSubarray(buffer, "\n#line ".toCharArray()) + ? new HashLine(buffer, path) + : new Regular(buffer, path); + } + + /** + * Line number table implementation for regular source files. Records + * offsets of newlines. + */ + @Immutable + private static class Regular extends LineNumberTable { + + /** + * A mapping from line number (line >= 1) to character offset into the file. + */ + private final int[] linestart; + private final Path path; + private final int bufferLength; + + private Regular(char[] buffer, Path path) { + // Compute the size. + int size = 2; + for (int i = 0; i < buffer.length; i++) { + if (buffer[i] == '\n') { + size++; + } + } + linestart = new int[size]; + + int index = 0; + linestart[index++] = 0; // The 0th line does not exist - so we fill something in + // to make sure the start pos for the 1st line ends up at + // linestart[1]. Using 0 is useful for tables that are + // completely empty. + linestart[index++] = 0; // The first line ("line 1") starts at offset 0. + + // Scan the buffer and record the offset of each line start. Doing this + // once upfront is faster than checking each char as it is pulled from + // the buffer. + for (int i = 0; i < buffer.length; i++) { + if (buffer[i] == '\n') { + linestart[index++] = i + 1; + } + } + this.bufferLength = buffer.length; + this.path = path; + } + + private int getLineAt(int pos) { + if (pos < 0) { + throw new IllegalArgumentException("Illegal position: " + pos); + } + int lowBoundary = 1, highBoundary = linestart.length - 1; + while (true) { + if ((highBoundary - lowBoundary) <= 1) { + if (linestart[highBoundary] > pos) { + return lowBoundary; + } else { + return highBoundary; + } + } + int medium = lowBoundary + ((highBoundary - lowBoundary) >> 1); + if (linestart[medium] > pos) { + highBoundary = medium; + } else { + lowBoundary = medium; + } + } + } + + @Override + LineAndColumn getLineAndColumn(int offset) { + int line = getLineAt(offset); + int column = offset - linestart[line] + 1; + return new LineAndColumn(line, column); + } + + @Override + Path getPath(int offset) { + return path; + } + + @Override + Pair<Integer, Integer> getOffsetsForLine(int line) { + if (line <= 0 || line >= linestart.length) { + throw new IllegalArgumentException("Illegal line: " + line); + } + return Pair.of(linestart[line], line < linestart.length - 1 + ? linestart[line + 1] + : bufferLength); + } + } + + /** + * Line number table implementation for source files that have been + * preprocessed. Ignores newlines and uses only #line directives. + */ + // TODO(bazel-team): Use binary search instead of linear search. + @Immutable + private static class HashLine extends LineNumberTable { + + /** + * Represents a "#line" directive + */ + private static class SingleHashLine implements Serializable { + final private int offset; + final private int line; + final private Path path; + + SingleHashLine(int offset, int line, Path path) { + this.offset = offset; + this.line = line; + this.path = path; + } + } + + private static final Pattern pattern = Pattern.compile("\n#line ([0-9]+) \"([^\"\\n]+)\""); + + private final List<SingleHashLine> table; + private final Path defaultPath; + private final int bufferLength; + + private HashLine(char[] buffer, Path defaultPath) { + // Not especially efficient, but that's fine: we just exec'd Python. + String bufString = new String(buffer); + Matcher m = pattern.matcher(bufString); + ImmutableList.Builder<SingleHashLine> tableBuilder = ImmutableList.builder(); + while (m.find()) { + tableBuilder.add(new SingleHashLine( + m.start(0) + 1, //offset (+1 to skip \n in pattern) + Integer.valueOf(m.group(1)), // line number + defaultPath.getRelative(m.group(2)))); // filename is an absolute path + } + this.table = tableBuilder.build(); + this.bufferLength = buffer.length; + this.defaultPath = defaultPath; + } + + @Override + LineAndColumn getLineAndColumn(int offset) { + int line = -1; + for (int ii = 0, len = table.size(); ii < len; ii++) { + SingleHashLine hash = table.get(ii); + if (hash.offset > offset) { + break; + } + line = hash.line; + } + return new LineAndColumn(line, 1); + } + + @Override + Path getPath(int offset) { + Path path = this.defaultPath; + for (int ii = 0, len = table.size(); ii < len; ii++) { + SingleHashLine hash = table.get(ii); + if (hash.offset > offset) { + break; + } + path = hash.path; + } + return path; + } + + /** + * Returns 0, 0 for an unknown line + */ + @Override + Pair<Integer, Integer> getOffsetsForLine(int line) { + for (int ii = 0, len = table.size(); ii < len; ii++) { + if (table.get(ii).line == line) { + return Pair.of(table.get(ii).offset, ii < len - 1 + ? table.get(ii + 1).offset + : bufferLength); + } + } + return Pair.of(0, 0); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ListComprehension.java b/src/main/java/com/google/devtools/build/lib/syntax/ListComprehension.java new file mode 100644 index 0000000..6a13ba8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/ListComprehension.java
@@ -0,0 +1,133 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Syntax node for lists comprehension expressions. + */ +public final class ListComprehension extends Expression { + + private final Expression elementExpression; + // This cannot be a map, because we need to both preserve order _and_ allow duplicate identifiers. + private final List<Map.Entry<Ident, Expression>> lists; + + /** + * [elementExpr (for var in listExpr)+] + */ + public ListComprehension(Expression elementExpression) { + this.elementExpression = elementExpression; + lists = new ArrayList<Map.Entry<Ident, Expression>>(); + } + + @Override + Object eval(Environment env) throws EvalException, InterruptedException { + if (lists.size() == 0) { + return convert(new ArrayList<>(), env); + } + + List<Map.Entry<Ident, Iterable<?>>> listValues = Lists.newArrayListWithCapacity(lists.size()); + int size = 1; + for (Map.Entry<Ident, Expression> list : lists) { + Object listValueObject = list.getValue().eval(env); + final Iterable<?> listValue = EvalUtils.toIterable(listValueObject, getLocation()); + int listSize = EvalUtils.size(listValue); + if (listSize == 0) { + return convert(new ArrayList<>(), env); + } + size *= listSize; + listValues.add(Maps.<Ident, Iterable<?>>immutableEntry(list.getKey(), listValue)); + } + List<Object> resultList = Lists.newArrayListWithCapacity(size); + evalLists(env, listValues, resultList); + return convert(resultList, env); + } + + private Object convert(List<Object> list, Environment env) throws EvalException { + if (env.isSkylarkEnabled()) { + return SkylarkList.list(list, getLocation()); + } else { + return list; + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append('[').append(elementExpression); + for (Map.Entry<Ident, Expression> list : lists) { + sb.append(" for ").append(list.getKey()).append(" in ").append(list.getValue()); + } + sb.append(']'); + return sb.toString(); + } + + public Expression getElementExpression() { + return elementExpression; + } + + public void add(Ident ident, Expression listExpression) { + lists.add(Maps.immutableEntry(ident, listExpression)); + } + + public List<Map.Entry<Ident, Expression>> getLists() { + return lists; + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + /** + * Evaluates element expression over all combinations of list element values. + * + * <p>Iterates over all elements in outermost list (list at index 0) and + * updates the value of the list variable in the environment on each + * iteration. If there are no other lists to iterate over added evaluation + * of the element expression to the result. Otherwise calls itself recursively + * with all the lists except the outermost. + */ + private void evalLists(Environment env, List<Map.Entry<Ident, Iterable<?>>> listValues, + List<Object> result) throws EvalException, InterruptedException { + Map.Entry<Ident, Iterable<?>> listValue = listValues.get(0); + for (Object listElement : listValue.getValue()) { + env.update(listValue.getKey().getName(), listElement); + if (listValues.size() == 1) { + result.add(elementExpression.eval(env)); + } else { + evalLists(env, listValues.subList(1, listValues.size()), result); + } + } + } + + @Override + SkylarkType validate(ValidationEnvironment env) throws EvalException { + for (Map.Entry<Ident, Expression> list : lists) { + // TODO(bazel-team): Get the type of elements + SkylarkType type = list.getValue().validate(env); + env.checkIterable(type, getLocation()); + env.update(list.getKey().getName(), SkylarkType.UNKNOWN, getLocation()); + } + elementExpression.validate(env); + return SkylarkType.of(SkylarkList.class); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ListLiteral.java b/src/main/java/com/google/devtools/build/lib/syntax/ListLiteral.java new file mode 100644 index 0000000..9437135 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/ListLiteral.java
@@ -0,0 +1,128 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import java.util.ArrayList; +import java.util.List; + +/** + * Syntax node for list and tuple literals. + * + * (Note that during evaluation, both list and tuple values are represented by + * java.util.List objects, the only difference between them being whether or not + * they are mutable.) + */ +public final class ListLiteral extends Expression { + + /** + * Types of the ListLiteral. + */ + public static enum Kind {LIST, TUPLE} + + private final Kind kind; + + private final List<Expression> exprs; + + private ListLiteral(Kind kind, List<Expression> exprs) { + this.kind = kind; + this.exprs = exprs; + } + + public static ListLiteral makeList(List<Expression> exprs) { + return new ListLiteral(Kind.LIST, exprs); + } + + public static ListLiteral makeTuple(List<Expression> exprs) { + return new ListLiteral(Kind.TUPLE, exprs); + } + + /** + * Returns the list of expressions for each element of the tuple. + */ + public List<Expression> getElements() { + return exprs; + } + + /** + * Returns true if this list is a tuple (a hash table, immutable list). + */ + public boolean isTuple() { + return kind == Kind.TUPLE; + } + + private static char startChar(Kind kind) { + switch(kind) { + case LIST: return '['; + case TUPLE: return '('; + } + return '['; + } + + private static char endChar(Kind kind) { + switch(kind) { + case LIST: return ']'; + case TUPLE: return ')'; + } + return ']'; + } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append(startChar(kind)); + String sep = ""; + for (Expression e : exprs) { + sb.append(sep); + sb.append(e); + sep = ", "; + } + sb.append(endChar(kind)); + return sb.toString(); + } + + @Override + Object eval(Environment env) throws EvalException, InterruptedException { + List<Object> result = new ArrayList<>(); + for (Expression expr : exprs) { + // Convert NPEs to EvalExceptions. + if (expr == null) { + throw new EvalException(getLocation(), "null expression in " + this); + } + result.add(expr.eval(env)); + } + if (env.isSkylarkEnabled()) { + return isTuple() + ? SkylarkList.tuple(result) : SkylarkList.list(result, getLocation()); + } else { + return EvalUtils.makeSequence(result, isTuple()); + } + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + SkylarkType validate(ValidationEnvironment env) throws EvalException { + SkylarkType type = SkylarkType.UNKNOWN; + if (!isTuple()) { + for (Expression expr : exprs) { + SkylarkType nextType = expr.validate(env); + type = type.infer(nextType, "list literal", expr.getLocation(), getLocation()); + } + } + return SkylarkType.of(SkylarkList.class, type.getType()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Literal.java b/src/main/java/com/google/devtools/build/lib/syntax/Literal.java new file mode 100644 index 0000000..9289081 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/Literal.java
@@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +/** + * Generic base class for primitive literals. + */ +public abstract class Literal<T> extends Expression { + + protected final T value; + + protected Literal(T value) { + this.value = value; + } + + /** + * Returns the value of this literal. + */ + public T getValue() { + return value; + } + + @Override + public String toString() { + return value.toString(); + } + + @Override + Object eval(Environment env) { + return value; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/LoadStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/LoadStatement.java new file mode 100644 index 0000000..6873995 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/LoadStatement.java
@@ -0,0 +1,78 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.List; + +/** + * Syntax node for an import statement. + */ +public final class LoadStatement extends Statement { + + private final ImmutableList<Ident> symbols; + private final PathFragment importPath; + + /** + * Constructs an import statement. + */ + LoadStatement(String path, List<Ident> symbols) { + this.symbols = ImmutableList.copyOf(symbols); + this.importPath = new PathFragment(path + ".bzl"); + } + + public ImmutableList<Ident> getSymbols() { + return symbols; + } + + public PathFragment getImportPath() { + return importPath; + } + + @Override + public String toString() { + return String.format("load(\"%s\", %s)", importPath, Joiner.on(", ").join(symbols)); + } + + @Override + void exec(Environment env) throws EvalException, InterruptedException { + for (Ident i : symbols) { + try { + if (i.getName().startsWith("_")) { + throw new EvalException(getLocation(), "symbol '" + i + "' is private and cannot " + + "be imported"); + } + env.importSymbol(getImportPath(), i.getName()); + } catch (Environment.NoSuchVariableException | Environment.LoadFailedException e) { + throw new EvalException(getLocation(), e.getMessage()); + } + } + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + void validate(ValidationEnvironment env) throws EvalException { + // TODO(bazel-team): implement semantical check. + for (Ident symbol : symbols) { + env.update(symbol.getName(), SkylarkType.UNKNOWN, getLocation()); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/MixedModeFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/MixedModeFunction.java new file mode 100644 index 0000000..0427157 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/MixedModeFunction.java
@@ -0,0 +1,187 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.Type.ConversionException; + +import java.util.List; +import java.util.Map; + +/** + * Abstract implementation of Function for functions that accept a mixture of + * positional and keyword parameters, as in Python. + */ +public abstract class MixedModeFunction extends AbstractFunction { + + // Nomenclature: + // "Parameters" are formal parameters of a function definition. + // "Arguments" are actual parameters supplied at the call site. + + // Number of regular named parameters (excluding *p and **p) in the + // equivalent Python function definition). + private final List<String> parameters; + + // Number of leading "parameters" which are mandatory + private final int numMandatoryParameters; + + // True if this function requires all arguments to be named + // TODO(bazel-team): replace this by a count of arguments before the * with optional arg, + // in the style Python 3 or PEP 3102. + private final boolean onlyNamedArguments; + + // Location of the function definition, or null for builtin functions. + protected final Location location; + + /** + * Constructs an instance of Function that supports Python-style mixed-mode + * parameter passing. + * + * @param parameters a list of named parameters + * @param numMandatoryParameters the number of leading parameters which are + * considered mandatory; the remaining ones may be omitted, in which + * case they will have the default value of null. + */ + public MixedModeFunction(String name, + Iterable<String> parameters, + int numMandatoryParameters, + boolean onlyNamedArguments) { + this(name, parameters, numMandatoryParameters, onlyNamedArguments, null); + } + + protected MixedModeFunction(String name, + Iterable<String> parameters, + int numMandatoryParameters, + boolean onlyNamedArguments, + Location location) { + super(name); + this.parameters = ImmutableList.copyOf(parameters); + this.numMandatoryParameters = numMandatoryParameters; + this.onlyNamedArguments = onlyNamedArguments; + this.location = location; + } + + @Override + public Object call(List<Object> args, + Map<String, Object> kwargs, + FuncallExpression ast, + Environment env) + throws EvalException, InterruptedException { + + // ast is null when called from Java (as there's no Skylark call site). + Location loc = ast == null ? location : ast.getLocation(); + if (onlyNamedArguments && args.size() > 0) { + throw new EvalException(loc, + getSignature() + " does not accept positional arguments"); + } + + if (kwargs == null) { + kwargs = ImmutableMap.<String, Object>of(); + } + + int numParams = parameters.size(); + int numArgs = args.size(); + Object[] namedArguments = new Object[numParams]; + + // first, positional arguments: + if (numArgs > numParams) { + throw new EvalException(loc, + "too many positional arguments in call to " + getSignature()); + } + for (int ii = 0; ii < numArgs; ++ii) { + namedArguments[ii] = args.get(ii); + } + + // TODO(bazel-team): here, support *varargs splicing + + // second, keyword arguments: + for (Map.Entry<String, Object> entry : kwargs.entrySet()) { + String keyword = entry.getKey(); + int pos = parameters.indexOf(keyword); + if (pos == -1) { + throw new EvalException(loc, + "unexpected keyword '" + keyword + + "' in call to " + getSignature()); + } else { + if (namedArguments[pos] != null) { + throw new EvalException(loc, getSignature() + + " got multiple values for keyword argument '" + keyword + "'"); + } + namedArguments[pos] = kwargs.get(keyword); + } + } + + // third, defaults: + for (int ii = 0; ii < numMandatoryParameters; ++ii) { + if (namedArguments[ii] == null) { + throw new EvalException(loc, + getSignature() + " received insufficient arguments"); + } + } + // (defaults are always null so nothing extra to do here.) + + try { + return call(namedArguments, ast, env); + } catch (ConversionException | IllegalArgumentException | IllegalStateException + | ClassCastException e) { + throw new EvalException(loc, e.getMessage()); + } + } + + /** + * Like Function.call, but generalised to support Python-style mixed-mode + * keyword and positional parameter passing. + * + * @param args an array of argument values corresponding to the list + * of named parameters passed to the constructor. + */ + protected Object call(Object[] args, FuncallExpression ast) + throws EvalException, ConversionException, InterruptedException { + throw new UnsupportedOperationException("Method not overridden"); + } + + /** + * Override this method instead of the one above, if you need to access + * the environment. + */ + protected Object call(Object[] args, FuncallExpression ast, Environment env) + throws EvalException, ConversionException, InterruptedException { + return call(args, ast); + } + + /** + * Render this object in the form of an equivalent Python function signature. + */ + public String getSignature() { + StringBuffer sb = new StringBuffer(); + sb.append(getName()).append('('); + int ii = 0; + int len = parameters.size(); + for (; ii < len; ++ii) { + String parameter = parameters.get(ii); + if (ii > 0) { + sb.append(", "); + } + sb.append(parameter); + if (ii >= numMandatoryParameters) { + sb.append(" = null"); + } + } + sb.append(')'); + return sb.toString(); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/NotExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/NotExpression.java new file mode 100644 index 0000000..5a13e79 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/NotExpression.java
@@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +/** + * As syntax node for the not boolean operation. + */ +public class NotExpression extends Expression { + + private final Expression expression; + + public NotExpression(Expression expression) { + this.expression = expression; + } + + Expression getExpression() { + return expression; + } + + @Override + Object eval(Environment env) throws EvalException, InterruptedException { + return !EvalUtils.toBoolean(expression.eval(env)); + } + + @Override + public String toString() { + return "not " + expression; + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + SkylarkType validate(ValidationEnvironment env) throws EvalException { + // Don't need type check here since EvalUtils.toBoolean() converts everything. + expression.validate(env); + return SkylarkType.BOOL; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Operator.java b/src/main/java/com/google/devtools/build/lib/syntax/Operator.java new file mode 100644 index 0000000..628570e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/Operator.java
@@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +/** + * Infix operators supported by the build language. + */ +public enum Operator { + + AND("and"), + EQUALS_EQUALS("=="), + GREATER(">"), + GREATER_EQUALS(">="), + IN("in"), + LESS("<"), + LESS_EQUALS("<="), + MINUS("-"), + MULT("*"), + NOT("not"), + NOT_EQUALS("!="), + OR("or"), + PERCENT("%"), + PLUS("+"); + + private final String name; + + private Operator(String name) { + this.name = name; + } + + @Override + public String toString() { + return name; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Parser.java b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java new file mode 100644 index 0000000..66c3c67 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
@@ -0,0 +1,1274 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.CachingPackageLocator; +import com.google.devtools.build.lib.syntax.DictionaryLiteral.DictionaryEntryLiteral; +import com.google.devtools.build.lib.syntax.IfStatement.ConditionalStatements; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Recursive descent parser for LL(2) BUILD language. + * Loosely based on Python 2 grammar. + * See https://docs.python.org/2/reference/grammar.html + * + */ +class Parser { + + /** + * Combines the parser result into a single value object. + */ + public static final class ParseResult { + /** The statements (rules, basically) from the parsed file. */ + public final List<Statement> statements; + + /** The comments from the parsed file. */ + public final List<Comment> comments; + + /** Whether the file contained any errors. */ + public final boolean containsErrors; + + public ParseResult(List<Statement> statements, List<Comment> comments, boolean containsErrors) { + // No need to copy here; when the object is created, the parser instance is just about to go + // out of scope and be garbage collected. + this.statements = Preconditions.checkNotNull(statements); + this.comments = Preconditions.checkNotNull(comments); + this.containsErrors = containsErrors; + } + } + + private static final EnumSet<TokenKind> STATEMENT_TERMINATOR_SET = + EnumSet.of(TokenKind.EOF, TokenKind.NEWLINE); + + private static final EnumSet<TokenKind> LIST_TERMINATOR_SET = + EnumSet.of(TokenKind.EOF, TokenKind.RBRACKET, TokenKind.SEMI); + + private static final EnumSet<TokenKind> DICT_TERMINATOR_SET = + EnumSet.of(TokenKind.EOF, TokenKind.RBRACE, TokenKind.SEMI); + + private static final EnumSet<TokenKind> EXPR_TERMINATOR_SET = EnumSet.of( + TokenKind.EOF, + TokenKind.COMMA, + TokenKind.COLON, + TokenKind.FOR, + TokenKind.PLUS, + TokenKind.MINUS, + TokenKind.PERCENT, + TokenKind.RPAREN, + TokenKind.RBRACKET); + + private Token token; // current lookahead token + private Token pushedToken = null; // used to implement LL(2) + + private static final boolean DEBUGGING = false; + + private final Lexer lexer; + private final EventHandler eventHandler; + private final List<Comment> comments; + private final boolean parsePython; + /** Whether advanced language constructs are allowed */ + private boolean skylarkMode = false; + + private static final Map<TokenKind, Operator> binaryOperators = + new ImmutableMap.Builder<TokenKind, Operator>() + .put(TokenKind.AND, Operator.AND) + .put(TokenKind.EQUALS_EQUALS, Operator.EQUALS_EQUALS) + .put(TokenKind.GREATER, Operator.GREATER) + .put(TokenKind.GREATER_EQUALS, Operator.GREATER_EQUALS) + .put(TokenKind.IN, Operator.IN) + .put(TokenKind.LESS, Operator.LESS) + .put(TokenKind.LESS_EQUALS, Operator.LESS_EQUALS) + .put(TokenKind.MINUS, Operator.MINUS) + .put(TokenKind.NOT_EQUALS, Operator.NOT_EQUALS) + .put(TokenKind.OR, Operator.OR) + .put(TokenKind.PERCENT, Operator.PERCENT) + .put(TokenKind.PLUS, Operator.PLUS) + .put(TokenKind.STAR, Operator.MULT) + .build(); + + private static final Map<TokenKind, Operator> augmentedAssignmentMethods = + new ImmutableMap.Builder<TokenKind, Operator>() + .put(TokenKind.PLUS_EQUALS, Operator.PLUS) // += // TODO(bazel-team): other similar operators + .build(); + + /** Highest precedence goes last. + * Based on: http://docs.python.org/2/reference/expressions.html#operator-precedence + **/ + private static final List<EnumSet<Operator>> operatorPrecedence = ImmutableList.of( + EnumSet.of(Operator.OR), + EnumSet.of(Operator.AND), + EnumSet.of(Operator.NOT), + EnumSet.of(Operator.EQUALS_EQUALS, Operator.NOT_EQUALS, Operator.LESS, Operator.LESS_EQUALS, + Operator.GREATER, Operator.GREATER_EQUALS, Operator.IN), + EnumSet.of(Operator.MINUS, Operator.PLUS), + EnumSet.of(Operator.MULT, Operator.PERCENT)); + + private Iterator<Token> tokens = null; + private int errorsCount; + private boolean recoveryMode; // stop reporting errors until next statement + + private CachingPackageLocator locator; + + private List<Path> includedFiles; + + private static final String PREPROCESSING_NEEDED = + "Add \"# PYTHON-PREPROCESSING-REQUIRED\" on the first line of the file"; + + private Parser(Lexer lexer, EventHandler eventHandler, CachingPackageLocator locator, + boolean parsePython) { + this.lexer = lexer; + this.eventHandler = eventHandler; + this.parsePython = parsePython; + this.tokens = lexer.getTokens().iterator(); + this.comments = new ArrayList<Comment>(); + this.locator = locator; + this.includedFiles = new ArrayList<Path>(); + this.includedFiles.add(lexer.getFilename()); + nextToken(); + } + + private Parser(Lexer lexer, EventHandler eventHandler, CachingPackageLocator locator) { + this(lexer, eventHandler, locator, false /* parsePython */); + } + + public Parser setSkylarkMode(boolean skylarkMode) { + this.skylarkMode = skylarkMode; + return this; + } + + /** + * Entry-point to parser that parses a build file with comments. All errors + * encountered during parsing are reported via "reporter". + */ + public static ParseResult parseFile( + Lexer lexer, EventHandler eventHandler, CachingPackageLocator locator, + boolean parsePython) { + Parser parser = new Parser(lexer, eventHandler, locator, parsePython); + List<Statement> statements = parser.parseFileInput(); + return new ParseResult(statements, parser.comments, + parser.errorsCount > 0 || lexer.containsErrors()); + } + + /** + * Entry-point to parser that parses a build file with comments. All errors + * encountered during parsing are reported via "reporter". Enable Skylark extensions + * that are not part of the core BUILD language. + */ + public static ParseResult parseFileForSkylark( + Lexer lexer, EventHandler eventHandler, CachingPackageLocator locator, + ValidationEnvironment validationEnvironment) { + Parser parser = new Parser(lexer, eventHandler, locator).setSkylarkMode(true); + List<Statement> statements = parser.parseFileInput(); + boolean hasSemanticalErrors = false; + try { + for (Statement statement : statements) { + statement.validate(validationEnvironment); + } + } catch (EvalException e) { + eventHandler.handle(Event.error(e.getLocation(), e.getMessage())); + hasSemanticalErrors = true; + } + return new ParseResult(statements, parser.comments, + parser.errorsCount > 0 || lexer.containsErrors() || hasSemanticalErrors); + } + + /** + * Entry-point to parser that parses a statement. All errors encountered + * during parsing are reported via "reporter". + */ + @VisibleForTesting + public static Statement parseStatement( + Lexer lexer, EventHandler eventHandler) { + return new Parser(lexer, eventHandler, null).parseSmallStatement(); + } + + /** + * Entry-point to parser that parses an expression. All errors encountered + * during parsing are reported via "reporter". The expression may be followed + * by newline tokens. + */ + @VisibleForTesting + public static Expression parseExpression(Lexer lexer, EventHandler eventHandler) { + Parser parser = new Parser(lexer, eventHandler, null); + Expression result = parser.parseExpression(); + while (parser.token.kind == TokenKind.NEWLINE) { + parser.nextToken(); + } + parser.expect(TokenKind.EOF); + return result; + } + + private void addIncludedFiles(List<Path> files) { + this.includedFiles.addAll(files); + } + + private void reportError(Location location, String message) { + errorsCount++; + // Limit the number of reported errors to avoid spamming output. + if (errorsCount <= 5) { + eventHandler.handle(Event.error(location, message)); + } + } + + private void syntaxError(Token token) { + if (!recoveryMode) { + String msg = token.kind == TokenKind.INDENT + ? "indentation error" + : "syntax error at '" + token + "'"; + reportError(lexer.createLocation(token.left, token.right), msg); + recoveryMode = true; + } + } + + // Consumes the current token. If it is not of the specified (expected) + // kind, reports a syntax error. + private boolean expect(TokenKind kind) { + boolean expected = token.kind == kind; + if (!expected) { + syntaxError(token); + } + nextToken(); + return expected; + } + + /** + * Consume tokens past the first token that has a kind that is in the set of + * teminatingTokens. + * @param terminatingTokens + * @return the end offset of the terminating token. + */ + private int syncPast(EnumSet<TokenKind> terminatingTokens) { + Preconditions.checkState(terminatingTokens.contains(TokenKind.EOF)); + while (!terminatingTokens.contains(token.kind)) { + nextToken(); + } + int end = token.right; + // read past the synchronization token + nextToken(); + return end; + } + + /** + * Consume tokens until we reach the first token that has a kind that is in + * the set of teminatingTokens. + * @param terminatingTokens + * @return the end offset of the terminating token. + */ + private int syncTo(EnumSet<TokenKind> terminatingTokens) { + // EOF must be in the set to prevent an infinite loop + Preconditions.checkState(terminatingTokens.contains(TokenKind.EOF)); + // read past the problematic token + int previous = token.right; + nextToken(); + int current = previous; + while (!terminatingTokens.contains(token.kind)) { + nextToken(); + previous = current; + current = token.right; + } + return previous; + } + + private void nextToken() { + if (pushedToken != null) { + token = pushedToken; + pushedToken = null; + } else { + if (token == null || token.kind != TokenKind.EOF) { + token = tokens.next(); + // transparently handle comment tokens + while (token.kind == TokenKind.COMMENT) { + makeComment(token); + token = tokens.next(); + } + } + } + if (DEBUGGING) { + System.err.print(token); + } + } + + private void pushToken(Token tokenToPush) { + if (pushedToken != null) { + throw new IllegalStateException("Exceeded LL(2) lookahead!"); + } + pushedToken = token; + token = tokenToPush; + } + + // create an error expression + private Ident makeErrorExpression(int start, int end) { + return setLocation(new Ident("$error$"), start, end); + } + + // Convenience wrapper around ASTNode.setLocation that returns the node. + private <NODE extends ASTNode> NODE + setLocation(NODE node, int startOffset, int endOffset) { + node.setLocation(lexer.createLocation(startOffset, endOffset)); + return node; + } + + // Another convenience wrapper method around ASTNode.setLocation + private <NODE extends ASTNode> NODE setLocation(NODE node, Location location) { + node.setLocation(location); + return node; + } + + // Convenience method that uses end offset from the last node. + private <NODE extends ASTNode> NODE setLocation(NODE node, int startOffset, ASTNode lastNode) { + return setLocation(node, startOffset, lastNode.getLocation().getEndOffset()); + } + + // create a funcall expression + private Expression makeFuncallExpression(Expression receiver, Ident function, + List<Argument> args, + int start, int end) { + if (function.getLocation() == null) { + function = setLocation(function, start, end); + } + boolean seenKeywordArg = false; + boolean seenKwargs = false; + for (Argument arg : args) { + if (arg.isPositional()) { + if (seenKeywordArg || seenKwargs) { + reportError(arg.getLocation(), "syntax error: non-keyword arg after keyword arg"); + return makeErrorExpression(start, end); + } + } else if (arg.isKwargs()) { + if (seenKwargs) { + reportError(arg.getLocation(), "there can be only one **kwargs argument"); + return makeErrorExpression(start, end); + } + seenKwargs = true; + } else { + seenKeywordArg = true; + } + } + + return setLocation(new FuncallExpression(receiver, function, args), start, end); + } + + // arg ::= IDENTIFIER '=' expr + // | expr + private Argument parseFunctionCallArgument() { + int start = token.left; + if (token.kind == TokenKind.IDENTIFIER) { + Token identToken = token; + String name = (String) token.value; + Ident ident = setLocation(new Ident(name), start, token.right); + nextToken(); + if (token.kind == TokenKind.EQUALS) { // it's a named argument + nextToken(); + Expression expr = parseExpression(); + return setLocation(new Argument(ident, expr), start, expr); + } else { // oops, back up! + pushToken(identToken); + } + } + // parse **expr + if (token.kind == TokenKind.STAR) { + expect(TokenKind.STAR); + expect(TokenKind.STAR); + Expression expr = parseExpression(); + return setLocation(new Argument(null, expr, true), start, expr); + } + // parse a positional argument + Expression expr = parseExpression(); + return setLocation(new Argument(expr), start, expr); + } + + // arg ::= IDENTIFIER '=' expr + // | IDENTIFIER + private Argument parseFunctionDefArgument(boolean onlyOptional) { + int start = token.left; + Ident ident = parseIdent(); + if (token.kind == TokenKind.EQUALS) { // there's a default value + nextToken(); + Expression expr = parseExpression(); + return setLocation(new Argument(ident, expr), start, expr); + } else if (onlyOptional) { + reportError(ident.getLocation(), + "Optional arguments are only allowed at the end of the argument list."); + } + return setLocation(new Argument(ident), start, ident); + } + + // funcall_suffix ::= '(' arg_list? ')' + private Expression parseFuncallSuffix(int start, Expression receiver, + Ident function) { + List<Argument> args = Collections.emptyList(); + expect(TokenKind.LPAREN); + int end; + if (token.kind == TokenKind.RPAREN) { + end = token.right; + nextToken(); // RPAREN + } else { + args = parseFunctionCallArguments(); // (includes optional trailing comma) + end = token.right; + expect(TokenKind.RPAREN); + } + return makeFuncallExpression(receiver, function, args, start, end); + } + + // selector_suffix ::= '.' IDENTIFIER + // |'.' IDENTIFIER funcall_suffix + private Expression parseSelectorSuffix(int start, Expression receiver) { + expect(TokenKind.DOT); + if (token.kind == TokenKind.IDENTIFIER) { + Ident ident = parseIdent(); + if (token.kind == TokenKind.LPAREN) { + return parseFuncallSuffix(start, receiver, ident); + } else { + return setLocation(new DotExpression(receiver, ident), start, token.right); + } + } else { + syntaxError(token); + int end = syncTo(EXPR_TERMINATOR_SET); + return makeErrorExpression(start, end); + } + } + + // arg_list ::= ( (arg ',')* arg ','? )? + private List<Argument> parseFunctionCallArguments() { + List<Argument> args = new ArrayList<>(); + // terminating tokens for an arg list + while (token.kind != TokenKind.RPAREN) { + if (token.kind == TokenKind.EOF) { + syntaxError(token); + break; + } + args.add(parseFunctionCallArgument()); + if (token.kind == TokenKind.COMMA) { + nextToken(); + } else { + break; + } + } + return args; + } + + // expr_list ::= ( (expr ',')* expr ','? )? + private List<Expression> parseExprList() { + List<Expression> list = new ArrayList<>(); + // terminating tokens for an expression list + while (token.kind != TokenKind.RPAREN && token.kind != TokenKind.RBRACKET) { + list.add(parseExpression()); + if (token.kind == TokenKind.COMMA) { + nextToken(); + } else { + break; + } + } + return list; + } + + // dict_entry_list ::= ( (dict_entry ',')* dict_entry ','? )? + private List<DictionaryEntryLiteral> parseDictEntryList() { + List<DictionaryEntryLiteral> list = new ArrayList<>(); + // the terminating token for a dict entry list + while (token.kind != TokenKind.RBRACE) { + list.add(parseDictEntry()); + if (token.kind == TokenKind.COMMA) { + nextToken(); + } else { + break; + } + } + return list; + } + + // dict_entry ::= expression ':' expression + private DictionaryEntryLiteral parseDictEntry() { + int start = token.left; + Expression key = parseExpression(); + expect(TokenKind.COLON); + Expression value = parseExpression(); + return setLocation(new DictionaryEntryLiteral(key, value), start, value); + } + + private ExpressionStatement mocksubincludeExpression( + String labelName, String file, Location location) { + List<Argument> args = new ArrayList<>(); + args.add(setLocation(new Argument(new StringLiteral(labelName, '"')), location)); + args.add(setLocation(new Argument(new StringLiteral(file, '"')), location)); + Ident mockIdent = setLocation(new Ident("mocksubinclude"), location); + Expression funCall = new FuncallExpression(null, mockIdent, args); + return setLocation(new ExpressionStatement(funCall), location); + } + + // parse a file from an include call + private void include(String labelName, List<Statement> list, Location location) { + if (locator == null) { + return; + } + + try { + Label label = Label.parseAbsolute(labelName); + String packageName = label.getPackageFragment().getPathString(); + Path packagePath = locator.getBuildFileForPackage(packageName); + if (packagePath == null) { + reportError(location, "Package '" + packageName + "' not found"); + list.add(mocksubincludeExpression(labelName, "", location)); + return; + } + Path path = packagePath.getParentDirectory(); + Path file = path.getRelative(label.getName()); + + if (this.includedFiles.contains(file)) { + reportError(location, "Recursive inclusion of file '" + path + "'"); + return; + } + ParserInputSource inputSource = ParserInputSource.create(file); + + // Insert call to the mocksubinclude function to get the dependencies right. + list.add(mocksubincludeExpression(labelName, file.toString(), location)); + + Lexer lexer = new Lexer(inputSource, eventHandler, parsePython); + Parser parser = new Parser(lexer, eventHandler, locator, parsePython); + parser.addIncludedFiles(this.includedFiles); + list.addAll(parser.parseFileInput()); + } catch (Label.SyntaxException e) { + reportError(location, "Invalid label '" + labelName + "'"); + } catch (IOException e) { + reportError(location, "Include of '" + labelName + "' failed: " + e.getMessage()); + list.add(mocksubincludeExpression(labelName, "", location)); + } + } + + // primary ::= INTEGER + // | STRING + // | STRING '.' IDENTIFIER funcall_suffix + // | IDENTIFIER + // | IDENTIFIER funcall_suffix + // | IDENTIFIER '.' selector_suffix + // | list_expression + // | '(' ')' // a tuple with zero elements + // | '(' expr ')' // a parenthesized expression + // | '(' expr ',' expr_list ')' // a tuple with n elements + // | dict_expression + // | '-' primary_with_suffix + private Expression parsePrimary() { + int start = token.left; + switch (token.kind) { + case INT: { + IntegerLiteral literal = new IntegerLiteral((Integer) token.value); + setLocation(literal, start, token.right); + nextToken(); + return literal; + } + case STRING: { + String value = (String) token.value; + int end = token.right; + char quoteChar = lexer.charAt(start); + nextToken(); + if (token.kind == TokenKind.STRING) { + reportError(lexer.createLocation(end, token.left), + "Implicit string concatenation is forbidden, use the + operator"); + } + StringLiteral literal = new StringLiteral(value, quoteChar); + setLocation(literal, start, end); + return literal; + } + case IDENTIFIER: { + Ident ident = parseIdent(); + if (token.kind == TokenKind.LPAREN) { // it's a function application + return parseFuncallSuffix(start, null, ident); + } else { + return ident; + } + } + case LBRACKET: { // it's a list + return parseListExpression(); + } + case LBRACE: { // it's a dictionary + return parseDictExpression(); + } + case LPAREN: { + nextToken(); + // check for the empty tuple literal + if (token.kind == TokenKind.RPAREN) { + ListLiteral literal = + ListLiteral.makeTuple(Collections.<Expression>emptyList()); + setLocation(literal, start, token.right); + nextToken(); + return literal; + } + // parse the first expression + Expression expression = parseExpression(); + if (token.kind == TokenKind.COMMA) { // it's a tuple + nextToken(); + // parse the rest of the expression tuple + List<Expression> tuple = parseExprList(); + // add the first expression to the front of the tuple + tuple.add(0, expression); + expect(TokenKind.RPAREN); + return setLocation( + ListLiteral.makeTuple(tuple), start, token.right); + } + setLocation(expression, start, token.right); + if (token.kind == TokenKind.RPAREN) { + nextToken(); + return expression; + } + syntaxError(token); + int end = syncTo(EXPR_TERMINATOR_SET); + return makeErrorExpression(start, end); + } + case MINUS: { + nextToken(); + + List<Argument> args = new ArrayList<>(); + Expression expr = parsePrimaryWithSuffix(); + args.add(setLocation(new Argument(expr), start, expr)); + return makeFuncallExpression(null, new Ident("-"), args, + start, token.right); + } + default: { + syntaxError(token); + int end = syncTo(EXPR_TERMINATOR_SET); + return makeErrorExpression(start, end); + } + } + } + + // primary_with_suffix ::= primary selector_suffix* + // | primary substring_suffix + private Expression parsePrimaryWithSuffix() { + int start = token.left; + Expression receiver = parsePrimary(); + while (true) { + if (token.kind == TokenKind.DOT) { + receiver = parseSelectorSuffix(start, receiver); + } else if (token.kind == TokenKind.LBRACKET) { + receiver = parseSubstringSuffix(start, receiver); + } else { + break; + } + } + return receiver; + } + + // substring_suffix ::= '[' expression? ':' expression? ']' + private Expression parseSubstringSuffix(int start, Expression receiver) { + List<Argument> args = new ArrayList<>(); + Expression startExpr; + Expression endExpr; + + expect(TokenKind.LBRACKET); + int loc1 = token.left; + if (token.kind == TokenKind.COLON) { + startExpr = setLocation(new IntegerLiteral(0), token.left, token.right); + } else { + startExpr = parseExpression(); + } + args.add(setLocation(new Argument(startExpr), loc1, startExpr)); + // This is a dictionary access + if (token.kind == TokenKind.RBRACKET) { + expect(TokenKind.RBRACKET); + return makeFuncallExpression(receiver, new Ident("$index"), args, + start, token.right); + } + // This is a substring + expect(TokenKind.COLON); + int loc2 = token.left; + if (token.kind == TokenKind.RBRACKET) { + endExpr = setLocation(new IntegerLiteral(Integer.MAX_VALUE), token.left, token.right); + } else { + endExpr = parseExpression(); + } + expect(TokenKind.RBRACKET); + + args.add(setLocation(new Argument(endExpr), loc2, endExpr)); + return makeFuncallExpression(receiver, new Ident("$substring"), args, + start, token.right); + } + + // loop_variables ::= '(' variables ')' + // | variables + // variables ::= ident (',' ident)* + private Ident parseForLoopVariables() { + int start = token.left; + boolean hasParen = false; + if (token.kind == TokenKind.LPAREN) { + hasParen = true; + nextToken(); + } + + // TODO(bazel-team): allow multiple variables in the core Blaze language too. + Ident firstIdent = parseIdent(); + boolean multipleVariables = false; + + while (token.kind == TokenKind.COMMA) { + multipleVariables = true; + nextToken(); + parseIdent(); + } + + if (hasParen) { + expect(TokenKind.RPAREN); + } + + int end = token.right; + if (multipleVariables && !parsePython) { + reportError(lexer.createLocation(start, end), + "For loops with multiple variables are not yet supported. " + + PREPROCESSING_NEEDED); + } + return multipleVariables ? makeErrorExpression(start, end) : firstIdent; + } + + // list_expression ::= '[' ']' + // |'[' expr ']' + // |'[' expr ',' expr_list ']' + // |'[' expr ('FOR' loop_variables 'IN' expr)+ ']' + private Expression parseListExpression() { + int start = token.left; + expect(TokenKind.LBRACKET); + if (token.kind == TokenKind.RBRACKET) { // empty List + ListLiteral literal = + ListLiteral.makeList(Collections.<Expression>emptyList()); + setLocation(literal, start, token.right); + nextToken(); + return literal; + } + Expression expression = parseExpression(); + Preconditions.checkNotNull(expression, + "null element in list in AST at %s:%s", token.left, token.right); + switch (token.kind) { + case RBRACKET: { // singleton List + ListLiteral literal = + ListLiteral.makeList(Collections.singletonList(expression)); + setLocation(literal, start, token.right); + nextToken(); + return literal; + } + case FOR: { // list comprehension + ListComprehension listComprehension = + new ListComprehension(expression); + do { + nextToken(); + Ident ident = parseForLoopVariables(); + if (token.kind == TokenKind.IN) { + nextToken(); + Expression listExpression = parseExpression(); + listComprehension.add(ident, listExpression); + } else { + break; + } + if (token.kind == TokenKind.RBRACKET) { + setLocation(listComprehension, start, token.right); + nextToken(); + return listComprehension; + } + } while (token.kind == TokenKind.FOR); + + syntaxError(token); + int end = syncPast(LIST_TERMINATOR_SET); + return makeErrorExpression(start, end); + } + case COMMA: { + nextToken(); + List<Expression> list = parseExprList(); + Preconditions.checkState(!list.contains(null), + "null element in list in AST at %s:%s", token.left, token.right); + list.add(0, expression); + if (token.kind == TokenKind.RBRACKET) { + ListLiteral literal = ListLiteral.makeList(list); + setLocation(literal, start, token.right); + nextToken(); + return literal; + } + syntaxError(token); + int end = syncPast(LIST_TERMINATOR_SET); + return makeErrorExpression(start, end); + } + default: { + syntaxError(token); + int end = syncPast(LIST_TERMINATOR_SET); + return makeErrorExpression(start, end); + } + } + } + + // dict_expression ::= '{' '}' + // |'{' dict_entry_list '}' + // |'{' dict_entry 'FOR' loop_variables 'IN' expr '}' + private Expression parseDictExpression() { + int start = token.left; + expect(TokenKind.LBRACE); + if (token.kind == TokenKind.RBRACE) { // empty List + DictionaryLiteral literal = + new DictionaryLiteral(ImmutableList.<DictionaryEntryLiteral>of()); + setLocation(literal, start, token.right); + nextToken(); + return literal; + } + DictionaryEntryLiteral entry = parseDictEntry(); + if (token.kind == TokenKind.FOR) { + // Dict comprehension + nextToken(); + Ident loopVar = parseForLoopVariables(); + expect(TokenKind.IN); + Expression listExpression = parseExpression(); + expect(TokenKind.RBRACE); + return setLocation(new DictComprehension( + entry.getKey(), entry.getValue(), loopVar, listExpression), start, token.right); + } + List<DictionaryEntryLiteral> entries = new ArrayList<>(); + entries.add(entry); + if (token.kind == TokenKind.COMMA) { + expect(TokenKind.COMMA); + entries.addAll(parseDictEntryList()); + } + if (token.kind == TokenKind.RBRACE) { + DictionaryLiteral literal = new DictionaryLiteral(entries); + setLocation(literal, start, token.right); + nextToken(); + return literal; + } + syntaxError(token); + int end = syncPast(DICT_TERMINATOR_SET); + return makeErrorExpression(start, end); + } + + private Ident parseIdent() { + if (token.kind != TokenKind.IDENTIFIER) { + syntaxError(token); + return makeErrorExpression(token.left, token.right); + } + Ident ident = new Ident(((String) token.value)); + setLocation(ident, token.left, token.right); + nextToken(); + return ident; + } + + // binop_expression ::= binop_expression OP binop_expression + // | parsePrimaryWithSuffix + // This function takes care of precedence between operators (see operatorPrecedence for + // the order), and it assumes left-to-right associativity. + private Expression parseBinOpExpression(int prec) { + int start = token.left; + Expression expr = parseExpression(prec + 1); + // The loop is not strictly needed, but it prevents risks of stack overflow. Depth is + // limited to number of different precedence levels (operatorPrecedence.size()). + for (;;) { + if (!binaryOperators.containsKey(token.kind)) { + return expr; + } + Operator operator = binaryOperators.get(token.kind); + if (!operatorPrecedence.get(prec).contains(operator)) { + return expr; + } + nextToken(); + Expression secondary = parseExpression(prec + 1); + expr = optimizeBinOpExpression(operator, expr, secondary); + setLocation(expr, start, secondary); + } + } + + // Optimize binary expressions. + // string literal + string literal can be concatenated into one string literal + // so we don't have to do the expensive string concatenation at runtime. + private Expression optimizeBinOpExpression( + Operator operator, Expression expr, Expression secondary) { + if (operator == Operator.PLUS) { + if (expr instanceof StringLiteral && secondary instanceof StringLiteral) { + StringLiteral left = (StringLiteral) expr; + StringLiteral right = (StringLiteral) secondary; + if (left.getQuoteChar() == right.getQuoteChar()) { + return new StringLiteral(left.getValue() + right.getValue(), left.getQuoteChar()); + } + } + } + return new BinaryOperatorExpression(operator, expr, secondary); + } + + private Expression parseExpression() { + return parseExpression(0); + } + + private Expression parseExpression(int prec) { + if (prec >= operatorPrecedence.size()) { + return parsePrimaryWithSuffix(); + } + if (token.kind == TokenKind.NOT && operatorPrecedence.get(prec).contains(Operator.NOT)) { + return parseNotExpression(prec); + } + return parseBinOpExpression(prec); + } + + // not_expr :== 'not' expr + private Expression parseNotExpression(int prec) { + int start = token.left; + expect(TokenKind.NOT); + Expression expression = parseExpression(prec + 1); + NotExpression notExpression = new NotExpression(expression); + return setLocation(notExpression, start, token.right); + } + + // file_input ::= ('\n' | stmt)* EOF + private List<Statement> parseFileInput() { + List<Statement> list = new ArrayList<>(); + while (token.kind != TokenKind.EOF) { + if (token.kind == TokenKind.NEWLINE) { + expect(TokenKind.NEWLINE); + } else { + parseTopLevelStatement(list); + } + } + return list; + } + + // load(STRING (COMMA STRING)*) + private void parseLoad(List<Statement> list) { + int start = token.left; + if (token.kind != TokenKind.STRING) { + expect(TokenKind.STRING); + return; + } + String path = (String) token.value; + nextToken(); + expect(TokenKind.COMMA); + + List<Ident> symbols = new ArrayList<>(); + if (token.kind == TokenKind.STRING) { + symbols.add(new Ident((String) token.value)); + } + expect(TokenKind.STRING); + while (token.kind == TokenKind.COMMA) { + expect(TokenKind.COMMA); + if (token.kind == TokenKind.STRING) { + symbols.add(new Ident((String) token.value)); + } + expect(TokenKind.STRING); + } + expect(TokenKind.RPAREN); + list.add(setLocation(new LoadStatement(path, symbols), start, token.left)); + } + + private void parseTopLevelStatement(List<Statement> list) { + // In Python grammar, there is no "top-level statement" and imports are + // considered as "small statements". We are a bit stricter than Python here. + int start = token.left; + + // Check if there is an include + if (token.kind == TokenKind.IDENTIFIER) { + Token identToken = token; + Ident ident = parseIdent(); + + if (ident.getName().equals("include") && token.kind == TokenKind.LPAREN && !skylarkMode) { + expect(TokenKind.LPAREN); + if (token.kind == TokenKind.STRING) { + include((String) token.value, list, lexer.createLocation(start, token.right)); + } + expect(TokenKind.STRING); + expect(TokenKind.RPAREN); + return; + } else if (ident.getName().equals("load") && token.kind == TokenKind.LPAREN) { + expect(TokenKind.LPAREN); + parseLoad(list); + return; + } + pushToken(identToken); // push the ident back to parse it as a statement + } + parseStatement(list, true); + } + + // simple_stmt ::= small_stmt (';' small_stmt)* ';'? NEWLINE + private void parseSimpleStatement(List<Statement> list) { + list.add(parseSmallStatement()); + + while (token.kind == TokenKind.SEMI) { + nextToken(); + if (token.kind == TokenKind.NEWLINE) { + break; + } + list.add(parseSmallStatement()); + } + expect(TokenKind.NEWLINE); + // This is a safe place to recover: There is a new line at top-level + // and the parser is at the end of a statement. + recoveryMode = false; + } + + // small_stmt ::= assign_stmt + // | expr + // | RETURN expr + // assign_stmt ::= expr ('=' | augassign) expr + // augassign ::= ('+=' ) + // Note that these are in Python, but not implemented here (at least for now): + // '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |'<<=' | '>>=' | '**=' | '//=' + // Semantic difference from Python: + // In Skylark, x += y is simple syntactic sugar for x = x + y. + // In Python, x += y is more or less equivalent to x = x + y, but if a method is defined + // on x.__iadd__(y), then it takes precedence, and in the case of lists it side-effects + // the original list (it doesn't do that on tuples); if no such method is defined it falls back + // to the x.__add__(y) method that backs x + y. In Skylark, we don't support this side-effect. + // Note also that there is a special casing to translate 'ident[key] = value' + // to 'ident = ident + {key: value}'. This is needed to support the pure version of Python-like + // dictionary assignment syntax. + private Statement parseSmallStatement() { + int start = token.left; + if (token.kind == TokenKind.RETURN) { + return parseReturnStatement(); + } + Expression expression = parseExpression(); + if (token.kind == TokenKind.EQUALS) { + nextToken(); + Expression rvalue = parseExpression(); + if (expression instanceof FuncallExpression) { + FuncallExpression func = (FuncallExpression) expression; + if (func.getFunction().getName().equals("$index") && func.getObject() instanceof Ident) { + // Special casing to translate 'ident[key] = value' to 'ident = ident + {key: value}' + // Note that the locations of these extra expressions are fake. + Preconditions.checkArgument(func.getArguments().size() == 1); + DictionaryLiteral dictRValue = setLocation(new DictionaryLiteral(ImmutableList.of( + setLocation(new DictionaryEntryLiteral(func.getArguments().get(0).getValue(), rvalue), + start, token.right))), start, token.right); + BinaryOperatorExpression binExp = setLocation(new BinaryOperatorExpression( + Operator.PLUS, func.getObject(), dictRValue), start, token.right); + return setLocation(new AssignmentStatement(func.getObject(), binExp), start, token.right); + } + } + return setLocation(new AssignmentStatement(expression, rvalue), start, rvalue); + } else if (augmentedAssignmentMethods.containsKey(token.kind)) { + Operator operator = augmentedAssignmentMethods.get(token.kind); + nextToken(); + Expression operand = parseExpression(); + int end = operand.getLocation().getEndOffset(); + return setLocation(new AssignmentStatement(expression, + setLocation(new BinaryOperatorExpression( + operator, expression, operand), start, end)), + start, end); + } else { + return setLocation(new ExpressionStatement(expression), start, expression); + } + } + + // if_stmt ::= IF expr ':' suite [ELIF expr ':' suite]* [ELSE ':' suite]? + private void parseIfStatement(List<Statement> list) { + int start = token.left; + List<ConditionalStatements> thenBlocks = new ArrayList<>(); + thenBlocks.add(parseConditionalStatements(TokenKind.IF)); + while (token.kind == TokenKind.ELIF) { + thenBlocks.add(parseConditionalStatements(TokenKind.ELIF)); + } + List<Statement> elseBlock = new ArrayList<>(); + if (token.kind == TokenKind.ELSE) { + expect(TokenKind.ELSE); + expect(TokenKind.COLON); + parseSuite(elseBlock); + } + Statement stmt = new IfStatement(thenBlocks, elseBlock); + list.add(setLocation(stmt, start, token.right)); + } + + // cond_stmts ::= [EL]IF expr ':' suite + private ConditionalStatements parseConditionalStatements(TokenKind tokenKind) { + int start = token.left; + expect(tokenKind); + Expression expr = parseExpression(); + expect(TokenKind.COLON); + List<Statement> thenBlock = new ArrayList<>(); + parseSuite(thenBlock); + ConditionalStatements stmt = new ConditionalStatements(expr, thenBlock); + return setLocation(stmt, start, token.right); + } + + // for_stmt ::= FOR IDENTIFIER IN expr ':' suite + private void parseForStatement(List<Statement> list) { + int start = token.left; + expect(TokenKind.FOR); + Ident ident = parseIdent(); + expect(TokenKind.IN); + Expression collection = parseExpression(); + expect(TokenKind.COLON); + List<Statement> block = new ArrayList<>(); + parseSuite(block); + Statement stmt = new ForStatement(ident, collection, block); + list.add(setLocation(stmt, start, token.right)); + } + + // def foo(bar1, bar2): + private void parseFunctionDefStatement(List<Statement> list) { + int start = token.left; + expect(TokenKind.DEF); + Ident ident = parseIdent(); + expect(TokenKind.LPAREN); + // parsing the function arguments, at this point only identifiers + // TODO(bazel-team): support proper arguments with default values and kwargs + List<Argument> args = parseFunctionDefArguments(); + expect(TokenKind.RPAREN); + expect(TokenKind.COLON); + List<Statement> block = new ArrayList<>(); + parseSuite(block); + FunctionDefStatement stmt = new FunctionDefStatement(ident, args, block); + list.add(setLocation(stmt, start, token.right)); + } + + private List<Argument> parseFunctionDefArguments() { + List<Argument> args = new ArrayList<>(); + Set<String> argNames = new HashSet<>(); + boolean onlyOptional = false; + while (token.kind != TokenKind.RPAREN) { + Argument arg = parseFunctionDefArgument(onlyOptional); + if (arg.hasValue()) { + onlyOptional = true; + } + args.add(arg); + if (argNames.contains(arg.getArgName())) { + reportError(lexer.createLocation(token.left, token.right), + "duplicate argument name in function definition"); + } + argNames.add(arg.getArgName()); + if (token.kind == TokenKind.COMMA) { + nextToken(); + } else { + break; + } + } + return args; + } + + // suite ::= simple_stmt + // | NEWLINE INDENT stmt+ OUTDENT + private void parseSuite(List<Statement> list) { + if (token.kind == TokenKind.NEWLINE) { + expect(TokenKind.NEWLINE); + if (token.kind != TokenKind.INDENT) { + reportError(lexer.createLocation(token.left, token.right), + "expected an indented block"); + return; + } + expect(TokenKind.INDENT); + while (token.kind != TokenKind.OUTDENT && token.kind != TokenKind.EOF) { + parseStatement(list, false); + } + expect(TokenKind.OUTDENT); + } else { + Statement stmt = parseSmallStatement(); + list.add(stmt); + expect(TokenKind.NEWLINE); + } + } + + // skipSuite does not check that the code is syntactically correct, it + // just skips based on indentation levels. + private void skipSuite() { + if (token.kind == TokenKind.NEWLINE) { + expect(TokenKind.NEWLINE); + if (token.kind != TokenKind.INDENT) { + reportError(lexer.createLocation(token.left, token.right), + "expected an indented block"); + return; + } + expect(TokenKind.INDENT); + + // Don't try to parse all the Python syntax, just skip the block + // until the corresponding outdent token. + int depth = 1; + while (depth > 0) { + // Because of the way the lexer works, this should never happen + Preconditions.checkState(token.kind != TokenKind.EOF); + + if (token.kind == TokenKind.INDENT) { + depth++; + } + if (token.kind == TokenKind.OUTDENT) { + depth--; + } + nextToken(); + } + + } else { + // the block ends at the newline token + // e.g. if x == 3: print "three" + syncTo(STATEMENT_TERMINATOR_SET); + } + } + + // stmt ::= simple_stmt + // | compound_stmt + private void parseStatement(List<Statement> list, boolean isTopLevel) { + if (token.kind == TokenKind.DEF && skylarkMode) { + if (!isTopLevel) { + reportError(lexer.createLocation(token.left, token.right), + "nested functions are not allowed. Move the function to top-level"); + } + parseFunctionDefStatement(list); + } else if (token.kind == TokenKind.IF && skylarkMode) { + parseIfStatement(list); + } else if (token.kind == TokenKind.FOR && skylarkMode) { + if (isTopLevel) { + reportError(lexer.createLocation(token.left, token.right), + "for loops are not allowed on top-level. Put it into a function"); + } + parseForStatement(list); + } else if (token.kind == TokenKind.IF + || token.kind == TokenKind.ELSE + || token.kind == TokenKind.FOR + || token.kind == TokenKind.CLASS + || token.kind == TokenKind.DEF + || token.kind == TokenKind.TRY) { + skipBlock(); + } else { + parseSimpleStatement(list); + } + } + + // return_stmt ::= RETURN expr + private ReturnStatement parseReturnStatement() { + int start = token.left; + expect(TokenKind.RETURN); + Expression expression = parseExpression(); + return setLocation(new ReturnStatement(expression), start, expression); + } + + // block ::= ('if' | 'for' | 'class') expr ':' suite + private void skipBlock() { + int start = token.left; + Token blockToken = token; + syncTo(EnumSet.of(TokenKind.COLON, TokenKind.EOF)); // skip over expression or name + if (!parsePython) { + reportError(lexer.createLocation(start, token.right), "syntax error at '" + + blockToken + "': This Python-style construct is not supported. " + + PREPROCESSING_NEEDED); + } + expect(TokenKind.COLON); + skipSuite(); + } + + // create a comment node + private void makeComment(Token token) { + comments.add(setLocation(new Comment((String) token.value), token.left, token.right)); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ParserInputSource.java b/src/main/java/com/google/devtools/build/lib/syntax/ParserInputSource.java new file mode 100644 index 0000000..488c762 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/ParserInputSource.java
@@ -0,0 +1,112 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.hash.HashCode; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; +import java.io.InputStream; + +/** + * An abstraction for reading input from a file or taking it as a pre-cooked + * char[] or String. + */ +public abstract class ParserInputSource { + + protected ParserInputSource() {} + + /** + * Returns the content of the input source. + */ + public abstract char [] getContent(); + + /** + * Returns the path of the input source. Note: Once constructed, this object + * will never re-read the content from path. + */ + public abstract Path getPath(); + + /** + * Create an input source instance by (eagerly) reading from the file at + * path. The file is assumed to be ISO-8859-1 encoded and smaller than + * 2 Gigs - these assumptions are reasonable for BUILD files, which is + * all we care about here. + */ + public static ParserInputSource create(Path path) throws IOException { + char[] content = FileSystemUtils.readContentAsLatin1(path); + if (path.getFileSize() > content.length) { + // This assertion is to help diagnose problems arising from the + // filesystem; see bugs and #859334 and #920195. + throw new IOException("Unexpected short read from file '" + path + + "' (expected " + path.getFileSize() + ", got " + content.length + " bytes)"); + } + return create(content, path); + } + + /** + * Create an input source from the given content, and associate path with + * this source. Path will be used in error messages etc. but we will *never* + * attempt to read the content from path. + */ + public static ParserInputSource create(String content, Path path) { + return create(content.toCharArray(), path); + } + + /** + * Create an input source from the given content, and associate path with + * this source. Path will be used in error messages etc. but we will *never* + * attempt to read the content from path. + */ + public static ParserInputSource create(final char[] content, final Path path) { + return new ParserInputSource() { + + @Override + public char[] getContent() { + return content; + } + + @Override + public Path getPath() { + return path; + } + }; + } + + /** + * Create an input source from the given input stream, and associate path + * with this source. 'path' will be used in error messages, etc, but will + * not (in general) be used to to read the content from path. + * + * (The exception is the case in which Python pre-processing is required; the + * path will be used to provide the input to the Python pre-processor. + * Arguably, we should just send the content as input to the subprocess + * instead of using the path, but it's not clear it's worth the effort.) + */ + public static ParserInputSource create(InputStream in, Path path) throws IOException { + try { + return create(new String(FileSystemUtils.readContentAsLatin1(in)), path); + } finally { + in.close(); + } + } + + /** + * Returns a hash code calculated from the string content of this file. + */ + public String contentHashCode() throws IOException { + return HashCode.fromBytes(getPath().getMD5Digest()).toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ReturnStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/ReturnStatement.java new file mode 100644 index 0000000..07032c2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/ReturnStatement.java
@@ -0,0 +1,75 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType; + +/** + * A wrapper Statement class for return expressions. + */ +public class ReturnStatement extends Statement { + + /** + * Exception sent by the return statement, to be caught by the function body. + */ + public class ReturnException extends EvalException { + Object value; + + public ReturnException(Location location, Object value) { + super(location, "Return statements must be inside a function"); + this.value = value; + } + + public Object getValue() { + return value; + } + } + + private final Expression returnExpression; + + public ReturnStatement(Expression returnExpression) { + this.returnExpression = returnExpression; + } + + @Override + void exec(Environment env) throws EvalException, InterruptedException { + throw new ReturnException(returnExpression.getLocation(), returnExpression.eval(env)); + } + + Expression getReturnExpression() { + return returnExpression; + } + + @Override + public String toString() { + return "return " + returnExpression; + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + @Override + void validate(ValidationEnvironment env) throws EvalException { + // TODO(bazel-team): save the return type in the environment, to type-check functions. + SkylarkFunctionType fct = env.getCurrentFunction(); + if (fct == null) { + throw new EvalException(getLocation(), "Return statements must be inside a function"); + } + SkylarkType resultType = returnExpression.validate(env); + fct.setReturnType(resultType, getLocation()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SelectorValue.java b/src/main/java/com/google/devtools/build/lib/syntax/SelectorValue.java new file mode 100644 index 0000000..4fb3bdb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/SelectorValue.java
@@ -0,0 +1,45 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import java.util.Map; + +/** + * The value passed to a select({...}) statement, e.g.: + * + * <pre> + * rule( + * name = 'myrule', + * deps = select({ + * 'a': [':adep'], + * 'b': [':bdep'], + * }) + * </pre> + */ +public final class SelectorValue { + Map<?, ?> dictionary; + + public SelectorValue(Map<?, ?> dictionary) { + this.dictionary = dictionary; + } + + public Map<?, ?> getDictionary() { + return dictionary; + } + + @Override + public String toString() { + return "selector({...})"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkBuiltin.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkBuiltin.java new file mode 100644 index 0000000..a2f0d1b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkBuiltin.java
@@ -0,0 +1,61 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +/** + * An annotation to mark built-in keyword argument methods accessible from Skylark. + */ +@Target({ElementType.FIELD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface SkylarkBuiltin { + + String name(); + + String doc(); + + Param[] mandatoryParams() default {}; + + Param[] optionalParams() default {}; + + boolean hidden() default false; + + Class<?> objectType() default Object.class; + + Class<?> returnType() default Object.class; + + boolean onlyLoadingPhase() default false; + + /** + * An annotation for parameters of Skylark built-in functions. + */ + @Retention(RetentionPolicy.RUNTIME) + public @interface Param { + + String name(); + + String doc(); + + Class<?> type() default Object.class; + + Class<?> generic1() default Object.class; + + boolean callbackEnabled() default false; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallable.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallable.java new file mode 100644 index 0000000..ae6987f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallable.java
@@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * A marker interface for Java methods which can be called from Skylark. + */ +@Target({ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface SkylarkCallable { + String name() default ""; + + String doc(); + + boolean hidden() default false; + + boolean structField() default false; + + boolean allowReturnNones() default false; +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallbackFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallbackFunction.java new file mode 100644 index 0000000..2e94be8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallbackFunction.java
@@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.collect.ImmutableList; + + +/** + * A helper class for calling Skylark functions from Java. + */ +public class SkylarkCallbackFunction { + + private final UserDefinedFunction callback; + private final FuncallExpression ast; + private final SkylarkEnvironment funcallEnv; + + public SkylarkCallbackFunction(UserDefinedFunction callback, FuncallExpression ast, + SkylarkEnvironment funcallEnv) { + this.callback = callback; + this.ast = ast; + this.funcallEnv = funcallEnv; + } + + public Object call(ClassObject ctx, Object... arguments) throws EvalException { + try { + return callback.call( + ImmutableList.<Object>builder().add(ctx).add(arguments).build(), null, ast, funcallEnv); + } catch (InterruptedException | ClassCastException + | IllegalArgumentException e) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkEnvironment.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkEnvironment.java new file mode 100644 index 0000000..7e6f414 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkEnvironment.java
@@ -0,0 +1,253 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * The environment for Skylark. + */ +public class SkylarkEnvironment extends Environment { + + /** + * This set contains the variable names of all the successful lookups from the global + * environment. This is necessary because if in a function definition something + * reads a global variable after which a local variable with the same name is assigned an + * Exception needs to be thrown. + */ + private final Set<String> readGlobalVariables = new HashSet<>(); + + private ImmutableList<String> stackTrace; + + @Nullable private String fileContentHashCode; + + /** + * Creates a Skylark Environment for function calling, from the global Environment of the + * caller Environment (which must be a Skylark Environment). + */ + public static SkylarkEnvironment createEnvironmentForFunctionCalling( + Environment callerEnv, SkylarkEnvironment definitionEnv, + UserDefinedFunction function) throws EvalException { + if (callerEnv.getStackTrace().contains(function.getName())) { + throw new EvalException(function.getLocation(), "Recursion was detected when calling '" + + function.getName() + "' from '" + Iterables.getLast(callerEnv.getStackTrace()) + "'"); + } + ImmutableList<String> stackTrace = new ImmutableList.Builder<String>() + .addAll(callerEnv.getStackTrace()) + .add(function.getName()) + .build(); + SkylarkEnvironment childEnv = + // Always use the caller Environment's EventHandler. We cannot assume that the + // definition Environment's EventHandler is still working properly. + new SkylarkEnvironment(definitionEnv, stackTrace, callerEnv.eventHandler); + try { + for (String varname : callerEnv.propagatingVariables) { + childEnv.updateAndPropagate(varname, callerEnv.lookup(varname)); + } + } catch (NoSuchVariableException e) { + // This should never happen. + throw new IllegalStateException(e); + } + childEnv.disabledVariables = callerEnv.disabledVariables; + childEnv.disabledNameSpaces = callerEnv.disabledNameSpaces; + return childEnv; + } + + private SkylarkEnvironment(SkylarkEnvironment definitionEnv, ImmutableList<String> stackTrace, + EventHandler eventHandler) { + super(definitionEnv.getGlobalEnvironment()); + this.stackTrace = stackTrace; + this.eventHandler = Preconditions.checkNotNull(eventHandler, + "EventHandler cannot be null in an Environment which calls into Skylark"); + } + + /** + * Creates a global SkylarkEnvironment. + */ + public SkylarkEnvironment(EventHandler eventHandler, String astFileContentHashCode) { + super(); + stackTrace = ImmutableList.of(); + this.eventHandler = eventHandler; + this.fileContentHashCode = astFileContentHashCode; + } + + @VisibleForTesting + public SkylarkEnvironment(EventHandler eventHandler) { + this(eventHandler, null); + } + + public SkylarkEnvironment(SkylarkEnvironment globalEnv) { + super(globalEnv); + stackTrace = ImmutableList.of(); + this.eventHandler = globalEnv.eventHandler; + } + + @Override + public ImmutableList<String> getStackTrace() { + return stackTrace; + } + + /** + * Clones this Skylark global environment. + */ + public SkylarkEnvironment cloneEnv(EventHandler eventHandler) { + Preconditions.checkArgument(isGlobalEnvironment()); + SkylarkEnvironment newEnv = new SkylarkEnvironment(eventHandler, this.fileContentHashCode); + for (Entry<String, Object> entry : env.entrySet()) { + newEnv.env.put(entry.getKey(), entry.getValue()); + } + for (Map.Entry<Class<?>, Map<String, Function>> functionMap : functions.entrySet()) { + newEnv.functions.put(functionMap.getKey(), functionMap.getValue()); + } + return newEnv; + } + + /** + * Returns the global environment. Only works for Skylark environments. For the global Skylark + * environment this method returns this Environment. + */ + public SkylarkEnvironment getGlobalEnvironment() { + // If there's a parent that's the global environment, otherwise this is. + return parent != null ? (SkylarkEnvironment) parent : this; + } + + /** + * Returns true if this is a Skylark global environment. + */ + public boolean isGlobalEnvironment() { + return parent == null; + } + + /** + * Returns true if varname has been read as a global variable. + */ + public boolean hasBeenReadGlobalVariable(String varname) { + return readGlobalVariables.contains(varname); + } + + @Override + public boolean isSkylarkEnabled() { + return true; + } + + /** + * @return the value from the environment whose name is "varname". + * @throws NoSuchVariableException if the variable is not defined in the environment. + */ + @Override + public Object lookup(String varname) throws NoSuchVariableException { + if (disabledVariables.contains(varname)) { + throw new NoSuchVariableException(varname); + } + Object value = env.get(varname); + if (value == null) { + if (parent != null && parent.hasVariable(varname)) { + readGlobalVariables.add(varname); + return parent.lookup(varname); + } + throw new NoSuchVariableException(varname); + } + return value; + } + + /** + * Like <code>lookup(String)</code>, but instead of throwing an exception in + * the case where "varname" is not defined, "defaultValue" is returned instead. + */ + @Override + public Object lookup(String varname, Object defaultValue) { + throw new UnsupportedOperationException(); + } + + /** + * Updates the value of variable "varname" in the environment, corresponding + * to an AssignmentStatement. + */ + @Override + public void update(String varname, Object value) { + Preconditions.checkNotNull(value, "update(value == null)"); + env.put(varname, value); + } + + /** + * Returns the class of the variable or null if the variable does not exist. This function + * works only in the local Environment, it doesn't check the global Environment. + */ + public Class<?> getVariableType(String varname) { + Object variable = env.get(varname); + return variable != null ? EvalUtils.getSkylarkType(variable.getClass()) : null; + } + + /** + * Removes the functions and the modules (i.e. the symbol of the module from the top level + * Environment and the functions attached to it) from the Environment which should be present + * only during the loading phase. + */ + public void disableOnlyLoadingPhaseObjects() { + List<String> objectsToRemove = new ArrayList<>(); + List<Class<?>> modulesToRemove = new ArrayList<>(); + for (Map.Entry<String, Object> entry : env.entrySet()) { + Object object = entry.getValue(); + if (object instanceof SkylarkFunction) { + if (((SkylarkFunction) object).isOnlyLoadingPhase()) { + objectsToRemove.add(entry.getKey()); + } + } else if (object.getClass().isAnnotationPresent(SkylarkModule.class)) { + if (object.getClass().getAnnotation(SkylarkModule.class).onlyLoadingPhase()) { + objectsToRemove.add(entry.getKey()); + modulesToRemove.add(entry.getValue().getClass()); + } + } + } + for (String symbol : objectsToRemove) { + disabledVariables.add(symbol); + } + for (Class<?> moduleClass : modulesToRemove) { + disabledNameSpaces.add(moduleClass); + } + } + + public void handleEvent(Event event) { + eventHandler.handle(event); + } + + /** + * Returns a hash code calculated from the hash code of this Environment and the + * transitive closure of other Environments it loads. + */ + public String getTransitiveFileContentHashCode() { + Fingerprint fingerprint = new Fingerprint(); + fingerprint.addString(Preconditions.checkNotNull(fileContentHashCode)); + // Calculate a new hash from the hash of the loaded Environments. + for (SkylarkEnvironment env : importedExtensions.values()) { + fingerprint.addString(env.getTransitiveFileContentHashCode()); + } + return fingerprint.hexDigestAndReset(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkFunction.java new file mode 100644 index 0000000..bd2cc83 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkFunction.java
@@ -0,0 +1,317 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.packages.Type.ConversionException; +import com.google.devtools.build.lib.syntax.EvalException.EvalExceptionWithJavaCause; + +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ExecutionException; + +/** + * A function class for Skylark built in functions. Supports mandatory and optional arguments. + * All usable arguments have to be specified. In case of ambiguous arguments (a parameter is + * specified as positional and keyword arguments in the function call) an exception is thrown. + */ +public abstract class SkylarkFunction extends AbstractFunction { + + private ImmutableList<String> parameters; + private ImmutableMap<String, SkylarkBuiltin.Param> parameterTypes; + private int mandatoryParamNum; + private boolean configured = false; + private Class<?> objectType; + private boolean onlyLoadingPhase; + + /** + * Creates a SkylarkFunction with the given name. + */ + public SkylarkFunction(String name) { + super(name); + } + + /** + * Configures the parameter of this Skylark function using the annotation. + */ + @VisibleForTesting + public void configure(SkylarkBuiltin annotation) { + Preconditions.checkState(!configured); + Preconditions.checkArgument(getName().equals(annotation.name()), + getName() + " != " + annotation.name()); + mandatoryParamNum = 0; + ImmutableList.Builder<String> paramListBuilder = ImmutableList.builder(); + ImmutableMap.Builder<String, SkylarkBuiltin.Param> paramTypeBuilder = ImmutableMap.builder(); + for (SkylarkBuiltin.Param param : annotation.mandatoryParams()) { + paramListBuilder.add(param.name()); + paramTypeBuilder.put(param.name(), param); + mandatoryParamNum++; + } + for (SkylarkBuiltin.Param param : annotation.optionalParams()) { + paramListBuilder.add(param.name()); + paramTypeBuilder.put(param.name(), param); + } + parameters = paramListBuilder.build(); + parameterTypes = paramTypeBuilder.build(); + this.objectType = annotation.objectType().equals(Object.class) ? null : annotation.objectType(); + this.onlyLoadingPhase = annotation.onlyLoadingPhase(); + configured = true; + } + + /** + * Returns true if the SkylarkFunction is configured. + */ + public boolean isConfigured() { + return configured; + } + + @Override + public Class<?> getObjectType() { + return objectType; + } + + public boolean isOnlyLoadingPhase() { + return onlyLoadingPhase; + } + + @Override + public Object call(List<Object> args, + Map<String, Object> kwargs, + FuncallExpression ast, + Environment env) + throws EvalException, InterruptedException { + + Preconditions.checkState(configured, "Function " + getName() + " was not configured"); + try { + ImmutableMap.Builder<String, Object> arguments = new ImmutableMap.Builder<>(); + if (objectType != null && !FuncallExpression.isNamespace(objectType)) { + arguments.put("self", args.remove(0)); + } + + int maxParamNum = parameters.size(); + int paramNum = args.size() + kwargs.size(); + + if (paramNum < mandatoryParamNum) { + throw new EvalException(ast.getLocation(), + String.format("incorrect number of arguments (got %s, expected at least %s)", + paramNum, mandatoryParamNum)); + } else if (paramNum > maxParamNum) { + throw new EvalException(ast.getLocation(), + String.format("incorrect number of arguments (got %s, expected at most %s)", + paramNum, maxParamNum)); + } + + for (int i = 0; i < mandatoryParamNum; i++) { + Preconditions.checkState(i < args.size() || kwargs.containsKey(parameters.get(i)), + String.format("missing mandatory parameter: %s", parameters.get(i))); + } + + for (int i = 0; i < args.size(); i++) { + checkTypeAndAddArg(parameters.get(i), args.get(i), arguments, ast.getLocation()); + } + + for (Entry<String, Object> kwarg : kwargs.entrySet()) { + int idx = parameters.indexOf(kwarg.getKey()); + if (idx < 0) { + throw new EvalException(ast.getLocation(), + String.format("unknown keyword argument: %s", kwarg.getKey())); + } + if (idx < args.size()) { + throw new EvalException(ast.getLocation(), + String.format("ambiguous argument: %s", kwarg.getKey())); + } + checkTypeAndAddArg(kwarg.getKey(), kwarg.getValue(), arguments, ast.getLocation()); + } + + return call(arguments.build(), ast, env); + } catch (ConversionException | IllegalArgumentException | IllegalStateException + | ClassCastException | ClassNotFoundException | ExecutionException e) { + if (e.getMessage() != null) { + throw new EvalException(ast.getLocation(), e.getMessage()); + } else { + // TODO(bazel-team): ideally this shouldn't happen, however we need this for debugging + throw new EvalExceptionWithJavaCause(ast.getLocation(), e); + } + } + } + + private void checkTypeAndAddArg(String paramName, Object value, + ImmutableMap.Builder<String, Object> arguments, Location loc) throws EvalException { + SkylarkBuiltin.Param param = parameterTypes.get(paramName); + if (param.callbackEnabled() && Function.class.isAssignableFrom(value.getClass())) { + // If we pass a function as an argument we trust the Function implementation with the type + // check. It's OK since the function needs to be called manually anyway. + arguments.put(paramName, value); + return; + } + if (!(param.type().isAssignableFrom(value.getClass()))) { + throw new EvalException(loc, String.format("expected %s for '%s' but got %s instead\n" + + "%s.%s: %s", + EvalUtils.getDataTypeNameFromClass(param.type()), paramName, + EvalUtils.getDatatypeName(value), getName(), paramName, param.doc())); + } + if (param.type().equals(SkylarkList.class)) { + checkGeneric(paramName, param, value, ((SkylarkList) value).getGenericType(), loc); + } else if (param.type().equals(SkylarkNestedSet.class)) { + checkGeneric(paramName, param, value, ((SkylarkNestedSet) value).getGenericType(), loc); + } + arguments.put(paramName, value); + } + + private void checkGeneric(String paramName, SkylarkBuiltin.Param param, Object value, + Class<?> genericType, Location loc) throws EvalException { + if (!genericType.equals(Object.class) && !param.generic1().isAssignableFrom(genericType)) { + String mainType = EvalUtils.getDataTypeNameFromClass(param.type()); + throw new EvalException(loc, String.format( + "expected %s of %ss for '%s' but got %s of %ss instead\n%s.%s: %s", + mainType, EvalUtils.getDataTypeNameFromClass(param.generic1()), + paramName, + EvalUtils.getDatatypeName(value), EvalUtils.getDataTypeNameFromClass(genericType), + getName(), paramName, param.doc())); + } + } + + /** + * The actual function call. All positional and keyword arguments are put in the + * arguments map. + */ + protected abstract Object call( + Map<String, Object> arguments, FuncallExpression ast, Environment env) throws EvalException, + ConversionException, + IllegalArgumentException, + IllegalStateException, + ClassCastException, + ClassNotFoundException, + ExecutionException; + + /** + * An intermediate class to provide a simpler interface for Skylark functions. + */ + public abstract static class SimpleSkylarkFunction extends SkylarkFunction { + + public SimpleSkylarkFunction(String name) { + super(name); + } + + @Override + protected final Object call( + Map<String, Object> arguments, FuncallExpression ast, Environment env) throws EvalException, + ConversionException, + IllegalArgumentException, + IllegalStateException, + ClassCastException, + ExecutionException { + return call(arguments, ast.getLocation()); + } + + /** + * The actual function call. All positional and keyword arguments are put in the + * arguments map. + */ + protected abstract Object call(Map<String, Object> arguments, Location loc) + throws EvalException, + ConversionException, + IllegalArgumentException, + IllegalStateException, + ClassCastException, + ExecutionException; + } + + public static <TYPE> Iterable<TYPE> castList(Object obj, final Class<TYPE> type) { + if (obj == null) { + return ImmutableList.of(); + } + return ((SkylarkList) obj).to(type); + } + + public static <TYPE> Iterable<TYPE> castList( + Object obj, final Class<TYPE> type, final String what) throws ConversionException { + if (obj == null) { + return ImmutableList.of(); + } + return Iterables.transform(Type.LIST.convert(obj, what), + new com.google.common.base.Function<Object, TYPE>() { + @Override + public TYPE apply(Object input) { + try { + return type.cast(input); + } catch (ClassCastException e) { + throw new IllegalArgumentException(String.format( + "expected %s type for '%s' but got %s instead", + EvalUtils.getDataTypeNameFromClass(type), what, + EvalUtils.getDatatypeName(input))); + } + } + }); + } + + public static <KEY_TYPE, VALUE_TYPE> ImmutableMap<KEY_TYPE, VALUE_TYPE> toMap( + Iterable<Map.Entry<KEY_TYPE, VALUE_TYPE>> obj) { + ImmutableMap.Builder<KEY_TYPE, VALUE_TYPE> builder = ImmutableMap.builder(); + for (Map.Entry<KEY_TYPE, VALUE_TYPE> entry : obj) { + builder.put(entry.getKey(), entry.getValue()); + } + return builder.build(); + } + + public static <KEY_TYPE, VALUE_TYPE> Iterable<Map.Entry<KEY_TYPE, VALUE_TYPE>> castMap(Object obj, + final Class<KEY_TYPE> keyType, final Class<VALUE_TYPE> valueType, final String what) { + if (obj == null) { + return ImmutableList.of(); + } + if (!(obj instanceof Map<?, ?>)) { + throw new IllegalArgumentException(String.format( + "expected a dictionary for %s but got %s instead", + what, EvalUtils.getDatatypeName(obj))); + } + return Iterables.transform(((Map<?, ?>) obj).entrySet(), + new com.google.common.base.Function<Map.Entry<?, ?>, Map.Entry<KEY_TYPE, VALUE_TYPE>>() { + // This is safe. We check the type of the key-value pairs for every entry in the Map. + // In Map.Entry the key always has the type of the first generic parameter, the + // value has the second. + @SuppressWarnings("unchecked") + @Override + public Map.Entry<KEY_TYPE, VALUE_TYPE> apply(Map.Entry<?, ?> input) { + if (keyType.isAssignableFrom(input.getKey().getClass()) + && valueType.isAssignableFrom(input.getValue().getClass())) { + return (Map.Entry<KEY_TYPE, VALUE_TYPE>) input; + } + throw new IllegalArgumentException(String.format( + "expected <%s, %s> type for '%s' but got <%s, %s> instead", + keyType.getSimpleName(), valueType.getSimpleName(), what, + EvalUtils.getDatatypeName(input.getKey()), + EvalUtils.getDatatypeName(input.getValue()))); + } + }); + } + + // TODO(bazel-team): this is only used in SkylarkRuleConfgiuredTargetBuilder, fix typing for + // structs then remove this. + public static <TYPE> TYPE cast(Object elem, Class<TYPE> type, String what, Location loc) + throws EvalException { + try { + return type.cast(elem); + } catch (ClassCastException e) { + throw new EvalException(loc, String.format("expected %s for '%s' but got %s instead", + type.getSimpleName(), what, EvalUtils.getDatatypeName(elem))); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkList.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkList.java new file mode 100644 index 0000000..ef9fe10 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkList.java
@@ -0,0 +1,373 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.events.Location; + +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +/** + * A class to handle lists and tuples in Skylark. + */ +@SkylarkModule(name = "list", + doc = "A language built-in type to support lists. Example of list literal:<br>" + + "<pre class=language-python>l = [1, 2, 3]</pre>" + + "Accessing elements is possible using indexing (starts from <code>0</code>):<br>" + + "<pre class=language-python>e = l[1] # e == 2</pre>" + + "Lists support the <code>+</code> operator to concatenate two lists. Example:<br>" + + "<pre class=language-python>l = [1, 2] + [3, 4] # l == [1, 2, 3, 4]\n" + + "l = [\"a\", \"b\"]\n" + + "l += [\"c\"] # l == [\"a\", \"b\", \"c\"]</pre>" + + "List elements have to be of the same type, <code>[1, 2, \"c\"]</code> results in an " + + "error. Lists - just like everything - are immutable, therefore <code>l[1] = \"a\"" + + "</code> is not supported.") +public abstract class SkylarkList implements Iterable<Object> { + + private final boolean tuple; + private final Class<?> genericType; + + private SkylarkList(boolean tuple, Class<?> genericType) { + this.tuple = tuple; + this.genericType = genericType; + } + + /** + * The size of the list. + */ + public abstract int size(); + + /** + * Returns true if the list is empty. + */ + public abstract boolean isEmpty(); + + /** + * Returns the i-th element of the list. + */ + public abstract Object get(int i); + + /** + * Returns true if this list is a tuple. + */ + public boolean isTuple() { + return tuple; + } + + @VisibleForTesting + public Class<?> getGenericType() { + return genericType; + } + + @Override + public String toString() { + return toList().toString(); + } + + // TODO(bazel-team): we should be very careful using this method. Check and remove + // auto conversions on the Java-Skylark interface if possible. + /** + * Converts this Skylark list to a Java list. + */ + public abstract List<?> toList(); + + @SuppressWarnings("unchecked") + public <T> Iterable<T> to(Class<T> type) { + Preconditions.checkArgument(this == EMPTY_LIST || type.isAssignableFrom(genericType)); + return (Iterable<T>) this; + } + + private static final class EmptySkylarkList extends SkylarkList { + private EmptySkylarkList(boolean tuple) { + super(tuple, Object.class); + } + + @Override + public Iterator<Object> iterator() { + return ImmutableList.of().iterator(); + } + + @Override + public int size() { + return 0; + } + + @Override + public boolean isEmpty() { + return true; + } + + @Override + public Object get(int i) { + throw new UnsupportedOperationException(); + } + + @Override + public List<?> toList() { + return isTuple() ? ImmutableList.of() : Lists.newArrayList(); + } + + @Override + public String toString() { + return "[]"; + } + } + + /** + * An empty Skylark list. + */ + public static final SkylarkList EMPTY_LIST = new EmptySkylarkList(false); + + private static final class SimpleSkylarkList extends SkylarkList { + private final ImmutableList<Object> list; + + private SimpleSkylarkList(ImmutableList<Object> list, boolean tuple, Class<?> genericType) { + super(tuple, genericType); + this.list = Preconditions.checkNotNull(list); + } + + @Override + public Iterator<Object> iterator() { + return list.iterator(); + } + + @Override + public int size() { + return list.size(); + } + + @Override + public boolean isEmpty() { + return list.isEmpty(); + } + + @Override + public Object get(int i) { + return list.get(i); + } + + @Override + public List<?> toList() { + return isTuple() ? list : Lists.newArrayList(list); + } + + @Override + public String toString() { + return list.toString(); + } + + @Override + public int hashCode() { + return list.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof SimpleSkylarkList)) { + return false; + } + SimpleSkylarkList other = (SimpleSkylarkList) obj; + return other.list.equals(this.list); + } + } + + /** + * A Skylark list to support lazy iteration (i.e. we only call iterator on the object this + * list masks when it's absolutely necessary). This is useful if iteration is expensive + * (e.g. NestedSet-s). Size(), get() and isEmpty() are expensive operations but + * concatenation is quick. + */ + private static final class LazySkylarkList extends SkylarkList { + private final Iterable<Object> iterable; + private ImmutableList<Object> list = null; + + private LazySkylarkList(Iterable<Object> iterable, boolean tuple, Class<?> genericType) { + super(tuple, genericType); + this.iterable = Preconditions.checkNotNull(iterable); + } + + @Override + public Iterator<Object> iterator() { + return iterable.iterator(); + } + + @Override + public int size() { + return Iterables.size(iterable); + } + + @Override + public boolean isEmpty() { + return Iterables.isEmpty(iterable); + } + + @Override + public Object get(int i) { + return getList().get(i); + } + + @Override + public List<?> toList() { + return getList(); + } + + private ImmutableList<Object> getList() { + if (list == null) { + list = ImmutableList.copyOf(iterable); + } + return list; + } + } + + /** + * A Skylark list to support quick concatenation of lists. Concatenation is O(1), + * size(), isEmpty() is O(n), get() is O(h). + */ + private static final class ConcatenatedSkylarkList extends SkylarkList { + private final SkylarkList left; + private final SkylarkList right; + + private ConcatenatedSkylarkList( + SkylarkList left, SkylarkList right, boolean tuple, Class<?> genericType) { + super(tuple, genericType); + this.left = Preconditions.checkNotNull(left); + this.right = Preconditions.checkNotNull(right); + } + + @Override + public Iterator<Object> iterator() { + return Iterables.concat(left, right).iterator(); + } + + @Override + public int size() { + // We shouldn't evaluate the size function until it's necessary, because it can be expensive + // for lazy lists (e.g. lists containing a NestedSet). + // TODO(bazel-team): make this class more clever to store the size and empty parameters + // for every non-LazySkylarkList member. + return left.size() + right.size(); + } + + @Override + public boolean isEmpty() { + return left.isEmpty() && right.isEmpty(); + } + + @Override + public Object get(int i) { + int leftSize = left.size(); + if (i < leftSize) { + return left.get(i); + } else { + return right.get(i - leftSize); + } + } + + @Override + public List<?> toList() { + return ImmutableList.<Object>builder().addAll(left).addAll(right).build(); + } + } + + /** + * Returns a Skylark list containing elements without a type check. Only use if all elements + * are of the same type. + */ + public static SkylarkList list(Collection<?> elements, Class<?> genericType) { + if (elements.isEmpty()) { + return EMPTY_LIST; + } + return new SimpleSkylarkList(ImmutableList.copyOf(elements), false, genericType); + } + + /** + * Returns a Skylark list containing elements without a type check and without creating + * an immutable copy. Therefore the iterable containing elements must be immutable + * (which is not checked here so callers must be extra careful). This way + * it's possibly to create a SkylarkList without requesting the original iterator. This + * can be useful for nested set - list conversions. + */ + @SuppressWarnings("unchecked") + public static SkylarkList lazyList(Iterable<?> elements, Class<?> genericType) { + return new LazySkylarkList((Iterable<Object>) elements, false, genericType); + } + + /** + * Returns a Skylark list containing elements. Performs type check and throws an exception + * in case the list contains elements of different type. + */ + public static SkylarkList list(Collection<?> elements, Location loc) throws EvalException { + if (elements.isEmpty()) { + return EMPTY_LIST; + } + return new SimpleSkylarkList( + ImmutableList.copyOf(elements), false, getGenericType(elements, loc)); + } + + private static Class<?> getGenericType(Collection<?> elements, Location loc) + throws EvalException { + Class<?> genericType = elements.iterator().next().getClass(); + for (Object element : elements) { + Class<?> type = element.getClass(); + if (!EvalUtils.getSkylarkType(genericType).equals(EvalUtils.getSkylarkType(type))) { + throw new EvalException(loc, String.format( + "Incompatible types in list: found a %s but the first element is a %s", + EvalUtils.getDataTypeNameFromClass(type), + EvalUtils.getDataTypeNameFromClass(genericType))); + } + } + return genericType; + } + + /** + * Returns a Skylark list created from Skylark lists left and right. Throws an exception + * if they are not of the same generic type. + */ + public static SkylarkList concat(SkylarkList left, SkylarkList right, Location loc) + throws EvalException { + if (left.isTuple() != right.isTuple()) { + throw new EvalException(loc, "cannot concatenate lists and tuples"); + } + if (left == EMPTY_LIST) { + return right; + } + if (right == EMPTY_LIST) { + return left; + } + if (!left.genericType.equals(right.genericType)) { + throw new EvalException(loc, String.format("cannot concatenate list of %s with list of %s", + EvalUtils.getDataTypeNameFromClass(left.genericType), + EvalUtils.getDataTypeNameFromClass(right.genericType))); + } + return new ConcatenatedSkylarkList(left, right, left.isTuple(), left.genericType); + } + + /** + * Returns a Skylark tuple containing elements. + */ + public static SkylarkList tuple(List<?> elements) { + // Tuple elements do not have to have the same type. + return new SimpleSkylarkList(ImmutableList.copyOf(elements), true, Object.class); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkModule.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkModule.java new file mode 100644 index 0000000..96421b2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkModule.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * An annotation to mark Skylark modules or Skylark accessible Java data types. + * A Skylark modules always corresponds to exactly one Java class. + */ +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +public @interface SkylarkModule { + + String name(); + + String doc(); + + boolean hidden() default false; + + boolean namespace() default false; + + boolean onlyLoadingPhase() default false; +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkNestedSet.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkNestedSet.java new file mode 100644 index 0000000..17fc55f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkNestedSet.java
@@ -0,0 +1,193 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.Location; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * A generic type safe NestedSet wrapper for Skylark. + */ +@SkylarkModule(name = "set", + doc = "A language built-in type to supports (nested) sets. " + + "Sets can be created using the global <code>set</code> function, and they " + + "support the <code>+</code> operator to extends and nest sets. Examples:<br>" + + "<pre class=language-python>s = set([1, 2])\n" + + "s += [3] # s == {1, 2, 3}\n" + + "s += set([4, 5]) # s == {1, 2, 3, {4, 5}}</pre>" + + "Note that in these examples <code>{..}</code> is not a valid literal to create sets. " + + "Sets have a fixed generic type, so <code>set([1]) + [\"a\"]</code> or " + + "<code>set([1]) + set([\"a\"])</code> results in an error.") +@Immutable +public final class SkylarkNestedSet implements Iterable<Object> { + + private final Class<?> genericType; + @Nullable private final List<Object> items; + @Nullable private final List<NestedSet<Object>> transitiveItems; + private final NestedSet<?> set; + + public SkylarkNestedSet(Order order, Object item, Location loc) throws EvalException { + this(order, Object.class, item, loc, new ArrayList<Object>(), + new ArrayList<NestedSet<Object>>()); + } + + public SkylarkNestedSet(SkylarkNestedSet left, Object right, Location loc) throws EvalException { + this(left.set.getOrder(), left.genericType, right, loc, + new ArrayList<Object>(checkItems(left.items, loc)), + new ArrayList<NestedSet<Object>>(checkItems(left.transitiveItems, loc))); + } + + private static <T> T checkItems(T items, Location loc) throws EvalException { + // SkylarkNestedSets created directly from ordinary NestedSets (those were created in a + // native rule) don't have directly accessible items and transitiveItems, so we cannot + // add more elements to them. + if (items == null) { + throw new EvalException(loc, "Cannot add more elements to this set. Sets created in " + + "native rules cannot be left side operands of the + operator."); + } + return items; + } + + // This is safe because of the type checking + @SuppressWarnings("unchecked") + private SkylarkNestedSet(Order order, Class<?> genericType, Object item, Location loc, + List<Object> items, List<NestedSet<Object>> transitiveItems) throws EvalException { + + // Adding the item + if (item instanceof SkylarkNestedSet) { + SkylarkNestedSet nestedSet = (SkylarkNestedSet) item; + if (!nestedSet.isEmpty()) { + genericType = checkType(genericType, nestedSet.genericType, loc); + transitiveItems.add((NestedSet<Object>) nestedSet.set); + } + } else if (item instanceof SkylarkList) { + // TODO(bazel-team): we should check ImmutableList here but it screws up genrule at line 43 + for (Object object : (SkylarkList) item) { + genericType = checkType(genericType, object.getClass(), loc); + items.add(object); + } + } else { + throw new EvalException(loc, + String.format("cannot add '%s'-s to nested sets", EvalUtils.getDatatypeName(item))); + } + this.genericType = Preconditions.checkNotNull(genericType, "type cannot be null"); + + // Initializing the real nested set + NestedSetBuilder<Object> builder = new NestedSetBuilder<Object>(order); + builder.addAll(items); + try { + for (NestedSet<Object> nestedSet : transitiveItems) { + builder.addTransitive(nestedSet); + } + } catch (IllegalStateException e) { + throw new EvalException(loc, e.getMessage()); + } + this.set = builder.build(); + this.items = ImmutableList.copyOf(items); + this.transitiveItems = ImmutableList.copyOf(transitiveItems); + } + + /** + * Returns a type safe SkylarkNestedSet. Use this instead of the constructor if possible. + */ + public static <T> SkylarkNestedSet of(Class<T> genericType, NestedSet<T> set) { + return new SkylarkNestedSet(genericType, set); + } + + /** + * A not type safe constructor for SkylarkNestedSet. It's discouraged to use it unless type + * generic safety is guaranteed from the caller side. + */ + SkylarkNestedSet(Class<?> genericType, NestedSet<?> set) { + // This is here for the sake of FuncallExpression. + this.genericType = Preconditions.checkNotNull(genericType, "type cannot be null"); + this.set = Preconditions.checkNotNull(set, "set cannot be null"); + this.items = null; + this.transitiveItems = null; + } + + private static Class<?> checkType(Class<?> builderType, Class<?> itemType, Location loc) + throws EvalException { + if (Map.class.isAssignableFrom(itemType) || SkylarkList.class.isAssignableFrom(itemType) + || ClassObject.class.isAssignableFrom(itemType)) { + throw new EvalException(loc, String.format("nested set item is composite (type of %s)", + EvalUtils.getDataTypeNameFromClass(itemType))); + } + if (!EvalUtils.isSkylarkImmutable(itemType)) { + throw new EvalException(loc, String.format("nested set item is not immutable (type of %s)", + EvalUtils.getDataTypeNameFromClass(itemType))); + } + if (builderType.equals(Object.class)) { + return itemType; + } + if (!EvalUtils.getSkylarkType(builderType).equals(EvalUtils.getSkylarkType(itemType))) { + throw new EvalException(loc, String.format( + "nested set item is type of %s but the nested set accepts only %s-s", + EvalUtils.getDataTypeNameFromClass(itemType), + EvalUtils.getDataTypeNameFromClass(builderType))); + } + return builderType; + } + + /** + * Returns the NestedSet embedded in this SkylarkNestedSet if it is of the parameter type. + */ + // The precondition ensures generic type safety + @SuppressWarnings("unchecked") + public <T> NestedSet<T> getSet(Class<T> type) { + // Empty sets don't need have to have a type since they don't have items + if (set.isEmpty()) { + return (NestedSet<T>) set; + } + Preconditions.checkArgument(type.isAssignableFrom(genericType), + String.format("Expected %s as a type but got %s", + EvalUtils.getDataTypeNameFromClass(type), + EvalUtils.getDataTypeNameFromClass(genericType))); + return (NestedSet<T>) set; + } + + // For some reason this cast is unsafe in Java + @SuppressWarnings("unchecked") + @Override + public Iterator<Object> iterator() { + return (Iterator<Object>) set.iterator(); + } + + public Collection<Object> toCollection() { + return ImmutableList.copyOf(set.toCollection()); + } + + public boolean isEmpty() { + return set.isEmpty(); + } + + @VisibleForTesting + public Class<?> getGenericType() { + return genericType; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkType.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkType.java new file mode 100644 index 0000000..04c345f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkType.java
@@ -0,0 +1,307 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.events.Location; + +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.lang.reflect.WildcardType; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A class representing types available in Skylark. + */ +public class SkylarkType { + + private static final class Global {} + + public static final SkylarkType UNKNOWN = new SkylarkType(Object.class); + public static final SkylarkType NONE = new SkylarkType(Environment.NoneType.class); + public static final SkylarkType GLOBAL = new SkylarkType(Global.class); + + public static final SkylarkType STRING = new SkylarkType(String.class); + public static final SkylarkType INT = new SkylarkType(Integer.class); + public static final SkylarkType BOOL = new SkylarkType(Boolean.class); + + private final Class<?> type; + + // TODO(bazel-team): Change this to SkylarkType and check generics of generics etc. + // Object.class is used for UNKNOWN. + private Class<?> generic1; + + public static SkylarkType of(Class<?> type, Class<?> generic1) { + return new SkylarkType(type, generic1); + } + + public static SkylarkType of(Class<?> type) { + if (type.equals(Object.class)) { + return SkylarkType.UNKNOWN; + } else if (type.equals(String.class)) { + return SkylarkType.STRING; + } else if (type.equals(Integer.class)) { + return SkylarkType.INT; + } else if (type.equals(Boolean.class)) { + return SkylarkType.BOOL; + } + return new SkylarkType(type); + } + + private SkylarkType(Class<?> type, Class<?> generic1) { + this.type = Preconditions.checkNotNull(type); + this.generic1 = Preconditions.checkNotNull(generic1); + } + + private SkylarkType(Class<?> type) { + this.type = Preconditions.checkNotNull(type); + this.generic1 = Object.class; + } + + public Class<?> getType() { + return type; + } + + Class<?> getGenericType1() { + return generic1; + } + + /** + * Returns the stronger type of this and o if they are compatible. Stronger means that + * the more information is available, e.g. STRING is stronger than UNKNOWN and + * LIST<STRING> is stronger than LIST<UNKNOWN>. Note than there's no type + * hierarchy in Skylark. + * <p>If they are not compatible an EvalException is thrown. + */ + SkylarkType infer(SkylarkType o, String name, Location thisLoc, Location originalLoc) + throws EvalException { + if (this == o) { + return this; + } + if (this == UNKNOWN || this.equals(SkylarkType.NONE)) { + return o; + } + if (o == UNKNOWN || o.equals(SkylarkType.NONE)) { + return this; + } + if (!type.equals(o.type)) { + throw new EvalException(thisLoc, String.format("bad %s: %s is incompatible with %s at %s", + name, + EvalUtils.getDataTypeNameFromClass(o.getType()), + EvalUtils.getDataTypeNameFromClass(this.getType()), + originalLoc)); + } + if (generic1.equals(Object.class)) { + return o; + } + if (o.generic1.equals(Object.class)) { + return this; + } + if (!generic1.equals(o.generic1)) { + throw new EvalException(thisLoc, String.format("bad %s: incompatible generic variable types " + + "%s with %s", + name, + EvalUtils.getDataTypeNameFromClass(o.generic1), + EvalUtils.getDataTypeNameFromClass(this.generic1))); + } + return this; + } + + boolean isStruct() { + return type.equals(ClassObject.class); + } + + boolean isList() { + return SkylarkList.class.isAssignableFrom(type); + } + + boolean isDict() { + return Map.class.isAssignableFrom(type); + } + + boolean isSet() { + return Set.class.isAssignableFrom(type); + } + + boolean isNset() { + // TODO(bazel-team): NestedSets are going to be a bit strange with 2 type info (validation + // and execution time). That can be cleaned up once we have complete type inference. + return SkylarkNestedSet.class.isAssignableFrom(type); + } + + boolean isSimple() { + return !isStruct() && !isDict() && !isList() && !isNset() && !isSet(); + } + + @Override + public String toString() { + return this == UNKNOWN ? "Unknown" : EvalUtils.getDataTypeNameFromClass(type); + } + + // hashCode() and equals() only uses the type field + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof SkylarkType)) { + return false; + } + SkylarkType o = (SkylarkType) other; + return this.type.equals(o.type); + } + + @Override + public int hashCode() { + return type.hashCode(); + } + + /** + * A class representing the type of a Skylark function. + */ + public static final class SkylarkFunctionType extends SkylarkType { + + private final String name; + @Nullable private SkylarkType returnType; + @Nullable private Location returnTypeLoc; + + public static SkylarkFunctionType of(String name) { + return new SkylarkFunctionType(name, null); + } + + public static SkylarkFunctionType of(String name, SkylarkType returnType) { + return new SkylarkFunctionType(name, returnType); + } + + private SkylarkFunctionType(String name, SkylarkType returnType) { + super(Function.class); + this.name = name; + this.returnType = returnType; + } + + public SkylarkType getReturnType() { + return returnType; + } + + /** + * Sets the return type of the function type if it's compatible with the existing return type. + * Note that setting NONE only has an effect if the return type hasn't been set previously. + */ + public void setReturnType(SkylarkType newReturnType, Location newLoc) throws EvalException { + if (returnType == null) { + returnType = newReturnType; + returnTypeLoc = newLoc; + } else if (newReturnType != SkylarkType.NONE) { + returnType = + returnType.infer(newReturnType, "return type of " + name, newLoc, returnTypeLoc); + if (returnType == newReturnType) { + returnTypeLoc = newLoc; + } + } + } + } + + private static boolean isTypeAllowedInSkylark(Object object) { + if (object instanceof NestedSet<?>) { + return false; + } else if (object instanceof List<?>) { + return false; + } + return true; + } + + /** + * Throws EvalException if the type of the object is not allowed to be present in Skylark. + */ + static void checkTypeAllowedInSkylark(Object object, Location loc) throws EvalException { + if (!isTypeAllowedInSkylark(object)) { + throw new EvalException(loc, + "Type is not allowed in Skylark: " + + object.getClass().getSimpleName()); + } + } + + private static Class<?> getGenericTypeFromMethod(Method method) { + // This is where we can infer generic type information, so SkylarkNestedSets can be + // created in a safe way. Eventually we should probably do something with Lists and Maps too. + ParameterizedType t = (ParameterizedType) method.getGenericReturnType(); + Type type = t.getActualTypeArguments()[0]; + if (type instanceof Class) { + return (Class<?>) type; + } + if (type instanceof WildcardType) { + WildcardType wildcard = (WildcardType) type; + Type upperBound = wildcard.getUpperBounds()[0]; + if (upperBound instanceof Class) { + // i.e. List<? extends SuperClass> + return (Class<?>) upperBound; + } + } + // It means someone annotated a method with @SkylarkCallable with no specific generic type info. + // We shouldn't annotate methods which return List<?> or List<T>. + throw new IllegalStateException("Cannot infer type from method signature " + method); + } + + /** + * Converts an object retrieved from a Java method to a Skylark-compatible type. + */ + static Object convertToSkylark(Object object, Method method) { + if (object instanceof NestedSet<?>) { + return new SkylarkNestedSet(getGenericTypeFromMethod(method), (NestedSet<?>) object); + } else if (object instanceof List<?>) { + return SkylarkList.list((List<?>) object, getGenericTypeFromMethod(method)); + } + return object; + } + + /** + * Converts an object to a Skylark-compatible type if possible. + */ + public static Object convertToSkylark(Object object, Location loc) throws EvalException { + if (object instanceof List<?>) { + return SkylarkList.list((List<?>) object, loc); + } + return object; + } + + /** + * Converts object from a Skylark-compatible wrapper type to its original type. + */ + public static Object convertFromSkylark(Object value) { + if (value instanceof SkylarkList) { + return ((SkylarkList) value).toList(); + } + return value; + } + + /** + * Creates a SkylarkType from the SkylarkBuiltin annotation. + */ + public static SkylarkType getReturnType(SkylarkBuiltin annotation) { + if (annotation.returnType().equals(Object.class)) { + return SkylarkType.UNKNOWN; + } + if (Function.class.isAssignableFrom(annotation.returnType())) { + return SkylarkFunctionType.of(annotation.name()); + } + return SkylarkType.of(annotation.returnType()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Statement.java b/src/main/java/com/google/devtools/build/lib/syntax/Statement.java new file mode 100644 index 0000000..ca89b1a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/Statement.java
@@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +/** + * Base class for all statements nodes in the AST. + */ +public abstract class Statement extends ASTNode { + + /** + * Executes the statement in the specified build environment, which may be + * modified. + * + * @throws EvalException if execution of the statement could not be completed. + */ + abstract void exec(Environment env) throws EvalException, InterruptedException; + + /** + * Checks the semantics of the Statement using the SkylarkEnvironment according to + * the rules of the Skylark language. The SkylarkEnvironment can be used e.g. to check + * variable type collision, read only variables, detecting recursion, existence of + * built-in variables, functions, etc. + * + * <p>The semantical check should be performed after the Skylark extension is loaded + * (i.e. is syntactically correct) and before is executed. The point of the semantical check + * is to make sure (as much as possible) that no error can occur during execution (Skylark + * programmers get a "compile time" error). It should also check execution branches (e.g. in + * if statements) that otherwise might never get executed. + * + * @throws EvalException if the Statement has a semantical error. + */ + abstract void validate(ValidationEnvironment env) throws EvalException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/StringLiteral.java b/src/main/java/com/google/devtools/build/lib/syntax/StringLiteral.java new file mode 100644 index 0000000..98d5045 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/StringLiteral.java
@@ -0,0 +1,56 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +/** + * Syntax node for a string literal. + */ +public final class StringLiteral extends Literal<String> { + + private final char quoteChar; + + public StringLiteral(String value, char quoteChar) { + super(value); + this.quoteChar = quoteChar; + } + + @Override + public String toString() { + return new StringBuilder() + .append(quoteChar) + .append(value.replace(Character.toString(quoteChar), "\\" + quoteChar)) + .append(quoteChar) + .toString(); + } + + @Override + public void accept(SyntaxTreeVisitor visitor) { + visitor.visit(this); + } + + /** + * Gets the quote character that was used for this string. For example, if + * the string was 'hello, world!', then this method returns '\''. + * + * @return the character used to quote the string. + */ + public char getQuoteChar() { + return quoteChar; + } + + @Override + SkylarkType validate(ValidationEnvironment env) throws EvalException { + return SkylarkType.STRING; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SyntaxTreeVisitor.java b/src/main/java/com/google/devtools/build/lib/syntax/SyntaxTreeVisitor.java new file mode 100644 index 0000000..5a95026 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/SyntaxTreeVisitor.java
@@ -0,0 +1,145 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.devtools.build.lib.syntax.DictionaryLiteral.DictionaryEntryLiteral; +import com.google.devtools.build.lib.syntax.IfStatement.ConditionalStatements; + +import java.util.List; +import java.util.Map; + +/** + * A visitor for visiting the nodes in the syntax tree left to right, top to + * bottom. + */ +public class SyntaxTreeVisitor { + + public void visit(ASTNode node) { + // dispatch to the node specific method + node.accept(this); + } + + public void visitAll(List<? extends ASTNode> nodes) { + for (ASTNode node : nodes) { + visit(node); + } + } + + // node specific visit methods + public void visit(Argument node) { + if (node.isNamed()) { + visit(node.getName()); + } + if (node.hasValue()) { + visit(node.getValue()); + } + } + + public void visit(BuildFileAST node) { + visitAll(node.getStatements()); + visitAll(node.getComments()); + } + + public void visit(BinaryOperatorExpression node) { + visit(node.getLhs()); + visit(node.getRhs()); + } + + public void visit(FuncallExpression node) { + visit(node.getFunction()); + visitAll(node.getArguments()); + } + + public void visit(Ident node) { + } + + public void visit(ListComprehension node) { + visit(node.getElementExpression()); + for (Map.Entry<Ident, Expression> list : node.getLists()) { + visit(list.getKey()); + visit(list.getValue()); + } + } + + public void accept(DictComprehension node) { + visit(node.getKeyExpression()); + visit(node.getValueExpression()); + visit(node.getLoopVar()); + visit(node.getListExpression()); + } + + public void visit(ListLiteral node) { + visitAll(node.getElements()); + } + + public void visit(IntegerLiteral node) { + } + + public void visit(StringLiteral node) { + } + + public void visit(AssignmentStatement node) { + visit(node.getLValue()); + visit(node.getExpression()); + } + + public void visit(ExpressionStatement node) { + visit(node.getExpression()); + } + + public void visit(IfStatement node) { + for (ConditionalStatements stmt : node.getThenBlocks()) { + visit(stmt); + } + for (Statement stmt : node.getElseBlock()) { + visit(stmt); + } + } + + public void visit(ConditionalStatements node) { + visit(node.getCondition()); + for (Statement stmt : node.getStmts()) { + visit(stmt); + } + } + + public void visit(FunctionDefStatement node) { + visit(node.getIdent()); + for (Argument arg : node.getArgs()) { + visit(arg); + } + for (Statement stmt : node.getStatements()) { + visit(stmt); + } + } + + public void visit(DictionaryLiteral node) { + for (DictionaryEntryLiteral entry : node.getEntries()) { + visit(entry); + } + } + + public void visit(DictionaryEntryLiteral node) { + visit(node.getKey()); + visit(node.getValue()); + } + + public void visit(NotExpression node) { + visit(node.getExpression()); + } + + public void visit(Comment node) { + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Token.java b/src/main/java/com/google/devtools/build/lib/syntax/Token.java new file mode 100644 index 0000000..e3bcfec --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/Token.java
@@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +/** + * A Token represents an actual lexeme; that is, a lexical unit, its location in + * the input text, its lexical kind (TokenKind) and any associated value. + */ +public class Token { + + public final TokenKind kind; + public final int left; + public final int right; + public final Object value; + + public Token(TokenKind kind, int left, int right) { + this(kind, left, right, null); + } + + public Token(TokenKind kind, int left, int right, Object value) { + this.kind = kind; + this.left = left; + this.right = right; + this.value = value; + } + + /** + * Constructs an easy-to-read string representation of token, suitable for use + * in user error messages. + */ + @Override + public String toString() { + // TODO(bazel-team): do proper escaping of string literals + return kind == TokenKind.STRING ? ("\"" + value + "\"") + : value == null ? kind.getPrettyName() + : value.toString(); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java b/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java new file mode 100644 index 0000000..f6dad9f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java
@@ -0,0 +1,83 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +/** + * A TokenKind is an enumeration of each different kind of lexical symbol. + */ +public enum TokenKind { + + AND("and"), + AS("as"), + CLASS("class"), + COLON(":"), + COMMA(","), + COMMENT("comment"), + DEF("def"), + DOT("."), + ELIF("elif"), + ELSE("else"), + EOF("EOF"), + EQUALS("="), + EQUALS_EQUALS("=="), + EXCEPT("except"), + FINALLY("finally"), + FOR("for"), + FROM("from"), + GREATER(">"), + GREATER_EQUALS(">="), + IDENTIFIER("identifier"), + IF("if"), + ILLEGAL("illegal character"), + IMPORT("import"), + IN("in"), + INDENT("indent"), + INT("integer"), + LBRACE("{"), + LBRACKET("["), + LESS("<"), + LESS_EQUALS("<="), + LPAREN("("), + MINUS("-"), + NEWLINE("newline"), + NOT("not"), + NOT_EQUALS("!="), + OR("or"), + OUTDENT("outdent"), + PERCENT("%"), + PLUS("+"), + PLUS_EQUALS("+="), + RBRACE("}"), + RBRACKET("]"), + RETURN("return"), + RPAREN(")"), + SEMI(";"), + STAR("*"), + STRING("string"), + TRY("try"); + + private final String prettyName; + + private TokenKind(String prettyName) { + this.prettyName = prettyName; + } + + /** + * Returns the pretty name for this token, for use in error messages for the user. + */ + public String getPrettyName() { + return prettyName; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java new file mode 100644 index 0000000..cd909a9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java
@@ -0,0 +1,115 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.events.Location; + +/** + * The actual function registered in the environment. This function is defined in the + * parsed code using {@link FunctionDefStatement}. + */ +public class UserDefinedFunction extends MixedModeFunction { + + private final ImmutableList<Argument> args; + private final ImmutableMap<String, Integer> argIndexes; + private final ImmutableMap<String, Object> defaultValues; + private final ImmutableList<Statement> statements; + private final SkylarkEnvironment definitionEnv; + + private static ImmutableList<String> argumentToStringList(ImmutableList<Argument> args) { + Function<Argument, String> function = new Function<Argument, String>() { + @Override + public String apply(Argument id) { + return id.getArgName(); + } + }; + return ImmutableList.copyOf(Lists.transform(args, function)); + } + + private static int mandatoryArgNum(ImmutableList<Argument> args) { + int mandatoryArgNum = 0; + for (Argument arg : args) { + if (!arg.hasValue()) { + mandatoryArgNum++; + } + } + return mandatoryArgNum; + } + + UserDefinedFunction(Ident function, ImmutableList<Argument> args, + ImmutableMap<String, Object> defaultValues, + ImmutableList<Statement> statements, SkylarkEnvironment definitionEnv) { + super(function.getName(), argumentToStringList(args), mandatoryArgNum(args), false, + function.getLocation()); + this.args = args; + this.statements = statements; + this.definitionEnv = definitionEnv; + this.defaultValues = defaultValues; + + ImmutableMap.Builder<String, Integer> argIndexes = new ImmutableMap.Builder<> (); + int i = 0; + for (Argument arg : args) { + if (!arg.isKwargs()) { // TODO(bazel-team): add varargs support? + argIndexes.put(arg.getArgName(), i++); + } + } + this.argIndexes = argIndexes.build(); + } + + public ImmutableList<Argument> getArgs() { + return args; + } + + public Integer getArgIndex(String s) { + return argIndexes.get(s); + } + + ImmutableMap<String, Object> getDefaultValues() { + return defaultValues; + } + + ImmutableList<Statement> getStatements() { + return statements; + } + + Location getLocation() { + return location; + } + + @Override + public Object call(Object[] namedArguments, FuncallExpression ast, Environment env) + throws EvalException, InterruptedException { + SkylarkEnvironment functionEnv = SkylarkEnvironment.createEnvironmentForFunctionCalling( + env, definitionEnv, this); + + // Registering the functions's arguments as variables in the local Environment + int i = 0; + for (Object arg : namedArguments) { + functionEnv.update(args.get(i++).getArgName(), arg); + } + + try { + for (Statement stmt : statements) { + stmt.exec(functionEnv); + } + } catch (ReturnStatement.ReturnException e) { + return e.getValue(); + } + return Environment.NONE; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ValidationEnvironment.java b/src/main/java/com/google/devtools/build/lib/syntax/ValidationEnvironment.java new file mode 100644 index 0000000..6afb96a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/syntax/ValidationEnvironment.java
@@ -0,0 +1,244 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.syntax; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.Stack; + +/** + * An Environment for the semantic checking of Skylark files. + * + * @see Statement#validate + * @see Expression#validate + */ +public class ValidationEnvironment { + + private final ValidationEnvironment parent; + + private Map<SkylarkType, Map<String, SkylarkType>> variableTypes = new HashMap<>(); + + private Map<String, Location> variableLocations = new HashMap<>(); + + private Set<String> readOnlyVariables = new HashSet<>(); + + // A stack of variable-sets which are read only but can be assigned in different + // branches of if-else statements. + private Stack<Set<String>> futureReadOnlyVariables = new Stack<>(); + + // The function we are currently validating. + private SkylarkFunctionType currentFunction; + + // Whether this validation environment is not modified therefore clonable or not. + private boolean clonable; + + public ValidationEnvironment( + ImmutableMap<SkylarkType, ImmutableMap<String, SkylarkType>> builtinVariableTypes) { + parent = null; + variableTypes = CollectionUtils.copyOf(builtinVariableTypes); + readOnlyVariables.addAll(builtinVariableTypes.get(SkylarkType.GLOBAL).keySet()); + clonable = true; + } + + private ValidationEnvironment(Map<SkylarkType, Map<String, SkylarkType>> builtinVariableTypes, + Set<String> readOnlyVariables) { + parent = null; + this.variableTypes = CollectionUtils.copyOf(builtinVariableTypes); + this.readOnlyVariables = new HashSet<>(readOnlyVariables); + clonable = false; + } + + @Override + public ValidationEnvironment clone() { + Preconditions.checkState(clonable); + return new ValidationEnvironment(variableTypes, readOnlyVariables); + } + + /** + * Creates a local ValidationEnvironment to validate user defined function bodies. + */ + public ValidationEnvironment(ValidationEnvironment parent, SkylarkFunctionType currentFunction) { + this.parent = parent; + this.variableTypes.put(SkylarkType.GLOBAL, new HashMap<String, SkylarkType>()); + this.currentFunction = currentFunction; + for (String var : parent.readOnlyVariables) { + if (!parent.variableLocations.containsKey(var)) { + // Mark built in global vars readonly. Variables defined in Skylark may be shadowed locally. + readOnlyVariables.add(var); + } + } + this.clonable = false; + } + + /** + * Returns true if this ValidationEnvironment is top level i.e. has no parent. + */ + public boolean isTopLevel() { + return parent == null; + } + + /** + * Updates the variable type if the new type is "stronger" then the old one. + * The old and the new vartype has to be compatible, otherwise an EvalException is thrown. + * The new type is stronger if the old one doesn't exist or unknown. + */ + public void update(String varname, SkylarkType newVartype, Location location) + throws EvalException { + checkReadonly(varname, location); + if (parent == null) { // top-level values are immutable + readOnlyVariables.add(varname); + if (!futureReadOnlyVariables.isEmpty()) { + // Currently validating an if-else statement + futureReadOnlyVariables.peek().add(varname); + } + } + SkylarkType oldVartype = variableTypes.get(SkylarkType.GLOBAL).get(varname); + if (oldVartype != null) { + newVartype = oldVartype.infer(newVartype, "variable '" + varname + "'", + location, variableLocations.get(varname)); + } + variableTypes.get(SkylarkType.GLOBAL).put(varname, newVartype); + variableLocations.put(varname, location); + clonable = false; + } + + private void checkReadonly(String varname, Location location) throws EvalException { + if (readOnlyVariables.contains(varname)) { + throw new EvalException(location, String.format("Variable %s is read only", varname)); + } + } + + public void checkIterable(SkylarkType type, Location loc) throws EvalException { + if (type == SkylarkType.UNKNOWN) { + // Until all the language is properly typed, we ignore Object types. + return; + } + if (!Iterable.class.isAssignableFrom(type.getType()) + && !Map.class.isAssignableFrom(type.getType()) + && !String.class.equals(type.getType())) { + throw new EvalException(loc, + "type '" + EvalUtils.getDataTypeNameFromClass(type.getType()) + "' is not iterable"); + } + } + + /** + * Returns true if the symbol exists in the validation environment. + */ + public boolean hasSymbolInEnvironment(String varname) { + return variableTypes.get(SkylarkType.GLOBAL).containsKey(varname) + || topLevel().variableTypes.get(SkylarkType.GLOBAL).containsKey(varname); + } + + /** + * Returns the type of the existing variable. + */ + public SkylarkType getVartype(String varname) { + SkylarkType type = variableTypes.get(SkylarkType.GLOBAL).get(varname); + if (type == null && parent != null) { + type = parent.getVartype(varname); + } + return Preconditions.checkNotNull(type, + String.format("Variable %s is not found in the validation environment", varname)); + } + + public SkylarkFunctionType getCurrentFunction() { + return currentFunction; + } + + /** + * Returns the return type of the function. + */ + public SkylarkType getReturnType(String funcName, Location loc) throws EvalException { + return getReturnType(SkylarkType.GLOBAL, funcName, loc); + } + + /** + * Returns the return type of the object function. + */ + public SkylarkType getReturnType(SkylarkType objectType, String funcName, Location loc) + throws EvalException { + // All functions are registered in the top level ValidationEnvironment. + Map<String, SkylarkType> functions = topLevel().variableTypes.get(objectType); + // TODO(bazel-team): eventually not finding the return type should be a validation error, + // because it means the function doesn't exist. First we have to make sure that we register + // every possible function before. + if (functions != null) { + SkylarkType functionType = functions.get(funcName); + if (functionType != null && functionType != SkylarkType.UNKNOWN) { + if (!(functionType instanceof SkylarkFunctionType)) { + throw new EvalException(loc, (objectType == SkylarkType.GLOBAL ? "" : objectType + ".") + + funcName + " is not a function"); + } + return ((SkylarkFunctionType) functionType).getReturnType(); + } + } + return SkylarkType.UNKNOWN; + } + + private ValidationEnvironment topLevel() { + return Preconditions.checkNotNull(parent == null ? this : parent); + } + + /** + * Adds a user defined function to the validation environment is not exists. + */ + public void updateFunction(String name, SkylarkFunctionType type, Location loc) + throws EvalException { + checkReadonly(name, loc); + if (variableTypes.get(SkylarkType.GLOBAL).containsKey(name)) { + throw new EvalException(loc, "function " + name + " already exists"); + } + variableTypes.get(SkylarkType.GLOBAL).put(name, type); + clonable = false; + } + + /** + * Starts a session with temporarily disabled readonly checking for variables between branches. + * This is useful to validate control flows like if-else when we know that certain parts of the + * code cannot both be executed. + */ + public void startTemporarilyDisableReadonlyCheckSession() { + futureReadOnlyVariables.add(new HashSet<String>()); + clonable = false; + } + + /** + * Finishes the session with temporarily disabled readonly checking. + */ + public void finishTemporarilyDisableReadonlyCheckSession() { + Set<String> variables = futureReadOnlyVariables.pop(); + readOnlyVariables.addAll(variables); + if (!futureReadOnlyVariables.isEmpty()) { + futureReadOnlyVariables.peek().addAll(variables); + } + clonable = false; + } + + /** + * Finishes a branch of temporarily disabled readonly checking. + */ + public void finishTemporarilyDisableReadonlyCheckBranch() { + readOnlyVariables.removeAll(futureReadOnlyVariables.peek()); + clonable = false; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/Directories.java b/src/main/java/com/google/devtools/build/lib/unix/Directories.java new file mode 100644 index 0000000..b6c3cda --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/unix/Directories.java
@@ -0,0 +1,87 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.unix; + +import com.google.devtools.build.lib.shell.AbnormalTerminationException; +import com.google.devtools.build.lib.shell.Command; +import com.google.devtools.build.lib.shell.CommandException; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; + +/** + * Provides utility methods for working with directories in a Unix environment. + */ +public final class Directories { + + /** + * Deletes a file or directory and all contents recursively, like {@code rm -rf file}. + * + * <p>If the file argument is a symbolic link, the link will be deleted but not the target of the + * link. If the argument is a directory, symbolic links within the directory will not be followed. + * If the argument does not exist, throws a FileNotFoundException. + * + * @param file the file or directory to delete + * @throws FileNotFoundException if the file or directory does not exist + * @throws IOException if an I/O error occurs + */ + public static void deleteRecursively(File file) throws IOException { + deleteRecursivelyImpl(file, true); + } + + /** + * Deletes a file or directory and all contents recursively, like {@code rm -rf file}, if it + * exists. + * + * <p>If the file argument is a symbolic link, the link will be deleted but not the target of the + * link. If the argument is a directory, symbolic links within the directory will not be followed. + * + * @param file the file or directory to delete + * @return {@code true} if the file or directory was deleted by this method; {@code false} if the + * file or directory could not be deleted because it did not exist + * @throws IOException if an I/O error occurs + */ + public static boolean deleteRecursivelyIfExists(File file) throws IOException { + return deleteRecursivelyImpl(file, false); + } + + private static boolean deleteRecursivelyImpl(File file, boolean failIfFileDoesNotExist) + throws IOException { + if (!file.exists()) { + if (failIfFileDoesNotExist) { + throw new FileNotFoundException(file.getPath()); + } else { + return false; + } + } + String filePath = file.getPath(); + if (!filePath.isEmpty() && filePath.charAt(0) == '-') { + filePath = "./" + filePath; + } + try { + new Command(new String[] {"/bin/rm", "-rf", filePath}).execute(); + } catch (AbnormalTerminationException e) { + String message = + e.getResult().getTerminationStatus() + ": " + new String(e.getResult().getStderr()); + throw new IOException(message, e); + } catch (CommandException e) { + throw new IOException(e); + } + return true; + } + + private Directories() {} +}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/ErrnoFileStatus.java b/src/main/java/com/google/devtools/build/lib/unix/ErrnoFileStatus.java new file mode 100644 index 0000000..fa0c3a7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/unix/ErrnoFileStatus.java
@@ -0,0 +1,94 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.unix; + +import com.google.devtools.build.lib.UnixJniLoader; + + +/** + * A subsclass of FileStatus which contains an errno. + * If there is an error, all other data fields are undefined. + */ +public class ErrnoFileStatus extends FileStatus { + + private final int errno; + + // These constants are passed in from JNI via ErrnoConstants. + public static final int ENOENT; + public static final int EACCES; + public static final int ELOOP; + public static final int ENOTDIR; + public static final int ENAMETOOLONG; + + static { + ErrnoConstants constants = ErrnoConstants.getErrnoConstants(); + ENOENT = constants.ENOENT; + EACCES = constants.EACCES; + ELOOP = constants.ELOOP; + ENOTDIR = constants.ENOTDIR; + ENAMETOOLONG = constants.ENAMETOOLONG; + } + + /** + * Constructs a ErrnoFileSatus instance. (Called only from JNI code.) + */ + private ErrnoFileStatus(int st_mode, int st_atime, int st_atimensec, int st_mtime, + int st_mtimensec, int st_ctime, int st_ctimensec, long st_size, + int st_dev, long st_ino) { + super(st_mode, st_atime, st_atimensec, st_mtime, st_mtimensec, st_ctime, st_ctimensec, st_size, + st_dev, st_ino); + this.errno = 0; + } + + /** + * Constructs a ErrnoFileSatus instance. (Called only from JNI code.) + */ + private ErrnoFileStatus(int errno) { + super(0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + this.errno = errno; + } + + public int getErrno() { + return errno; + } + + public boolean hasError() { + // errno = 0 means the operation succeeded. + return errno != 0; + } + + // Used to transfer the constants from native to java code. + private static class ErrnoConstants { + + // These are set in JNI. + private int ENOENT; + private int EACCES; + private int ELOOP; + private int ENOTDIR; + private int ENAMETOOLONG; + + public static ErrnoConstants getErrnoConstants() { + ErrnoConstants constants = new ErrnoConstants(); + constants.initErrnoConstants(); + return constants; + } + + static { + UnixJniLoader.loadJni(); + } + + private native void initErrnoConstants(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/FileAccessException.java b/src/main/java/com/google/devtools/build/lib/unix/FileAccessException.java new file mode 100644 index 0000000..9ea4c6e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/unix/FileAccessException.java
@@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.unix; + +import java.io.IOException; + +/** + * An IOException subclass that is thrown when a POSIX filesystem call returns + * an EACCES errno. The message is generally "Permission denied". + */ +public class FileAccessException extends IOException { + /** + * Constructs a <code>FileAccessException</code> with <code>null</code> + * as its error detail message. + */ + public FileAccessException() { + super(); + } + + /** + * Constructs an <code>FileAccessException</code> with the specified detail + * message. The error message string <code>s</code> can later be + * retrieved by the <code>{@link java.lang.Throwable#getMessage}</code> + * method of class <code>java.lang.Throwable</code>. + * + * @param s the detail message. + */ + public FileAccessException(String s) { + super(s); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/FileStatus.java b/src/main/java/com/google/devtools/build/lib/unix/FileStatus.java new file mode 100644 index 0000000..10f4d99 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/unix/FileStatus.java
@@ -0,0 +1,262 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.unix; + +/** + * <p>Equivalent to UNIX's "struct stat", a FileStatus instance contains + * various bits of metadata about a directory entry. + * + * <p>The Java SDK provides access to some but not all of the information + * available via the stat(2) and lstat(2) syscalls, but often requires that + * multiple calls be made to obtain it. By reifying stat buffers as Java + * objects and providing a wrapper around the stat/lstat calls, we give client + * applications access to the richer file metadata and enable a reduction in + * the number of system calls, which is critical for high-performance tools. + * + * <p>This class is optimized for memory usage. Operations that are not yet + * required for any client are intentionally unimplemented to save space. + * Currently, we only support these fields: st_mode, st_size, st_atime, + * st_atimensec, st_mtime, st_mtimensec, st_ctime, st_ctimensec, st_dev, st_ino. + * Methods that require other fields throw UnsupportedOperationException. + */ +public class FileStatus { + + private final int st_mode; + private final int st_atime; // (unsigned) + private final int st_atimensec; // (unsigned) + private final int st_mtime; // (unsigned) + private final int st_mtimensec; // (unsigned) + private final int st_ctime; // (unsigned) + private final int st_ctimensec; // (unsigned) + private final long st_size; + private final int st_dev; + private final long st_ino; + + /** + * Constructs a FileStatus instance. (Called only from JNI code.) + */ + protected FileStatus(int st_mode, int st_atime, int st_atimensec, int st_mtime, int st_mtimensec, + int st_ctime, int st_ctimensec, long st_size, int st_dev, long st_ino) { + this.st_mode = st_mode; + this.st_atime = st_atime; + this.st_atimensec = st_atimensec; + this.st_mtime = st_mtime; + this.st_mtimensec = st_mtimensec; + this.st_ctime = st_ctime; + this.st_ctimensec = st_ctimensec; + this.st_size = st_size; + this.st_dev = st_dev; + this.st_ino = st_ino; + } + + /** + * Returns the device number of this inode. + */ + public int getDeviceNumber() { + return st_dev; + } + + /** + * Returns the number of this inode. Inode numbers are (usually) unique for + * a given device. + */ + public long getInodeNumber() { + return st_ino; + } + + /** + * Returns true iff this file is a regular file. + */ + public boolean isRegularFile() { + return (st_mode & S_IFMT) == S_IFREG; + } + + /** + * Returns true iff this file is a directory. + */ + public boolean isDirectory() { + return (st_mode & S_IFMT) == S_IFDIR; + } + + /** + * Returns true iff this file is a symbolic link. + */ + public boolean isSymbolicLink() { + return (st_mode & S_IFMT) == S_IFLNK; + } + + /** + * Returns true iff this file is a character device. + */ + public boolean isCharacterDevice() { + return (st_mode & S_IFMT) == S_IFCHR; + } + + /** + * Returns true iff this file is a block device. + */ + public boolean isBlockDevice() { + return (st_mode & S_IFMT) == S_IFBLK; + } + + /** + * Returns true iff this file is a FIFO. + */ + public boolean isFIFO() { + return (st_mode & S_IFMT) == S_IFIFO; + } + + /** + * Returns true iff this file is a UNIX-domain socket. + */ + public boolean isSocket() { + return (st_mode & S_IFMT) == S_IFSOCK; + } + + /** + * Returns true iff this file has its "set UID" bit set. + */ + public boolean isSetUserId() { + return (st_mode & S_ISUID) != 0; + } + + /** + * Returns true iff this file has its "set GID" bit set. + */ + public boolean isSetGroupId() { + return (st_mode & S_ISGID) != 0; + } + + /** + * Returns true iff this file has its "sticky" bit set. See UNIX manuals for + * explanation. + */ + public boolean isSticky() { + return (st_mode & S_ISVTX) != 0; + } + + /** + * Returns the user/group/other permissions part of the mode bits (i.e. + * st_mode masked with 0777), interpreted according to longstanding UNIX + * tradition. + */ + public int getPermissions() { + return st_mode & S_IRWXA; + } + + /** + * Returns the total size, in bytes, of this file. + */ + public long getSize() { + return st_size; + } + + /** + * Returns the last access time of this file (seconds since UNIX epoch). + */ + public long getLastAccessTime() { + return unsignedIntToLong(st_atime); + } + + /** + * Returns the fractional part of the last access time of this file (nanoseconds). + */ + public long getFractionalLastAccessTime() { + return unsignedIntToLong(st_atimensec); + } + + /** + * Returns the last modified time of this file (seconds since UNIX epoch). + */ + public long getLastModifiedTime() { + return unsignedIntToLong(st_mtime); + } + + /** + * Returns the fractional part of the last modified time of this file (nanoseconds). + */ + public long getFractionalLastModifiedTime() { + return unsignedIntToLong(st_mtimensec); + } + + /** + * Returns the last change time of this file (seconds since UNIX epoch). + */ + public long getLastChangeTime() { + return unsignedIntToLong(st_ctime); + } + + /** + * Returns the fractional part of the last change time of this file (nanoseconds). + */ + public long getFractionalLastChangeTime() { + return unsignedIntToLong(st_ctimensec); + } + + //////////////////////////////////////////////////////////////////////// + + @Override + public String toString() { + return String.format("FileStatus(mode=0%06o,size=%d,mtime=%d)", + st_mode, st_size, st_mtime); + } + + @Override + public int hashCode() { + return st_mode; + } + + //////////////////////////////////////////////////////////////////////// + // Platform-specific details. These fields are public so that they can + // be used from other packages. See POSIX and/or Linux manuals for details. + // + // These need to be kept in sync with the native code and system call + // interface. (The unit tests ensure that.) Of course, this decoding could + // be done in the JNI code to ensure maximum portability, but (a) we don't + // expect we'll need that any time soon, and (b) that would require eager + // rather than on-demand bitmunging of all attributes. In any case, it's not + // part of the interface so it can be easily changed later if necessary. + + public static final int S_IFMT = 0170000; // mask: filetype bitfields + public static final int S_IFSOCK = 0140000; // socket + public static final int S_IFLNK = 0120000; // symbolic link + public static final int S_IFREG = 0100000; // regular file + public static final int S_IFBLK = 0060000; // block device + public static final int S_IFDIR = 0040000; // directory + public static final int S_IFCHR = 0020000; // character device + public static final int S_IFIFO = 0010000; // fifo + public static final int S_ISUID = 0004000; // set UID bit + public static final int S_ISGID = 0002000; // set GID bit (see below) + public static final int S_ISVTX = 0001000; // sticky bit (see below) + public static final int S_IRWXA = 00777; // mask: all permissions + public static final int S_IRWXU = 00700; // mask: file owner permissions + public static final int S_IRUSR = 00400; // owner has read permission + public static final int S_IWUSR = 00200; // owner has write permission + public static final int S_IXUSR = 00100; // owner has execute permission + public static final int S_IRWXG = 00070; // mask: group permissions + public static final int S_IRGRP = 00040; // group has read permission + public static final int S_IWGRP = 00020; // group has write permission + public static final int S_IXGRP = 00010; // group has execute permission + public static final int S_IRWXO = 00007; // mask: other permissions + public static final int S_IROTH = 00004; // others have read permission + public static final int S_IWOTH = 00002; // others have write permisson + public static final int S_IXOTH = 00001; // others have execute permission + + public static final int S_IEXEC = 00111; // owner, group, world execute + + static long unsignedIntToLong(int i) { + return (i & 0x7FFFFFFF) - (long) (i & 0x80000000); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/FilesystemUtils.java b/src/main/java/com/google/devtools/build/lib/unix/FilesystemUtils.java new file mode 100644 index 0000000..462ed9c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/unix/FilesystemUtils.java
@@ -0,0 +1,442 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.unix; + +import com.google.common.hash.HashCode; +import com.google.devtools.build.lib.UnixJniLoader; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.LogManager; +import java.util.logging.Logger; + +/** + * Utility methods for access to UNIX filesystem calls not exposed by the Java + * SDK. Exception messages are selected to be consistent with those generated + * by the java.io package where appropriate--see package javadoc for details. + */ +public final class FilesystemUtils { + + private FilesystemUtils() {} + + /** + * Returns true iff the file identified by 'path' is a symbolic link. Has + * similar semantics to POSIX stat(2) syscall, with all errors being mapped to + * a false return. + * + * @param path the file of interest + * @return true iff path exists, is accessible and is a symlink + */ + public static boolean isSymbolicLink(File path) { + try { + return lstat(path.toString()).isSymbolicLink(); + } catch (IOException e) { + return false; + } + } + + + /** + * Returns true iff the file identified by 'path' is a directory. Has + * similar semantics to POSIX stat(2) syscall, with all errors being mapped to + * a false return. + * + * @param path the file of interest + * @return true iff path exists, is accessible and is a symlink + */ + public static boolean isDirectory(String path) { + try { + return lstat(path).isDirectory(); + } catch (IOException e) { + return false; + } + } + + + /** + * Marks the file or directory {@code path} as executable. (Non-atomic) + * + * @see File#setReadOnly + * + * @param path the file of interest + * @throws FileAccessException if path can't be accessed + * @throws FileNotFoundException if path doesn't exist + * @throws IOException for other filesystem or path errors + */ + public static void setExecutable(File path) throws IOException { + String p = path.toString(); + chmod(p, stat(p).getPermissions() | FileStatus.S_IEXEC); + } + + /** + * Marks the file or directory {@code path} as owner writable. (Non-atomic) + * + * @see File#setReadOnly + * + * @param path the file of interest + * @throws FileAccessException if path can't be accessed + * @throws FileNotFoundException if path doesn't exist + * @throws IOException for other filesystem or path errors + */ + public static void setWritable(File path) throws IOException { + String p = path.toString(); + chmod(p, stat(p).getPermissions() | FileStatus.S_IWUSR); + } + + /** + * Changes permissions of a file. + * + * @param path the file whose mode to change. + * @param mode the mode bits within 07777, interpreted according to + * long-standing UNIX tradition. + * @throws IOException if the chmod() syscall failed. + */ + public static void chmod(File path, int mode) throws IOException { + int mask = FileStatus.S_ISUID | + FileStatus.S_ISGID | + FileStatus.S_ISVTX | + FileStatus.S_IRWXA; + chmod(path.toString(), mode & mask); + } + + /* + * Native-based implementation + */ + + static { + if (!java.nio.charset.Charset.defaultCharset().name().equals("ISO-8859-1")) { + // Defer the Logger call, so we don't deadlock if this is called from Logger + // initialization code. + new Thread() { + @Override + public void run() { + // wait (if necessary) until the logging system is initialized + synchronized (LogManager.getLogManager()) {} + Logger.getLogger("com.google.devtools.build.lib.unix.FilesystemUtils").log(Level.FINE, + "WARNING: Default character set is not latin1; java.io.File and " + + "com.google.devtools.build.lib.unix.FilesystemUtils will represent some filenames " + + "differently."); + } + }.start(); + } + UnixJniLoader.loadJni(); + } + + /** + * Native wrapper around Linux readlink(2) call. + * + * @param path the file of interest + * @return the pathname to which the symbolic link 'path' links + * @throws IOException iff the readlink() call failed + */ + public static native String readlink(String path) throws IOException; + + /** + * Native wrapper around POSIX chmod(2) syscall: Changes the file access + * permissions of 'path' to 'mode'. + * + * @param path the file of interest + * @param mode the POSIX type and permission mode bits to set + * @throws IOException iff the chmod() call failed. + */ + public static native void chmod(String path, int mode) throws IOException; + + /** + * Native wrapper around POSIX symlink(2) syscall. + * + * @param oldpath the file to link to + * @param newpath the new path for the link + * @throws IOException iff the symlink() syscall failed. + */ + public static native void symlink(String oldpath, String newpath) + throws IOException; + + /** + * Native wrapper around POSIX stat(2) syscall. + * + * @param path the file to stat. + * @return a FileStatus instance containing the metadata. + * @throws IOException if the stat() syscall failed. + */ + public static native FileStatus stat(String path) throws IOException; + + /** + * Native wrapper around POSIX lstat(2) syscall. + * + * @param path the file to lstat. + * @return a FileStatus instance containing the metadata. + * @throws IOException if the lstat() syscall failed. + */ + public static native FileStatus lstat(String path) throws IOException; + + /** + * Native wrapper around POSIX stat(2) syscall. + * + * @param path the file to stat. + * @return an ErrnoFileStatus instance containing the metadata. + * If there was an error, the return value's hasError() method + * will return true, and all stat information is undefined. + */ + public static native ErrnoFileStatus errnoStat(String path); + + /** + * Native wrapper around POSIX lstat(2) syscall. + * + * @param path the file to lstat. + * @return an ErrnoFileStatus instance containing the metadata. + * If there was an error, the return value's hasError() method + * will return true, and all stat information is undefined. + */ + public static native ErrnoFileStatus errnoLstat(String path); + + /** + * Native wrapper around POSIX utime(2) syscall. + * + * Note: negative file times are interpreted as unsigned time_t. + * + * @param path the file whose times to change. + * @param now if true, ignore actime/modtime parameters and use current time. + * @param actime the file access time in seconds since the UNIX epoch. + * @param modtime the file modification time in seconds since the UNIX epoch. + * @throws IOException if the utime() syscall failed. + */ + public static native void utime(String path, boolean now, + int actime, int modtime) throws IOException; + + /** + * Native wrapper around POSIX mkdir(2) syscall. + * + * Caveat: errno==EEXIST is mapped to the return value "false", not + * IOException. It requires an additional stat() to determine if mkdir + * failed because the directory already exists. + * + * @param path the directory to create. + * @param mode the mode with which to create the directory. + * @return true if the directory was successfully created; false if the + * system call returned EEXIST because some kind of a file (not necessarily + * a directory) already exists. + * @throws IOException if the mkdir() syscall failed for any other reason. + */ + public static native boolean mkdir(String path, int mode) + throws IOException; + + /** + * Native wrapper around POSIX opendir(2)/readdir(3)/closedir(3) syscall. + * + * @param path the directory to read. + * @return the list of directory entries in the order they were returned by + * the system, excluding "." and "..". + * @throws IOException if the call to opendir failed for any reason. + */ + public static String[] readdir(String path) throws IOException { + return readdir(path, ReadTypes.NONE).names; + } + + /** + * An enum for specifying now the types of the individual entries returned by + * {@link #readdir(String, ReadTypes)} is to be returned. + */ + public enum ReadTypes { + NONE('n'), // Do not read types + NOFOLLOW('d'), // Do not follow symlinks + FOLLOW('f'); // Follow symlinks; never returns "SYMLINK" and returns "UNKNOWN" when dangling + + private final char code; + + private ReadTypes(char code) { + this.code = code; + } + + private char getCode() { + return code; + } + } + + /** + * A compound return type for readdir(), analogous to struct dirent[] in C. A low memory profile + * is critical for this class, as instances are expected to be kept around for caching for + * potentially a long time. + */ + public static final class Dirents { + + /** + * The type of the directory entry. + */ + public enum Type { + FILE, + DIRECTORY, + SYMLINK, + UNKNOWN; + + private static Type forChar(char c) { + if (c == 'f') { + return Type.FILE; + } else if (c == 'd') { + return Type.DIRECTORY; + } else if (c == 's') { + return Type.SYMLINK; + } else { + return Type.UNKNOWN; + } + } + } + + /** The names of the entries in a directory. */ + private final String[] names; + /** + * An optional (nullable) array of entry types, corresponding positionally + * to the "names" field. The types are: + * 'd': a subdirectory + * 'f': a regular file + * 's': a symlink (only returned with {@code NOFOLLOW}) + * '?': anything else + * Note that unlike libc, this implementation of readdir() follows + * symlinks when determining these types. + * + * <p>This is intentionally a byte array rather than a array of enums to save memory. + */ + private final byte[] types; + + /** called from JNI */ + public Dirents(String[] names, byte[] types) { + this.names = names; + this.types = types; + } + + public int size() { + return names.length; + } + + public boolean hasTypes() { + return types != null; + } + + public String getName(int i) { + return names[i]; + } + + public Type getType(int i) { + return Type.forChar((char) types[i]); + } + } + + /** + * Native wrapper around POSIX opendir(2)/readdir(3)/closedir(3) syscall. + * + * @param path the directory to read. + * @param readTypes How the types of individual entries should be returned. If {@code NONE}, + * the "types" field in the result will be null. + * @return a Dirents object, containing "names", the list of directory entries + * (excluding "." and "..") in the order they were returned by the system, + * and "types", an array of entry types (file, directory, etc) corresponding + * positionally to "names". + * @throws IOException if the call to opendir failed for any reason. + */ + public static Dirents readdir(String path, ReadTypes readTypes) throws IOException { + // Passing enums to native code is possible, but onerous; we use a char instead. + return readdir(path, readTypes.getCode()); + } + + private static native Dirents readdir(String path, char typeCode) + throws IOException; + + /** + * Native wrapper around POSIX rename(2) syscall. + * + * @param oldpath the source location. + * @param newpath the destination location. + * @throws IOException if the rename failed for any reason. + */ + public static native void rename(String oldpath, String newpath) + throws IOException; + + /** + * Native wrapper around POSIX remove(3) C library call. + * + * @param path the file or directory to remove. + * @return true iff the file was actually deleted by this call. + * @throws IOException if the remove failed, but the file was present prior to the call. + */ + public static native boolean remove(String path) throws IOException; + + /******************************************************************** + * * + * Linux extended file attributes * + * * + ********************************************************************/ + + /** + * Native wrapper around Linux getxattr(2) syscall. + * + * @param path the file whose extended attribute is to be returned. + * @param name the name of the extended attribute key. + * @return the value of the extended attribute associated with 'path', if + * any, or null if no such attribute is defined (ENODATA). + * @throws IOException if the call failed for any other reason. + */ + public static native byte[] getxattr(String path, String name) + throws IOException; + + /** + * Native wrapper around Linux lgetxattr(2) syscall. (Like getxattr, but + * does not follow symbolic links.) + * + * @param path the file whose extended attribute is to be returned. + * @param name the name of the extended attribute key. + * @return the value of the extended attribute associated with 'path', if + * any, or null if no such attribute is defined (ENODATA). + * @throws IOException if the call failed for any other reason. + */ + public static native byte[] lgetxattr(String path, String name) + throws IOException; + + /** + * Returns the MD5 digest of the specified file, following symbolic links. + * + * @param path the file whose MD5 digest is required. + * @return the MD5 digest, as a 16-byte array. + * @throws IOException if the call failed for any reason. + */ + static native byte[] md5sumAsBytes(String path) throws IOException; + + /** + * Returns the MD5 digest of the specified file, following symbolic links. + * + * @param path the file whose MD5 digest is required. + * @return the MD5 digest, as a {@link HashCode} + * @throws IOException if the call failed for any reason. + */ + public static HashCode md5sum(String path) throws IOException { + return HashCode.fromBytes(md5sumAsBytes(path)); + } + + /** + * Removes entire directory tree. Doesn't follow symlinks. + * + * @param path the file or directory to remove. + * @throws IOException if the remove failed. + */ + public static void rmTree(String path) throws IOException { + if (isDirectory(path)) { + String[] contents = readdir(path); + for (String entry : contents) { + rmTree(path + "/" + entry); + } + } + remove(path.toString()); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/LocalClientSocket.java b/src/main/java/com/google/devtools/build/lib/unix/LocalClientSocket.java new file mode 100644 index 0000000..46980da --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/unix/LocalClientSocket.java
@@ -0,0 +1,117 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.unix; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.SocketException; + +/** + * <p>An implementation of client Socket for local (AF_UNIX) sockets. + * + * <p>This class intentionally doesn't extend java.net.Socket although it + * has some similarity to it. The java.net class hierarchy is a terrible mess + * and is inextricably coupled to the Internet Protocol. + * + * <p>This code is not intended to be portable to non-UNIX platforms. + */ +public class LocalClientSocket extends LocalSocket { + + /** + * Constructs an unconnected local client socket. + * + * @throws IOException if the socket could not be created. + */ + public LocalClientSocket() throws IOException { + super(); + } + + /** + * Constructs a client socket and connects it to the specified address. + * + * @throws IOException if either of the socket/connect operations failed. + */ + public LocalClientSocket(LocalSocketAddress address) throws IOException { + super(); + connect(address); + } + + /** + * Connect to the specified server. Blocks until the server accepts the + * connection. + * + * @throws IOException if the connection failed. + */ + public synchronized void connect(LocalSocketAddress address) + throws IOException { + checkNotClosed(); + if (state == State.CONNECTED) { + throw new SocketException("socket is already connected"); + } + connect(fd, address.getName().toString()); // JNI + this.address = address; + this.state = State.CONNECTED; + } + + /** + * Returns the input stream for reading from the server. + * + * @param closeSocket close the socket when this input stream is closed. + * @throws IOException if there was a problem. + */ + public synchronized InputStream getInputStream(final boolean closeSocket) throws IOException { + checkConnected(); + checkInputNotShutdown(); + return new FileInputStream(fd) { + @Override + public void close() throws IOException { + if (closeSocket) { + LocalClientSocket.this.close(); + } + } + }; + } + + /** + * Returns the input stream for reading from the server. + * + * @throws IOException if there was a problem. + */ + public synchronized InputStream getInputStream() throws IOException { + return getInputStream(false); + } + + /** + * Returns the output stream for writing to the server. + * + * @throws IOException if there was a problem. + */ + public synchronized OutputStream getOutputStream() throws IOException { + checkConnected(); + checkOutputNotShutdown(); + return new FileOutputStream(fd) { + @Override public void close() { + // Don't close the file descriptor. + } + }; + } + + @Override + public String toString() { + return "LocalClientSocket(" + address + ")"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/LocalServerSocket.java b/src/main/java/com/google/devtools/build/lib/unix/LocalServerSocket.java new file mode 100644 index 0000000..4eb1265 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/unix/LocalServerSocket.java
@@ -0,0 +1,173 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.unix; + +import java.io.FileDescriptor; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.Socket; +import java.net.SocketException; +import java.net.SocketTimeoutException; + +/** + * <p>An implementation of ServerSocket for local (AF_UNIX) sockets. + * + * <p>This class intentionally doesn't extend java.net.ServerSocket although it + * has some similarity to it. The java.net class hierarchy is a terrible mess + * and is inextricably coupled to the Internet Protocol. + * + * <p>This code is not intended to be portable to non-UNIX platforms. + */ +public class LocalServerSocket extends LocalSocket { + + // Socket timeout in milliseconds. No timeout by default. + private long soTimeoutMillis = 0; + + /** + * Constructs an unbound local server socket. + */ + public LocalServerSocket() throws IOException { + super(); + } + + /** + * Constructs a server socket, binds it to the specified address, and + * listens for incoming connections with the specified backlog. + * + * @throws IOException if any of the socket/bind/listen operations failed. + */ + public LocalServerSocket(LocalSocketAddress address, int backlog) + throws IOException { + this(); + bind(address); + listen(backlog); + } + + /** + * Constructs a server socket, binds it to the specified address, and begin + * listening for incoming connections using the default backlog. + * + * @throws IOException if any of the socket/bind/listen operations failed. + */ + public LocalServerSocket(LocalSocketAddress address) throws IOException { + this(address, 50); + } + + /** + * Specifies the timeout in milliseconds for accept(). Setting it to + * zero means an indefinite timeout. + */ + public void setSoTimeout(long timeoutMillis) { + soTimeoutMillis = timeoutMillis; + } + + /** + * Returns the current timeout in milliseconds. + */ + public long getSoTimeout() { + return soTimeoutMillis; + } + + /** + * Binds the specified address to this socket. The socket must be unbound. + * This causes the filesystem entry to appear. + * + * @throws IOException if the bind failed. + */ + public synchronized void bind(LocalSocketAddress address) + throws IOException { + if (address == null) { + throw new NullPointerException("address"); + } + checkNotClosed(); + if (state != State.NEW) { + throw new SocketException("socket is already bound to an address"); + } + bind(fd, address.getName().toString()); // JNI + this.address = address; + this.state = State.BOUND; + } + + /** + * Listen for incoming connections on a socket using the specfied backlog. + * The socket must be bound but not already listening. + * + * @throws IOException if the listen failed. + */ + public synchronized void listen(int backlog) throws IOException { + if (backlog < 1) { + throw new IllegalArgumentException("backlog=" + backlog); + } + checkNotClosed(); + if (address == null) { + throw new SocketException("socket has no address bound"); + } + if (state == State.LISTENING) { + throw new SocketException("socket is already listening"); + } + listen(fd, backlog); // JNI + this.state = State.LISTENING; + } + + /** + * Blocks until a connection is made to this socket and accepts it, returning + * a new socket connected to the client. + * + * @return the new socket connected to the client. + * @throws IOException if an error occurs when waiting for a connection. + * @throws SocketTimeoutException if a timeout was previously set with + * setSoTimeout and the timeout has been reached. + * @throws InterruptedIOException if the thread is interrupted when the + * method is blocked. + */ + public synchronized Socket accept() + throws IOException, SocketTimeoutException, InterruptedIOException { + if (state != State.LISTENING) { + throw new SocketException("socket is not in listening state"); + } + + // Throws a SocketTimeoutException if timeout. + if (soTimeoutMillis != 0) { + poll(fd, soTimeoutMillis); // JNI + } + + FileDescriptor clientFd = new FileDescriptor(); + accept(fd, clientFd); // JNI + final LocalSocketImpl impl = new LocalSocketImpl(clientFd); + return new Socket(impl) { + @Override + public boolean isConnected() { + return true; + } + @Override + public synchronized void close() throws IOException { + if (isClosed()) { + return; + } else { + super.close(); + // Workaround for the fact that super.created==false because we + // created the impl ourselves. As a result, super.close() doesn't + // call impl.close(). *Sigh*, java.net is horrendous. + // (Perhaps we should dispense with Socket/SocketImpl altogether?) + impl.close(); + } + } + }; + } + + @Override + public String toString() { + return "LocalServerSocket(" + address + ")"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/LocalSocket.java b/src/main/java/com/google/devtools/build/lib/unix/LocalSocket.java new file mode 100644 index 0000000..c9d1c91 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/unix/LocalSocket.java
@@ -0,0 +1,217 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.unix; + +import com.google.devtools.build.lib.UnixJniLoader; + +import java.io.Closeable; +import java.io.FileDescriptor; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.SocketException; +import java.net.SocketTimeoutException; + +/** + * Abstract superclass for client and server local sockets. + */ +abstract class LocalSocket implements Closeable { + + protected enum State { + NEW, + BOUND, // server only + LISTENING, // server only + CONNECTED, // client only + CLOSED, + } + + protected LocalSocketAddress address = null; + protected FileDescriptor fd = new FileDescriptor(); + protected State state; + protected boolean inputShutdown = false; + protected boolean outputShutdown = false; + + /** + * Constructs an unconnected local socket. + */ + protected LocalSocket() throws IOException { + socket(fd); + if (!fd.valid()) { + throw new IOException("Couldn't create socket!"); + } + this.state = State.NEW; + } + + /** + * Returns the address of the endpoint this socket is bound to. + * + * @return a <code>SocketAddress</code> representing the local endpoint of + * this socket. + */ + public LocalSocketAddress getLocalSocketAddress() { + return address; + } + + /** + * Closes this socket. This operation is idempotent. + * + * To be consistent with Java Socket, the shutdown states of the socket are + * not changed. This makes it easier to port applications between Socket and + * LocalSocket. + * + * @throws IOException if an I/O error occurred when closing the socket. + */ + @Override + public synchronized void close() throws IOException { + if (state == State.CLOSED) { + return; + } + // Closes the file descriptor if it has not been closed by the + // input/output streams. + if (!fd.valid()) { + throw new IllegalStateException("LocalSocket.close(-1)"); + } + close(fd); + if (fd.valid()) { + throw new IllegalStateException("LocalSocket.close() did not set fd to -1"); + } + this.state = State.CLOSED; + } + + /** + * Returns the closed state of the ServerSocket. + * + * @return true if the socket has been closed + */ + public synchronized boolean isClosed() { + // If the file descriptor has been closed by the input/output + // streams, marks the socket as closed too. + return state == State.CLOSED; + } + + /** + * Returns the connected state of the ClientSocket. + * + * @return true if the socket is currently connected. + */ + public synchronized boolean isConnected() { + return state == State.CONNECTED; + } + + protected synchronized void checkConnected() throws SocketException { + if (!isConnected()) { + throw new SocketException("Transport endpoint is not connected"); + } + } + + protected synchronized void checkNotClosed() throws SocketException { + if (isClosed()) { + throw new SocketException("socket is closed"); + } + } + + /** + * Returns the shutdown state of the input channel. + * + * @return true is the input channel of the socket is shutdown. + */ + public synchronized boolean isInputShutdown() { + return inputShutdown; + } + + /** + * Returns the shutdown state of the output channel. + * + * @return true is the input channel of the socket is shutdown. + */ + public synchronized boolean isOutputShutdown() { + return outputShutdown; + } + + protected synchronized void checkInputNotShutdown() throws SocketException { + if (isInputShutdown()) { + throw new SocketException("Socket input is shutdown"); + } + } + + protected synchronized void checkOutputNotShutdown() throws SocketException { + if (isOutputShutdown()) { + throw new SocketException("Socket output is shutdown"); + } + } + + static final int SHUT_RD = 0; // Mapped to BSD SHUT_RD in JNI. + static final int SHUT_WR = 1; // Mapped to BSD SHUT_WR in JNI. + + public synchronized void shutdownInput() throws IOException { + checkNotClosed(); + checkConnected(); + checkInputNotShutdown(); + inputShutdown = true; + shutdown(fd, SHUT_RD); + } + + public synchronized void shutdownOutput() throws IOException { + checkNotClosed(); + checkConnected(); + checkOutputNotShutdown(); + outputShutdown = true; + shutdown(fd, SHUT_WR); + } + + //////////////////////////////////////////////////////////////////////// + // JNI: + + static { + UnixJniLoader.loadJni(); + } + + // The native calls below are thin wrappers around linux system calls. The + // semantics remains the same except for poll(). See the comments for the + // method. + // + // Note: FileDescriptor is a box for a mutable integer that is visible only + // to native code. + + // Generic operations: + protected static native void socket(FileDescriptor server) + throws IOException; + static native void close(FileDescriptor server) + throws IOException; + /** + * Shut down part of a full-duplex connection + * @param code Must be either SHUT_RD or SHUT_WR + */ + static native void shutdown(FileDescriptor fd, int code) + throws IOException; + + /** + * This method checks waits for the given file descriptor to become available for read. + * If timeoutMillis passed and there is no activity, a SocketTimeoutException will be thrown. + */ + protected static native void poll(FileDescriptor read, long timeoutMillis) + throws IOException, SocketTimeoutException, InterruptedIOException; + + // Server operations: + protected static native void bind(FileDescriptor server, String filename) + throws IOException; + protected static native void listen(FileDescriptor server, int backlog) + throws IOException; + protected static native void accept(FileDescriptor server, + FileDescriptor client) + throws IOException; + + // Client operations: + protected static native void connect(FileDescriptor client, String filename) + throws IOException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/LocalSocketAddress.java b/src/main/java/com/google/devtools/build/lib/unix/LocalSocketAddress.java new file mode 100644 index 0000000..b92a04d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/unix/LocalSocketAddress.java
@@ -0,0 +1,56 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.unix; + +import java.io.File; +import java.net.SocketAddress; + +/** + * An implementation of SocketAddress for naming local sockets, i.e. files in + * the UNIX file system. + */ +public class LocalSocketAddress extends SocketAddress { + + private final File name; + + /** + * Constructs a SocketAddress for the specified file. + */ + public LocalSocketAddress(File name) { + this.name = name; + } + + /** + * Returns the filename of this local socket address. + */ + public File getName() { + return name; + } + + @Override + public String toString() { + return name.toString(); + } + + @Override + public boolean equals(Object other) { + return other instanceof LocalSocketAddress && + ((LocalSocketAddress) other).name.equals(this.name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/LocalSocketImpl.java b/src/main/java/com/google/devtools/build/lib/unix/LocalSocketImpl.java new file mode 100644 index 0000000..aee473a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/unix/LocalSocketImpl.java
@@ -0,0 +1,168 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.unix; + +import com.google.devtools.build.lib.UnixJniLoader; + +import java.io.Closeable; +import java.io.FileDescriptor; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.SocketAddress; +import java.net.SocketImpl; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A simple implementation of SocketImpl for sockets that wrap a UNIX + * file-descriptor. This SocketImpl assumes that the socket is already + * created, bound, connected and supports no socket options or out-of-band + * features. This is used to implement server-side accepted client sockets + * (i.e. those returned by {@link LocalServerSocket#accept}). + */ +class LocalSocketImpl extends SocketImpl { + private static final Logger logger = + Logger.getLogger(LocalSocketImpl.class.getName()); + + static { + UnixJniLoader.loadJni(); + init(); + } + + // The logic here is a little twisted, to support JDK7 and JDK8. + + // 1) In JDK7, the FileDescriptor class keeps a reference count of + // instances using the fd, and closes it when it goes to 0. The + // reference count is only decremented by the finalizer for a + // given class. When the call to close() happens, the fd is + // closed regardless of the current state of the refcount. + // + // 2) In JDK8, every instance that uses the fd registers a Closeable + // with the FileDescriptor. Since the FileDescriptor has a + // reference to every user, only when all of the users and the + // FileDescriptor get GC'd does the finalizer run. An explicit + // call to close() calls FileDescriptor.closeAll(), which + // force-closes all of the users. + + // So, in our case: + + // 1) ref() increments the refcount in JDK7, and registers with the + // FD in JDK8. + + // 2) unref() decrements the refcount in JDK7, and does nothing in + // JDK8. + + // 3) The finalizer decrements the refcount in JDK7, and simply + // calls close() in JDK8 (where we don't have to worry about + // multiple live users of the FD). The close() method itself is + // idempotent. + + // 4) close() calls fd.closeAll in JDK8, which, in turn, calls + // closer.close(). In JDK7, close() calls closer.close() + // explicitly. + private static native void init(); + private static native void ref(FileDescriptor fd, Closeable closeable); + private static native boolean unref(FileDescriptor fd); + private static native boolean close0(FileDescriptor fd, Closeable closeable); + + private final boolean isInitialized; + private final Closeable closer = new Closeable() { + AtomicBoolean isClosed = new AtomicBoolean(false); + @Override public void close() throws IOException { + if (isClosed.compareAndSet(false, true)) { + LocalSocket.close(fd); + } + } + }; + + // Note to callers: if you pass a FD into this constructor, this + // instance is now responsible for closing it (in the sense of + // LocalSocket.close()). If some other instance tries to close it, + // then terrible things will happen. + LocalSocketImpl(FileDescriptor fd) { + this.fd = fd; // (inherited field) + ref(fd, closer); + isInitialized = true; + } + + @Override protected void finalize() { + try { + if (isInitialized) { + if (!unref(fd)) { + // JDK8 codepath + close0(fd, closer); + } + } + } catch (Exception e) { + logger.log(Level.WARNING, "Unable to access FileDescriptor class - " + + "may cause a file descriptor leak", e); + } + } + @Override protected InputStream getInputStream() { + return new FileInputStream(getFileDescriptor()); + } + @Override protected OutputStream getOutputStream() { + return new FileOutputStream(getFileDescriptor()); + } + @Override protected void close() throws IOException { + if (fd.valid()) { + if (!close0(fd, closer)) { + // JDK7 codepath + closer.close(); + } + } + } + + // Unused: + @Override + public void setOption(int optID, Object value) { + throw new UnsupportedOperationException("setOption"); + } + @Override + public Object getOption(int optID) { + throw new UnsupportedOperationException("getOption"); + } + @Override protected void create(boolean stream) { + throw new UnsupportedOperationException("create"); + } + @Override protected void connect(String host, int port) { + throw new UnsupportedOperationException("connect"); + } + @Override protected void connect(InetAddress address, int port) { + throw new UnsupportedOperationException("connect2"); + } + @Override protected void connect(SocketAddress address, int timeout) { + throw new UnsupportedOperationException("connect3"); + } + @Override protected void bind(InetAddress host, int port) { + throw new UnsupportedOperationException("bind"); + } + @Override protected void listen(int backlog) { + throw new UnsupportedOperationException("listen"); + } + @Override protected void accept(SocketImpl s) { + throw new UnsupportedOperationException("accept"); + } + @Override protected int available() { + throw new UnsupportedOperationException("available"); + } + @Override protected void sendUrgentData(int i) { + throw new UnsupportedOperationException("sendUrgentData"); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/ProcessUtils.java b/src/main/java/com/google/devtools/build/lib/unix/ProcessUtils.java new file mode 100644 index 0000000..5288e17 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/unix/ProcessUtils.java
@@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.unix; + +import com.google.devtools.build.lib.UnixJniLoader; + + +/** + * Various utilities related to UNIX processes. + */ +public final class ProcessUtils { + + private ProcessUtils() {} + + static { + UnixJniLoader.loadJni(); + } + + /** + * Native wrapper around POSIX getgid(2). + * + * @return the real group ID of the current process. + */ + public static native int getgid(); + + /** + * Native wrapper around POSIX getpid(2) syscall. + * + * @return the process ID of this process. + */ + public static native int getpid(); + + /** + * Native wrapper around POSIX getuid(2). + * + * @return the real user ID of the current process. + */ + public static native int getuid(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/AbruptExitException.java b/src/main/java/com/google/devtools/build/lib/util/AbruptExitException.java new file mode 100644 index 0000000..ff62a7e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/AbruptExitException.java
@@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +/** + * An exception thrown by various error conditions that are severe enough to halt the command (e.g. + * even a --keep_going build). These typically need to signal to the handling code what happened. + * Therefore, these exceptions contain a recommended ExitCode allowing the exception to "set" a + * returned numeric exit code. + * + * When an instance of this exception is thrown, Blaze will try to halt as soon as reasonably + * possible. + */ +public class AbruptExitException extends Exception { + + private final ExitCode exitCode; + + public AbruptExitException(String message, ExitCode exitCode) { + super(message); + this.exitCode = exitCode; + } + + public AbruptExitException(String message, ExitCode exitCode, Throwable cause) { + super(message, cause); + this.exitCode = exitCode; + } + + public AbruptExitException(ExitCode exitCode, Throwable cause) { + super(cause); + this.exitCode = exitCode; + } + + public AbruptExitException(ExitCode exitCode) { + this.exitCode = exitCode; + } + + public ExitCode getExitCode() { + return exitCode; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/AbstractIndexer.java b/src/main/java/com/google/devtools/build/lib/util/AbstractIndexer.java new file mode 100644 index 0000000..4b61fe6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/AbstractIndexer.java
@@ -0,0 +1,37 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Abstract class for string indexers. + */ +abstract class AbstractIndexer implements StringIndexer { + + /** + * Conversion from String to byte[]. + */ + protected static byte[] string2bytes(String string) { + return string.getBytes(UTF_8); + } + + /** + * Conversion from byte[] to String. + */ + protected static String bytes2string(byte[] bytes) { + return new String(bytes, UTF_8); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/AnsiStrippingOutputStream.java b/src/main/java/com/google/devtools/build/lib/util/AnsiStrippingOutputStream.java new file mode 100644 index 0000000..6c6b878 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/AnsiStrippingOutputStream.java
@@ -0,0 +1,176 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * A pass-thru {@link OutputStream} that strips ANSI control codes. + */ +public class AnsiStrippingOutputStream extends OutputStream { + // The idea is straightforward: the regexp for ANSI control codes is + // \x1b\[[;0-9]*[a-zA-Z] . Implementing it as a stream is a little ugly, + // though. + + private enum State { + NORMAL, + AFTER_ESCAPE, + PARAMETER, + } + + private byte[] outputBuffer; + private int outputBufferPos; + + private static final int ESCAPE_BUFFER_LENGTH = 128; + private byte[] escapeCodeBuffer; + private int escapeCodeBufferPos; + private OutputStream output; + private State state; + + public AnsiStrippingOutputStream(OutputStream output) { + this.output = output; + escapeCodeBuffer = new byte[ESCAPE_BUFFER_LENGTH]; + escapeCodeBufferPos = 0; + state = State.NORMAL; + } + + @Override + public synchronized void write(int b) throws IOException { + // As per the contract of OutputStream.write(int) + byte[] array = { (byte) (b & 0xff) }; + write(array, 0, 1); + } + + @Override + public synchronized void write(byte b[], int off, int len) throws IOException { + int i = 0; + if (state == State.NORMAL) { + + // Avoid outputBuffer allocation entirely if that's possible + while ((i < len) && (b[off + i] != 0x1b)) { + i++; + } + if (i == len) { + output.write(b, off, len); + return; + } + } + + // In the worst case, the contents of the escape buffer and the contents + // of the input buffer are both copied to the output, so the length of the + // output buffer should be the sum of the length of both these buffers. + outputBuffer = new byte[len + ESCAPE_BUFFER_LENGTH]; + System.arraycopy(b, off, outputBuffer, 0, i); + outputBufferPos = i; + + for (; i < len; i++) { + processByte(b[off + i]); + } + + try { + output.write(outputBuffer, 0, outputBufferPos); + } finally { + outputBuffer = null; // Make it possible to garbage collect the array + } + } + + private void processByte(byte b) { + switch (state) { + case NORMAL: + if (escapeCodeBufferPos != 0) { + throw new IllegalStateException(); + } + if (b == 0x1b) { + state = State.AFTER_ESCAPE; + addByteToEscapeBuffer(b); + } else { + dumpByte(b); + } + break; + + case AFTER_ESCAPE: + if (b == '[') { + state = State.PARAMETER; + addByteToEscapeBuffer(b); + } else if (b == 0x1b) { + dumpEscapeBuffer(); + state = State.AFTER_ESCAPE; + addByteToEscapeBuffer(b); + } else { + dumpEscapeBuffer(); + dumpByte(b); + state = State.NORMAL; + } + break; + + case PARAMETER: + if ((b >= '0' && b <= '9') || b == ';') { + // Parameter continues + addByteToEscapeBuffer(b); + } else if ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z')) { + // Found a control sequence, discard it and revert to normal state + discardEscapeBuffer(); + state = State.NORMAL; + } else if (b == 0x1b) { + // Another escape sequence begins immediately after, and this is + // an illegal escape sequence + dumpEscapeBuffer(); + state = State.AFTER_ESCAPE; + addByteToEscapeBuffer(b); + } else { + // Illegal control sequence, output it + dumpEscapeBuffer(); + state = State.NORMAL; + } + break; + } + } + + private void addByteToEscapeBuffer(byte b) { + escapeCodeBuffer[escapeCodeBufferPos++] = b; + if (escapeCodeBufferPos == ESCAPE_BUFFER_LENGTH) { + // Buffer full. Assume that no sane code emits an ANSI control code this + // long and revert to normal state. + dumpEscapeBuffer(); + state = State.NORMAL; + } + } + + private void discardEscapeBuffer() { + escapeCodeBufferPos = 0; + } + + private void dumpByte(byte b) { + outputBuffer[outputBufferPos++] = b; + } + + private void dumpEscapeBuffer() { + System.arraycopy(escapeCodeBuffer, 0, + outputBuffer, outputBufferPos, escapeCodeBufferPos); + outputBufferPos += escapeCodeBufferPos; + escapeCodeBufferPos = 0; + } + + @Override + public void flush() throws IOException { + output.flush(); + } + + @Override + public void close() throws IOException { + output.close(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/BinaryPredicate.java b/src/main/java/com/google/devtools/build/lib/util/BinaryPredicate.java new file mode 100644 index 0000000..c7709e2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/BinaryPredicate.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import com.google.common.base.Predicate; + +import javax.annotation.Nullable; + +/** + * A two-argument version of {@link Predicate} that determines a true or false value for pairs of + * inputs. + * + * <p>Just as a {@link Predicate} is useful for filtering iterables of values, a {@link + * BinaryPredicate} is useful for filtering iterables of paired values, like {@link + * java.util.Map.Entry} or {@link Pair}. + * + * <p>See {@link Predicate} for implementation notes and advice. + */ +public interface BinaryPredicate<X, Y> { + + /** + * Applies this {@link BinaryPredicate} to the given objects. + * + * @return the value of this predicate when applied to inputs {@code x, y} + */ + boolean apply(@Nullable X x, @Nullable Y y); +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/BlazeClock.java b/src/main/java/com/google/devtools/build/lib/util/BlazeClock.java new file mode 100644 index 0000000..72806dd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/BlazeClock.java
@@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.JavaClock; + +/** + * Provides the clock implementation used by Blaze, which is {@link JavaClock} + * by default, but can be overridden at runtime. Note that if you set this + * clock, you also have to set the clock used by the Profiler. + */ +@ThreadSafe +public abstract class BlazeClock { + + private BlazeClock() { + } + + private static volatile Clock instance = new JavaClock(); + + /** + * Returns singleton instance of the clock + */ + public static Clock instance() { + return instance; + } + + /** + * Overrides default clock instance. + */ + public static synchronized void setClock(Clock clock) { + instance = clock; + } + + public static long nanoTime() { + return instance().nanoTime(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/CanonicalStringIndexer.java b/src/main/java/com/google/devtools/build/lib/util/CanonicalStringIndexer.java new file mode 100644 index 0000000..618e88b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/CanonicalStringIndexer.java
@@ -0,0 +1,113 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.util.StringCanonicalizer; + +import java.util.Map; + +/** + * A string indexer backed by a map and reverse index lookup. + * Every unique string is stored in memory exactly once. + */ +@ThreadSafe +public class CanonicalStringIndexer extends AbstractIndexer { + + private static final int NOT_FOUND = -1; + + // This is similar to (Synchronized) BiMap. + // These maps *must* be weakly threadsafe to ensure thread safety for string + // indexer as a whole. Specifically, mutating operations are serialized, but + // read-only operations may be executed concurrently with mutators. + private final Map<String, Integer> stringToInt; + private final Map<Integer, String> intToString; + + /* + * Creates an indexer instance from two backing maps. These maps may be + * pre-initialized with data, but *must*: + * a. Support read-only operations done concurrently with mutations. + * Note that mutations will be serialized. + * b. Be reverse mappings of each other, if pre-initialized. + */ + public CanonicalStringIndexer(Map<String, Integer> stringToInt, + Map<Integer, String> intToString) { + Preconditions.checkArgument(stringToInt.size() == intToString.size()); + this.stringToInt = stringToInt; + this.intToString = intToString; + } + + + @Override + public synchronized void clear() { + stringToInt.clear(); + intToString.clear(); + } + + @Override + public int size() { + return intToString.size(); + } + + @Override + public int getOrCreateIndex(String s) { + Integer i = stringToInt.get(s); + if (i == null) { + synchronized (this) { + // First, make sure another thread hasn't just added the entry: + i = stringToInt.get(s); + if (i != null) { + return i; + } + + int ind = intToString.size(); + s = StringCanonicalizer.intern(s); + stringToInt.put(s, ind); + intToString.put(ind, s); + return ind; + } + } else { + return i; + } + } + + @Override + public int getIndex(String s) { + Integer i = stringToInt.get(s); + return (i == null) ? NOT_FOUND : i; + } + + @Override + public synchronized boolean addString(String s) { + int originalSize = size(); + getOrCreateIndex(s); + return (size() > originalSize); + } + + @Override + public String getStringForIndex(int i) { + return intToString.get(i); + } + + @Override + public synchronized String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("size = ").append(size()).append("\n"); + for (Map.Entry<String, Integer> entry : stringToInt.entrySet()) { + builder.append(entry.getKey()).append(" <==> ").append(entry.getValue()).append("\n"); + } + return builder.toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/Clock.java b/src/main/java/com/google/devtools/build/lib/util/Clock.java new file mode 100644 index 0000000..878cb11 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/Clock.java
@@ -0,0 +1,33 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +/** + * This class provides an interface for a pluggable clock. + */ +public interface Clock { + + /** + * Returns the current time in milliseconds. The milliseconds are counted from midnight + * Jan 1, 1970. + */ + long currentTimeMillis(); + + /** + * Returns the current time in nanoseconds. The nanoseconds are measured relative to some + * unknown, but fixed event. Unfortunately, a sequence of calls to this method is *not* + * guaranteed to return non-decreasing values, so callers should be tolerant to this behavior. + */ + long nanoTime(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/CommandBuilder.java b/src/main/java/com/google/devtools/build/lib/util/CommandBuilder.java new file mode 100644 index 0000000..372802d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/CommandBuilder.java
@@ -0,0 +1,176 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.CharMatcher; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.shell.Command; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Implements OS aware {@link Command} builder. At this point only Linux, Mac + * and Windows XP are supported. + * + * <p>Builder will also apply heuristic to identify trivial cases where + * unix-like command lines could be automatically converted into the + * Windows-compatible form. + * + * <p>TODO(bazel-team): (2010) Some of the code here is very similar to the + * {@link com.google.devtools.build.lib.shell.Shell} class. This should be looked at. + */ +public final class CommandBuilder { + + private static final List<String> SHELLS = ImmutableList.of("/bin/sh", "/bin/bash"); + + private static final Splitter ARGV_SPLITTER = Splitter.on(CharMatcher.anyOf(" \t")); + + private final OS system; + private final List<String> argv = new ArrayList<>(); + private final Map<String, String> env = new HashMap<>(); + private File workingDir = null; + private boolean useShell = false; + + public CommandBuilder() { + this(OS.getCurrent()); + } + + @VisibleForTesting + CommandBuilder(OS system) { + this.system = system; + } + + public CommandBuilder addArg(String arg) { + Preconditions.checkNotNull(arg, "Argument must not be null"); + argv.add(arg); + return this; + } + + public CommandBuilder addArgs(Iterable<String> args) { + Preconditions.checkArgument(!Iterables.contains(args, null), "Arguments must not be null"); + Iterables.addAll(argv, args); + return this; + } + + public CommandBuilder addArgs(String... args) { + return addArgs(Arrays.asList(args)); + } + + public CommandBuilder addEnv(Map<String, String> env) { + Preconditions.checkNotNull(env); + this.env.putAll(env); + return this; + } + + public CommandBuilder emptyEnv() { + env.clear(); + return this; + } + + public CommandBuilder setEnv(Map<String, String> env) { + emptyEnv(); + addEnv(env); + return this; + } + + public CommandBuilder setWorkingDir(Path path) { + Preconditions.checkNotNull(path); + workingDir = path.getPathFile(); + return this; + } + + public CommandBuilder useTempDir() { + workingDir = new File(System.getProperty("java.io.tmpdir")); + return this; + } + + public CommandBuilder useShell(boolean useShell) { + this.useShell = useShell; + return this; + } + + private boolean argvStartsWithSh() { + return argv.size() >= 2 && SHELLS.contains(argv.get(0)) && "-c".equals(argv.get(1)); + } + + private String[] transformArgvForLinux() { + // If command line already starts with "/bin/sh -c", ignore useShell attribute. + if (useShell && !argvStartsWithSh()) { + // c.g.io.base.shell.Shell.shellify() actually concatenates argv into the space-separated + // string here. Not sure why, but we will do the same. + return new String[] { "/bin/sh", "-c", Joiner.on(' ').join(argv) }; + } + return argv.toArray(new String[argv.size()]); + } + + private String[] transformArgvForWindows() { + List<String> modifiedArgv; + // Heuristic: replace "/bin/sh -c" with something more appropriate for Windows. + if (argvStartsWithSh()) { + useShell = true; + modifiedArgv = Lists.newArrayList(argv.subList(2, argv.size())); + } else { + modifiedArgv = Lists.newArrayList(argv); + } + + if (!modifiedArgv.isEmpty()) { + // args can contain whitespace, so figure out the first word + String argv0 = modifiedArgv.get(0); + String command = ARGV_SPLITTER.split(argv0).iterator().next(); + + // Automatically enable CMD.EXE use if we are executing something else besides "*.exe" file. + if (!command.toLowerCase().endsWith(".exe")) { + useShell = true; + } + } else { + // This is degenerate "/bin/sh -c" case. We ensure that Windows behavior is identical + // to the Linux - call shell that will do nothing. + useShell = true; + } + if (useShell) { + // /S - strip first and last quotes and execute everything else as is. + // /E:ON - enable extended command set. + // /V:ON - enable delayed variable expansion + // /D - ignore AutoRun registry entries. + // /C - execute command. This must be the last option before the command itself. + return new String[] { "CMD.EXE", "/S", "/E:ON", "/V:ON", "/D", "/C", + "\"" + Joiner.on(' ').join(modifiedArgv) + "\"" }; + } else { + return modifiedArgv.toArray(new String[argv.size()]); + } + } + + public Command build() { + Preconditions.checkState(system != OS.UNKNOWN, "Unidentified operating system"); + Preconditions.checkNotNull(workingDir, "Working directory must be set"); + Preconditions.checkState(argv.size() > 0, "At least one argument is expected"); + + return new Command( + system == OS.WINDOWS ? transformArgvForWindows() : transformArgvForLinux(), + env, workingDir); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/CommandDescriptionForm.java b/src/main/java/com/google/devtools/build/lib/util/CommandDescriptionForm.java new file mode 100644 index 0000000..8d37275 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/CommandDescriptionForm.java
@@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +/** + * Forms in which a command can be described by {@link CommandFailureUtils#describeCommand}. + */ +public enum CommandDescriptionForm { + /** + * A form that is usually suitable for identifying the command but not for + * re-executing it. The working directory and environment are not shown, and + * the arguments are truncated to a maximum of a few hundred bytes. + */ + ABBREVIATED, + + /** + * A form that is complete and suitable for a user to copy and paste into a + * shell. On Linux, the command is placed in a subshell so it has no side + * effects on the user's shell. On Windows, this is not implemented, but the + * side effects in question are less severe (no "exec"). + */ + COMPLETE, + + /** + * A form that is complete and does not isolate side effects. Suitable for + * launch scripts, i.e., "blaze run --script_path". + */ + COMPLETE_UNISOLATED, +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/CommandFailureUtils.java b/src/main/java/com/google/devtools/build/lib/util/CommandFailureUtils.java new file mode 100644 index 0000000..9178f985 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/CommandFailureUtils.java
@@ -0,0 +1,252 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; + +import java.io.File; +import java.util.Collection; +import java.util.Comparator; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Utility methods for describing command failures. + * See also the CommandUtils class. + * Unlike that one, this class does not depend on Command; + * instead, it just manipulates command lines represented as + * Collection<String>. + */ +public class CommandFailureUtils { + + // Interface that provides building blocks when describing command. + private interface DescribeCommandImpl { + void describeCommandBeginIsolate(StringBuilder message); + void describeCommandEndIsolate(StringBuilder message); + void describeCommandCwd(String cwd, StringBuilder message); + void describeCommandEnvPrefix(StringBuilder message); + void describeCommandEnvVar(StringBuilder message, Map.Entry<String, String> entry); + void describeCommandElement(StringBuilder message, String commandElement); + void describeCommandExec(StringBuilder message); + } + + private static final class LinuxDescribeCommandImpl implements DescribeCommandImpl { + + @Override + public void describeCommandBeginIsolate(StringBuilder message) { + message.append("("); + } + + @Override + public void describeCommandEndIsolate(StringBuilder message) { + message.append(")"); + } + + @Override + public void describeCommandCwd(String cwd, StringBuilder message) { + message.append("cd ").append(ShellEscaper.escapeString(cwd)).append(" && \\\n "); + } + + @Override + public void describeCommandEnvPrefix(StringBuilder message) { + message.append("env - \\\n "); + } + + @Override + public void describeCommandEnvVar(StringBuilder message, Map.Entry<String, String> entry) { + message.append(ShellEscaper.escapeString(entry.getKey())).append('=') + .append(ShellEscaper.escapeString(entry.getValue())).append(" \\\n "); + } + + @Override + public void describeCommandElement(StringBuilder message, String commandElement) { + message.append(ShellEscaper.escapeString(commandElement)); + } + + @Override + public void describeCommandExec(StringBuilder message) { + message.append("exec "); + } + } + + // TODO(bazel-team): (2010) Add proper escaping. We can't use ShellUtils.shellEscape() as it is + // incompatible with CMD.EXE syntax, but something else might be needed. + private static final class WindowsDescribeCommandImpl implements DescribeCommandImpl { + + @Override + public void describeCommandBeginIsolate(StringBuilder message) { + // TODO(bazel-team): Implement this. + } + + @Override + public void describeCommandEndIsolate(StringBuilder message) { + // TODO(bazel-team): Implement this. + } + + @Override + public void describeCommandCwd(String cwd, StringBuilder message) { + message.append("cd ").append(cwd).append("\n"); + } + + @Override + public void describeCommandEnvPrefix(StringBuilder message) { } + + @Override + public void describeCommandEnvVar(StringBuilder message, Map.Entry<String, String> entry) { + message.append("SET ").append(entry.getKey()).append('=') + .append(entry.getValue()).append("\n "); + } + + @Override + public void describeCommandElement(StringBuilder message, String commandElement) { + message.append(commandElement); + } + + @Override + public void describeCommandExec(StringBuilder message) { + // TODO(bazel-team): Implement this if possible for greater efficiency. + } + } + + private static final DescribeCommandImpl describeCommandImpl = + OS.getCurrent() == OS.WINDOWS ? new WindowsDescribeCommandImpl() + : new LinuxDescribeCommandImpl(); + + private CommandFailureUtils() {} // Prevent instantiation. + + private static Comparator<Map.Entry<String, String>> mapEntryComparator = + new Comparator<Map.Entry<String, String>>() { + @Override + public int compare(Map.Entry<String, String> x, Map.Entry<String, String> y) { + // A map can never have two keys with the same value, so we only need to compare the keys. + return x.getKey().compareTo(y.getKey()); + } + }; + + /** + * Construct a string that describes the command. + * Currently this returns a message of the form "foo bar baz", + * with shell meta-characters appropriately quoted and/or escaped, + * prefixed (if verbose is true) with an "env" command to set the environment. + * + * @param form Form of the command to generate; see the documentation of the + * {@link CommandDescriptionForm} values. + */ + public static String describeCommand(CommandDescriptionForm form, + Collection<String> commandLineElements, + @Nullable Map<String, String> environment, @Nullable String cwd) { + Preconditions.checkNotNull(form); + final int APPROXIMATE_MAXIMUM_MESSAGE_LENGTH = 200; + StringBuilder message = new StringBuilder(); + int size = commandLineElements.size(); + int numberRemaining = size; + if (form == CommandDescriptionForm.COMPLETE) { + describeCommandImpl.describeCommandBeginIsolate(message); + } + if (form != CommandDescriptionForm.ABBREVIATED) { + if (cwd != null) { + describeCommandImpl.describeCommandCwd(cwd, message); + } + /* + * On Linux, insert an "exec" keyword to save a fork in "blaze run" + * generated scripts. If we use "env" as a wrapper, the "exec" needs to + * be applied to the entire "env" invocation. + * + * On Windows, this is a no-op. + */ + describeCommandImpl.describeCommandExec(message); + /* + * Java does not provide any way to invoke a subprocess with the environment variables + * in a specified order. The order of environment variables in the 'environ' array + * (which is set by the 'envp' parameter to the execve() system call) + * is determined by the order of iteration on a HashMap constructed inside Java's + * ProcessBuilder class (in the ProcessEnvironment class), which is nondeterministic. + * + * Nevertheless, we *print* the environment variables here in sorted order, rather + * than in the potentially nondeterministic order that will be actually used. + * This is slightly dubious... in theory a process's behaviour could depend on the order + * of the environment variables passed to it. (For example, the order of environment + * variables in the environ array affects the output of '/usr/bin/env'.) + * However, in practice very few processes depend on the order of the environment + * variables, and using a deterministic sorted order here makes Blaze's output more + * deterministic and easier to read. So this seems the lesser of two evils... I think. + * Anyway, it's not like we have much choice... even if we wanted to, there's no way to + * print out the nondeterministic order that will actually be used, since there's + * no way to guarantee that the iteration over entrySet() here will return the same + * sequence as the iteration over entrySet() inside the ProcessBuilder class + * (in ProcessEnvironment.StringEnvironment.toEnvironmentBlock()). + */ + if (environment != null) { + describeCommandImpl.describeCommandEnvPrefix(message); + for (Map.Entry<String, String> entry : + Ordering.from(mapEntryComparator).sortedCopy(environment.entrySet())) { + message.append(" "); + describeCommandImpl.describeCommandEnvVar(message, entry); + } + } + } + for (String commandElement : commandLineElements) { + if (form == CommandDescriptionForm.ABBREVIATED && + message.length() + commandElement.length() > APPROXIMATE_MAXIMUM_MESSAGE_LENGTH) { + message.append( + " ... (remaining " + numberRemaining + " argument(s) skipped)"); + break; + } else { + if (numberRemaining < size) { + message.append(' '); + } + describeCommandImpl.describeCommandElement(message, commandElement); + numberRemaining--; + } + } + if (form == CommandDescriptionForm.COMPLETE) { + describeCommandImpl.describeCommandEndIsolate(message); + } + return message.toString(); + } + + /** + * Construct an error message that describes a failed command invocation. + * Currently this returns a message of the form "error executing command foo + * bar baz". + */ + public static String describeCommandError(boolean verbose, + Collection<String> commandLineElements, + Map<String, String> env, String cwd) { + CommandDescriptionForm form = verbose + ? CommandDescriptionForm.COMPLETE + : CommandDescriptionForm.ABBREVIATED; + return "error executing command " + (verbose ? "\n " : "") + + describeCommand(form, commandLineElements, env, cwd); + } + + /** + * Construct an error message that describes a failed command invocation. + * Currently this returns a message of the form "foo failed: error executing + * command /dir/foo bar baz". + */ + public static String describeCommandFailure(boolean verbose, + Collection<String> commandLineElements, + Map<String, String> env, String cwd) { + String commandName = commandLineElements.iterator().next(); + // Extract the part of the command name after the last "/", if any. + String shortCommandName = new File(commandName).getName(); + return shortCommandName + " failed: " + + describeCommandError(verbose, commandLineElements, env, cwd); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/CommandUtils.java b/src/main/java/com/google/devtools/build/lib/util/CommandUtils.java new file mode 100644 index 0000000..e6c0011 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/CommandUtils.java
@@ -0,0 +1,88 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.devtools.build.lib.shell.AbnormalTerminationException; +import com.google.devtools.build.lib.shell.Command; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.shell.CommandResult; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; + +/** + * Utility methods relating to the {@link Command} class. + */ +public class CommandUtils { + + private CommandUtils() {} // Prevent instantiation. + + private static Collection<String> commandLine(Command command) { + return Arrays.asList(command.getCommandLineElements()); + } + + private static Map<String, String> env(Command command) { + return command.getEnvironmentVariables(); + } + + private static String cwd(Command command) { + return command.getWorkingDirectory() == null ? null : command.getWorkingDirectory().getPath(); + } + + /** + * Construct an error message that describes a failed command invocation. + * Currently this returns a message of the form "error executing command foo + * bar baz". + */ + public static String describeCommandError(boolean verbose, Command command) { + return CommandFailureUtils.describeCommandError(verbose, commandLine(command), env(command), + cwd(command)); + } + + /** + * Construct an error message that describes a failed command invocation. + * Currently this returns a message of the form "foo failed: error executing + * command /dir/foo bar baz". + */ + public static String describeCommandFailure(boolean verbose, Command command) { + return CommandFailureUtils.describeCommandFailure(verbose, commandLine(command), env(command), + cwd(command)); + } + + /** + * Construct an error message that describes a failed command invocation. + * Currently this returns a message of the form "foo failed: error executing + * command /dir/foo bar baz: exception message", with the + * command's stdout and stderr output appended if available. + */ + public static String describeCommandFailure(boolean verbose, CommandException exception) { + String message = describeCommandFailure(verbose, exception.getCommand()) + ": " + + exception.getMessage(); + if (exception instanceof AbnormalTerminationException) { + CommandResult result = ((AbnormalTerminationException) exception).getResult(); + try { + return message + "\n" + + new String(result.getStdout()) + + new String(result.getStderr()); + } catch (IllegalStateException e) { + // This can happen if the command didn't save stdout/stderr, + // so ignore this exception and fall through to the ordinary case. + } + } + return message; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/CompactStringIndexer.java b/src/main/java/com/google/devtools/build/lib/util/CompactStringIndexer.java new file mode 100644 index 0000000..698758d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/CompactStringIndexer.java
@@ -0,0 +1,546 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +import java.util.ArrayList; + +/** + * Provides memory-efficient bidirectional mapping String <-> unique integer. + * Uses byte-wise compressed prefix trie internally. + * <p> + * Class allows index retrieval for the given string, addition of the new + * index and string retrieval for the given index. It also allows efficient + * serialization of the internal data structures. + * <p> + * Internally class stores list of nodes with each node containing byte[] + * representation of compressed trie node: + * <pre> + * varint32 parentIndex; // index of the parent node + * varint32 keylen; // length of the node key + * byte[keylen] key; // node key data + * repeated jumpEntry { // Zero or more jump entries, referencing child nodes + * byte key // jump key (first byte of the child node key) + * varint32 nodeIndex // child index + * } + * <p> + * Note that jumpEntry key byte is actually duplicated in the child node + * instance. This is done to improve performance of the index->string + * lookup (so we can avoid jump table parsing during this lookup). + * <p> + * Root node of the trie must have parent id pointing to itself. + * <p> + * TODO(bazel-team): (2010) Consider more fine-tuned locking mechanism - e.g. + * distinguishing between read and write locks. + */ +@ThreadSafe +public class CompactStringIndexer extends AbstractIndexer { + + private static final int NOT_FOUND = -1; + + private ArrayList<byte[]> nodes; // Compressed prefix trie nodes. + private int rootId; // Root node id. + + /* + * Creates indexer instance. + */ + public CompactStringIndexer (int expectedCapacity) { + Preconditions.checkArgument(expectedCapacity > 0); + nodes = Lists.newArrayListWithExpectedSize(expectedCapacity); + rootId = NOT_FOUND; + } + + /** + * Allocates new node index. Must be called only from + * synchronized methods. + */ + private int allocateIndex() { + nodes.add(null); + return nodes.size() - 1; + } + + /** + * Replaces given node record with the new one. Must be called only from + * synchronized methods. + * <p> + * Subclasses can override this method to be notified when an update actually + * takes place. + */ + @ThreadCompatible + protected void updateNode(int index, byte[] content) { + nodes.set(index, content); + } + + /** + * Returns parent id for the given node content. + * + * @return parent node id + */ + private int getParentId(byte[] content) { + int[] intHolder = new int[1]; + VarInt.getVarInt(content, 0, intHolder); + return intHolder[0]; + } + + /** + * Creates new node using specified key suffix. Must be called from + * synchronized methods. + * + * @param parentNode parent node id + * @param key original key that is being added to the indexer + * @param offset node key offset in the original key. + * + * @return new node id corresponding to the given key + */ + private int createNode(int parentNode, byte[] key, int offset) { + int index = allocateIndex(); + + int len = key.length - offset; + Preconditions.checkState(len >= 0); + + // Content consists of parent id, key length and key. There are no jump records. + byte[] content = new byte[VarInt.varIntSize(parentNode) + VarInt.varIntSize(len) + len]; + // Add parent id. + int contentOffset = VarInt.putVarInt(parentNode, content, 0); + // Add node key length. + contentOffset = VarInt.putVarInt(len, content, contentOffset); + // Add node key content. + System.arraycopy(key, offset, content, contentOffset, len); + + updateNode(index, content); + return index; + } + + /** + * Updates jump entry index in the given node. + * + * @param node node id to update + * @param oldIndex old jump entry index + * @param newIndex updated jump entry index + */ + private void updateJumpEntry(int node, int oldIndex, int newIndex) { + byte[] content = nodes.get(node); + int[] intHolder = new int[1]; + int offset = VarInt.getVarInt(content, 0, intHolder); // parent id + offset = VarInt.getVarInt(content, offset, intHolder); // key length + offset += intHolder[0]; // Offset now points to the first jump entry. + while (offset < content.length) { + int next = VarInt.getVarInt(content, offset + 1, intHolder); // jump index + if (intHolder[0] == oldIndex) { + // Substitute oldIndex value with newIndex. + byte[] newContent = + new byte[content.length + VarInt.varIntSize(newIndex) - VarInt.varIntSize(oldIndex)]; + System.arraycopy(content, 0, newContent, 0, offset + 1); + offset = VarInt.putVarInt(newIndex, newContent, offset + 1); + System.arraycopy(content, next, newContent, offset, content.length - next); + updateNode(node, newContent); + return; + } else { + offset = next; + } + } + StringBuilder builder = new StringBuilder().append("Index ").append(oldIndex) + .append(" is not present in the node ").append(node).append(", "); + dumpNodeContent(builder, content); + throw new IllegalArgumentException(builder.toString()); + } + + /** + * Creates new branch node content at the predefined location, splitting + * prefix from the given node and optionally adding another child node + * jump entry. + * + * @param originalNode node that will be split + * @param newBranchNode new branch node id + * @param splitOffset offset at which to split original node key + * @param indexKey optional additional jump key + * @param childIndex optional additional jump index. Optional jump entry will + * be skipped if this index is set to NOT_FOUND. + */ + private void createNewBranchNode(int originalNode, int newBranchNode, int splitOffset, + byte indexKey, int childIndex) { + byte[] content = nodes.get(originalNode); + int[] intHolder = new int[1]; + int keyOffset = VarInt.getVarInt(content, 0, intHolder); // parent id + + // If original node is a root node, new branch node will become new root. So set parent id + // appropriately (for root node it is set to the node's own id). + int parentIndex = (originalNode == intHolder[0] ? newBranchNode : intHolder[0]); + + keyOffset = VarInt.getVarInt(content, keyOffset, intHolder); // key length + Preconditions.checkState(intHolder[0] >= splitOffset); + // Calculate new content size. + int newSize = VarInt.varIntSize(parentIndex) + + VarInt.varIntSize(splitOffset) + splitOffset + + 1 + VarInt.varIntSize(originalNode) + + (childIndex != NOT_FOUND ? 1 + VarInt.varIntSize(childIndex) : 0); + // New content consists of parent id, new key length, truncated key and two jump records. + byte[] newContent = new byte[newSize]; + // Add parent id. + int contentOffset = VarInt.putVarInt(parentIndex, newContent, 0); + // Add adjusted key length. + contentOffset = VarInt.putVarInt(splitOffset, newContent, contentOffset); + // Add truncated key content and first jump key. + System.arraycopy(content, keyOffset, newContent, contentOffset, splitOffset + 1); + // Add index for the first jump key. + contentOffset = VarInt.putVarInt(originalNode, newContent, contentOffset + splitOffset + 1); + // If requested, add additional jump entry. + if (childIndex != NOT_FOUND) { + // Add second jump key. + newContent[contentOffset] = indexKey; + // Add index for the second jump key. + VarInt.putVarInt(childIndex, newContent, contentOffset + 1); + } + updateNode(newBranchNode, newContent); + } + + /** + * Inject newly created branch node into the trie data structure. Method + * will update parent node jump entry to point to the new branch node (or + * will update root id if branch node becomes new root) and will truncate + * key prefix from the original node that was split (that prefix now + * resides in the branch node). + * + * @param originalNode node that will be split + * @param newBranchNode new branch node id + * @param commonPrefixLength how many bytes should be split into the new branch node. + */ + private void injectNewBranchNode(int originalNode, int newBranchNode, int commonPrefixLength) { + byte[] content = nodes.get(originalNode); + + int parentId = getParentId(content); + if (originalNode == parentId) { + rootId = newBranchNode; // update root index + } else { + updateJumpEntry(parentId, originalNode, newBranchNode); + } + + // Truncate prefix from the original node and set its parent to the our new branch node. + int[] intHolder = new int[1]; + int suffixOffset = VarInt.getVarInt(content, 0, intHolder); // parent id + suffixOffset = VarInt.getVarInt(content, suffixOffset, intHolder); // key length + int len = intHolder[0] - commonPrefixLength; + Preconditions.checkState(len >= 0); + suffixOffset += commonPrefixLength; + // New content consists of parent id, new key length and duplicated key suffix. + byte[] newContent = new byte[VarInt.varIntSize(newBranchNode) + VarInt.varIntSize(len) + + (content.length - suffixOffset)]; + // Add parent id. + int contentOffset = VarInt.putVarInt(newBranchNode, newContent, 0); + // Add new key length. + contentOffset = VarInt.putVarInt(len, newContent, contentOffset); + // Add key and jump table. + System.arraycopy(content, suffixOffset, newContent, contentOffset, + content.length - suffixOffset); + updateNode(originalNode, newContent); + } + + /** + * Adds new child node (that uses specified key suffix) to the given + * current node. + * Example: + * <pre> + * Had "ab". Adding "abcd". + * + * 1:"ab",'c'->2 + * 1:"ab" -> \ + * 2:"cd" + * </pre> + */ + private int addChildNode(int parentNode, byte[] key, int keyOffset) { + int child = createNode(parentNode, key, keyOffset); + + byte[] content = nodes.get(parentNode); + // Add jump table entry to the parent node. + int entryOffset = content.length; + // New content consists of original content and additional jump record. + byte[] newContent = new byte[entryOffset + 1 + VarInt.varIntSize(child)]; + // Copy original content. + System.arraycopy(content, 0, newContent, 0, entryOffset); + // Add jump key. + newContent[entryOffset] = key[keyOffset]; + // Add jump index. + VarInt.putVarInt(child, newContent, entryOffset + 1); + + updateNode(parentNode, newContent); + return child; + } + + /** + * Splits node into two at the specified offset. + * Example: + * <pre> + * Had "abcd". Adding "ab". + * + * 2:"ab",'c'->1 + * 1:"abcd" -> \ + * 1:"cd" + * </pre> + */ + private int splitNodeSuffix(int nodeToSplit, int commonPrefixLength) { + int newBranchNode = allocateIndex(); + // Create new node with truncated key. + createNewBranchNode(nodeToSplit, newBranchNode, commonPrefixLength, (byte) 0, NOT_FOUND); + injectNewBranchNode(nodeToSplit, newBranchNode, commonPrefixLength); + + return newBranchNode; + } + + /** + * Splits node into two at the specified offset and adds another leaf. + * Example: + * <pre> + * Had "abcd". Adding "abef". + * + * 3:"ab",'c'->1,'e'->2 + * 1:"abcd" -> / \ + * 1:"cd" 2:"ef" + * </pre> + */ + private int addBranch(int nodeToSplit, byte[] key, int offset, int commonPrefixLength) { + int newBranchNode = allocateIndex(); + int child = createNode(newBranchNode, key, offset + commonPrefixLength); + // Create new node with the truncated key and reference to the new child node. + createNewBranchNode(nodeToSplit, newBranchNode, commonPrefixLength, + key[offset + commonPrefixLength], child); + injectNewBranchNode(nodeToSplit, newBranchNode, commonPrefixLength); + + return child; + } + + private int findOrCreateIndexInternal(int node, byte[] key, int offset, + boolean createIfNotFound) { + byte[] content = nodes.get(node); + int[] intHolder = new int[1]; + int contentOffset = VarInt.getVarInt(content, 0, intHolder); // parent id + contentOffset = VarInt.getVarInt(content, contentOffset, intHolder); // key length + int skyKeyLen = intHolder[0]; + int remainingKeyLen = key.length - offset; + int minKeyLen = remainingKeyLen > skyKeyLen ? skyKeyLen : remainingKeyLen; + + // Compare given key/offset content with the node key. Skip first key byte for recursive + // calls - this byte is equal to the byte in the jump entry and was already compared. + for (int i = (offset > 0 ? 1 : 0); i < minKeyLen; i++) { // compare key + if (key[offset + i] != content[contentOffset + i]) { + // Mismatch found somewhere in the middle of the node key. If requested, node + // should be split and another leaf added for the new key. + return createIfNotFound ? addBranch(node, key, offset, i) : NOT_FOUND; + } + } + + if (remainingKeyLen > minKeyLen) { + // Node key matched portion of the key - find appropriate jump entry. If found - recursion. + // If not - mismatch (we will add new child node if requested). + contentOffset += skyKeyLen; + while (contentOffset < content.length) { + if (key[offset + skyKeyLen] == content[contentOffset]) { // compare index value + VarInt.getVarInt(content, contentOffset + 1, intHolder); + // Found matching jump entry - recursively compare the child. + return findOrCreateIndexInternal(intHolder[0], key, offset + skyKeyLen, + createIfNotFound); + } else { + // Jump entry key does not match. Skip rest of the entry data. + contentOffset = VarInt.getVarInt(content, contentOffset + 1, intHolder); + } + } + // There are no matching jump entries - report mismatch or create a new leaf if necessary. + return createIfNotFound ? addChildNode(node, key, offset + skyKeyLen) : NOT_FOUND; + } else if (skyKeyLen > minKeyLen) { + // Key suffix is a subset of the node key. Report mismatch or split the node if requested). + return createIfNotFound ? splitNodeSuffix(node, minKeyLen) : NOT_FOUND; + } else { + // Node key exactly matches key suffix - return associated index value. + return node; + } + } + + private synchronized int findOrCreateIndex(byte[] key, boolean createIfNotFound) { + if (rootId == NOT_FOUND) { + // Root node does not seem to exist - create it if needed. + if (createIfNotFound) { + rootId = createNode(0, key, 0); + Preconditions.checkState(rootId == 0); + return 0; + } else { + return NOT_FOUND; + } + } + return findOrCreateIndexInternal(rootId, key, 0, createIfNotFound); + } + + private byte[] reconstructKeyInternal(int node, int suffixSize) { + byte[] content = nodes.get(node); + Preconditions.checkNotNull(content); + int[] intHolder = new int[1]; + int contentOffset = VarInt.getVarInt(content, 0, intHolder); // parent id + int parentNode = intHolder[0]; + contentOffset = VarInt.getVarInt(content, contentOffset, intHolder); // key length + int len = intHolder[0]; + byte[] key; + if (node != parentNode) { + // We haven't reached root node yet. Make a recursive call, adjusting suffix length. + key = reconstructKeyInternal(parentNode, suffixSize + len); + } else { + // We are in a root node. Finally allocate array for the key. It will be filled up + // on our way back from recursive call tree. + key = new byte[suffixSize + len]; + } + // Fill appropriate portion of the full key with the node key content. + System.arraycopy(content, contentOffset, key, key.length - suffixSize - len, len); + return key; + } + + private byte[] reconstructKey(int node) { + return reconstructKeyInternal(node, 0); + } + + /* (non-Javadoc) + * @see com.google.devtools.build.lib.util.StringIndexer#clear() + */ + @Override + public synchronized void clear() { + nodes.clear(); + } + + /* (non-Javadoc) + * @see com.google.devtools.build.lib.util.StringIndexer#size() + */ + @Override + public synchronized int size() { + return nodes.size(); + } + + protected int getOrCreateIndexForBytes(byte[] bytes) { + return findOrCreateIndex(bytes, true); + } + + protected synchronized boolean addBytes(byte[] bytes) { + int count = nodes.size(); + int index = getOrCreateIndexForBytes(bytes); + return index >= count; + } + + protected int getIndexForBytes(byte[] bytes) { + return findOrCreateIndex(bytes, false); + } + + /* (non-Javadoc) + * @see com.google.devtools.build.lib.util.StringIndexer#getOrCreateIndex(java.lang.String) + */ + @Override + public int getOrCreateIndex(String s) { + return getOrCreateIndexForBytes(string2bytes(s)); + } + + /* (non-Javadoc) + * @see com.google.devtools.build.lib.util.StringIndexer#getIndex(java.lang.String) + */ + @Override + public int getIndex(String s) { + return getIndexForBytes(string2bytes(s)); + } + + /* (non-Javadoc) + * @see com.google.devtools.build.lib.util.StringIndexer#addString(java.lang.String) + */ + @Override + public boolean addString(String s) { + return addBytes(string2bytes(s)); + } + + protected synchronized byte[] getBytesForIndex(int i) { + Preconditions.checkArgument(i >= 0); + if (i >= nodes.size()) { + return null; + } + return reconstructKey(i); + } + + /* (non-Javadoc) + * @see com.google.devtools.build.lib.util.StringIndexer#getStringForIndex(int) + */ + @Override + public String getStringForIndex(int i) { + byte[] bytes = getBytesForIndex(i); + return bytes != null ? bytes2string(bytes) : null; + } + + private void dumpNodeContent(StringBuilder builder, byte[] content) { + int[] intHolder = new int[1]; + int offset = VarInt.getVarInt(content, 0, intHolder); + builder.append("parent: ").append(intHolder[0]); + offset = VarInt.getVarInt(content, offset, intHolder); + int len = intHolder[0]; + builder.append(", len: ").append(len).append(", key: \"") + .append(new String(content, offset, len, UTF_8)).append('"'); + offset += len; + while (offset < content.length) { + builder.append(", '").append(new String(content, offset, 1, UTF_8)).append("': "); + offset = VarInt.getVarInt(content, offset + 1, intHolder); + builder.append(intHolder[0]); + } + builder.append(", size: ").append(content.length); + } + + private int dumpContent(StringBuilder builder, int node, int indent, boolean[] seen) { + for(int i = 0; i < indent; i++) { + builder.append(" "); + } + builder.append(node).append(": "); + if (node >= nodes.size()) { + builder.append("OUT_OF_BOUNDS\n"); + return 0; + } else if (seen[node]) { + builder.append("ALREADY_SEEN\n"); + return 0; + } + seen[node] = true; + byte[] content = nodes.get(node); + if (content == null) { + builder.append("NULL\n"); + return 0; + } + dumpNodeContent(builder, content); + builder.append("\n"); + int contentSize = content.length; + + int[] intHolder = new int[1]; + int contentOffset = VarInt.getVarInt(content, 0, intHolder); // parent id + contentOffset = VarInt.getVarInt(content, contentOffset, intHolder); // key length + contentOffset += intHolder[0]; + while (contentOffset < content.length) { + contentOffset = VarInt.getVarInt(content, contentOffset + 1, intHolder); + contentSize += dumpContent(builder, intHolder[0], indent + 1, seen); + } + return contentSize; + } + + @Override + public synchronized String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("size = ").append(nodes.size()).append("\n"); + if (nodes.size() > 0) { + int contentSize = dumpContent(builder, rootId, 0, new boolean[nodes.size()]); + builder.append("contentSize = ").append(contentSize).append("\n"); + } + return builder.toString(); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/DependencySet.java b/src/main/java/com/google/devtools/build/lib/util/DependencySet.java new file mode 100644 index 0000000..788037d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/DependencySet.java
@@ -0,0 +1,225 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Representation of a set of file dependencies for a given output file. There + * are generally one input dependency and a bunch of include dependencies. The + * files are stored as {@code PathFragment}s and may be relative or absolute. + * <p> + * The serialized format read and written is equivalent and compatible with the + * ".d" file produced by the -MM for a given out (.o) file. + * <p> + * The file format looks like: + * + * <pre> + * {outfile}: \ + * {infile} \ + * {include} \ + * ... \ + * {include} + * </pre> + * + * @see "http://gcc.gnu.org/onlinedocs/gcc-4.2.1/gcc/Preprocessor-Options.html#Preprocessor-Options" + */ +public final class DependencySet { + + private static final Pattern DOTD_MERGED_LINE_SEPARATOR = Pattern.compile("\\\\[\n\r]+"); + private static final Pattern DOTD_LINE_SEPARATOR = Pattern.compile("[\n\r]+"); + private static final Pattern DOTD_DEP = Pattern.compile("(?:[^\\s\\\\]++|\\\\ |\\\\)+"); + + /** + * The set of dependent files that this DependencySet embodies. May be + * relative or absolute PathFragments. A tree set is used to ensure that we + * write them out in a consistent order. + */ + private final Collection<PathFragment> dependencies = new ArrayList<>(); + + private final Path root; + private String outputFileName; + + /** + * Get output file name for which dependencies are included in this DependencySet. + */ + public String getOutputFileName() { + return outputFileName; + } + + public void setOutputFileName(String outputFileName) { + this.outputFileName = outputFileName; + } + + /** + * Constructs a new empty DependencySet instance. + */ + public DependencySet(Path root) { + this.root = root; + } + + /** + * Gets an unmodifiable view of the set of dependencies in PathFragment form + * from this DependencySet instance. + */ + public Collection<PathFragment> getDependencies() { + return Collections.unmodifiableCollection(dependencies); + } + + /** + * Adds a given collection of dependencies in Path form to this DependencySet + * instance. Paths are converted to root-relative + */ + public void addDependencies(Collection<Path> deps) { + for (Path d : deps) { + addDependency(d.relativeTo(root)); + } + } + + /** + * Adds a given dependency in PathFragment form to this DependencySet + * instance. + */ + public void addDependency(PathFragment dep) { + dependencies.add(Preconditions.checkNotNull(dep)); + } + + /** + * Reads a dotd file into this DependencySet instance. + */ + public DependencySet read(Path dotdFile) throws IOException { + return process(FileSystemUtils.readContent(dotdFile)); + } + + /** + * Parses a .d file. + * + * <p>Performance-critical! In large C++ builds there are lots of .d files to read, and some of + * them reach into hundreds of kilobytes. + */ + public DependencySet process(byte[] content) { + // true if there is a CR in the input. + boolean cr = content.length > 0 && content[0] == '\r'; + // true if there is more than one line in the input, not counting \-wrapped lines. + boolean multiline = false; + + byte prevByte = ' '; + for (int i = 1; i < content.length; i++) { + byte b = content[i]; + if (cr || b == '\r') { + // CR found, abort since our little loop here does not deal with CR/LFs. + cr = true; + break; + } + if (b == '\n') { + // Merge lines wrapped using backslashes. + if (prevByte == '\\') { + content[i] = ' '; + content[i - 1] = ' '; + } else { + multiline = true; + } + } + prevByte = b; + } + + if (!cr && content.length > 0 && content[content.length - 1] == '\n') { + content[content.length - 1] = ' '; + } + + String s = new String(content, StandardCharsets.UTF_8); + if (cr) { + s = DOTD_MERGED_LINE_SEPARATOR.matcher(s).replaceAll(" ").trim(); + multiline = true; + } + return process(s, multiline); + } + + private DependencySet process(String contents, boolean multiline) { + String[] lines; + if (!multiline) { + // Microoptimization: skip the usually unnecessary expensive-ish splitting step if there is + // only one target. This saves about 20% of CPU time. + lines = new String[] { contents }; + } else { + lines = DOTD_LINE_SEPARATOR.split(contents); + } + + for (String line : lines) { + // Split off output file name. + int pos = line.indexOf(':'); + if (pos == -1) { + continue; + } + outputFileName = line.substring(0, pos); + + String deps = line.substring(pos + 1); + + Matcher m = DOTD_DEP.matcher(deps); + while (m.find()) { + String token = m.group(); + // Process escaped spaces. + if (token.contains("\\ ")) { + token = token.replace("\\ ", " "); + } + dependencies.add(new PathFragment(token).normalize()); + } + } + return this; + } + + /** + * Writes this DependencySet object for a specified output file under the root + * dir, and with a given suffix. + */ + public void write(Path outFile, String suffix) throws IOException { + Path dotdFile = + outFile.getRelative(FileSystemUtils.replaceExtension(outFile.asFragment(), suffix)); + + PrintStream out = new PrintStream(dotdFile.getOutputStream()); + try { + out.print(outFile.relativeTo(root) + ": "); + for (PathFragment d : dependencies) { + out.print(" \\\n " + d.getPathString()); // should already be root relative + } + out.println(); + } finally { + out.close(); + } + } + + @Override + public boolean equals(Object other) { + return other instanceof DependencySet + && ((DependencySet) other).dependencies.equals(dependencies); + } + + @Override + public int hashCode() { + return dependencies.hashCode(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ExitCode.java b/src/main/java/com/google/devtools/build/lib/util/ExitCode.java new file mode 100644 index 0000000..8307538 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/ExitCode.java
@@ -0,0 +1,181 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.common.base.Objects; + +import java.util.Collection; +import java.util.HashMap; + +/** + * <p>Anything marked FAILURE is generally from a problem with the source code + * under consideration. In these cases, a re-run in an identical client should + * produce an identical return code all things being constant. + * + * <p>Anything marked as an ERROR is generally a problem unrelated to the + * source code itself. It is either something wrong with the user's command + * line or the user's machine or environment. + * + * <p>Note that these exit codes should be kept consistent with the codes + * returned by Blaze's launcher in //devtools/blaze/main:blaze.cc + */ +public class ExitCode { + // Tracks all exit codes defined here and elsewhere in Bazel. + private static final HashMap<Integer, ExitCode> exitCodeRegistry = new HashMap<>(); + + public static final ExitCode SUCCESS = ExitCode.create(0, "SUCCESS"); + public static final ExitCode BUILD_FAILURE = ExitCode.create(1, "BUILD_FAILURE"); + public static final ExitCode PARSING_FAILURE = ExitCode.createUnregistered(1, "PARSING_FAILURE"); + public static final ExitCode COMMAND_LINE_ERROR = ExitCode.create(2, "COMMAND_LINE_ERROR"); + public static final ExitCode TESTS_FAILED = ExitCode.create(3, "TESTS_FAILED"); + public static final ExitCode PARTIAL_ANALYSIS_FAILURE = + ExitCode.createUnregistered(3, "PARTIAL_ANALYSIS_FAILURE"); + public static final ExitCode NO_TESTS_FOUND = ExitCode.create(4, "NO_TESTS_FOUND"); + public static final ExitCode RUN_FAILURE = ExitCode.create(6, "RUN_FAILURE"); + public static final ExitCode ANALYSIS_FAILURE = ExitCode.create(7, "ANALYSIS_FAILURE"); + public static final ExitCode INTERRUPTED = ExitCode.create(8, "INTERRUPTED"); + public static final ExitCode OOM_ERROR = ExitCode.createInfrastructureFailure(33, "OOM_ERROR"); + public static final ExitCode LOCAL_ENVIRONMENTAL_ERROR = + ExitCode.createInfrastructureFailure(36, "LOCAL_ENVIRONMENTAL_ERROR"); + public static final ExitCode BLAZE_INTERNAL_ERROR = + ExitCode.createInfrastructureFailure(37, "BLAZE_INTERNAL_ERROR"); + public static final ExitCode RESERVED = ExitCode.createInfrastructureFailure(40, "RESERVED"); + /* + exit codes [50..60] and 253 are reserved for site specific wrappers to Bazel. + */ + + /** + * Creates and returns an ExitCode. Requires a unique exit code number. + * + * @param code the int value for this exit code + * @param name a human-readable description + */ + public static ExitCode create(int code, String name) { + return new ExitCode(code, name, /*infrastructureFailure=*/false, /*register=*/true); + } + + /** + * Creates and returns an ExitCode that represents an infrastructure failure. + * + * @param code the int value for this exit code + * @param name a human-readable description + */ + public static ExitCode createInfrastructureFailure(int code, String name) { + return new ExitCode(code, name, /*infrastructureFailure=*/true, /*register=*/true); + } + + /** + * Creates and returns an ExitCode that has the same numeric code as another ExitCode. This is to + * allow the duplicate error codes listed above to be registered, but is private to prevent other + * users from creating duplicate error codes in the future. + * + * @param code the int value for this exit code + * @param name a human-readable description + */ + private static ExitCode createUnregistered(int code, String name) { + return new ExitCode(code, name, /*infrastructureFailure=*/false, /*register=*/false); + } + + /** + * Add the given exit code to the registry. + * + * @param exitCode the exit code to register + * @throws IllegalStateException if the numeric exit code is already in the registry. + */ + private static void register(ExitCode exitCode) { + synchronized (exitCodeRegistry) { + int codeNum = exitCode.getNumericExitCode(); + if (exitCodeRegistry.containsKey(codeNum)) { + throw new IllegalStateException( + "Exit code " + codeNum + " (" + exitCode.name + ") already registered"); + } + exitCodeRegistry.put(codeNum, exitCode); + } + } + + /** + * Returns all registered ExitCodes. + */ + public static Collection<ExitCode> values() { + synchronized (exitCodeRegistry) { + return exitCodeRegistry.values(); + } + } + + private final int numericExitCode; + private final String name; + private final boolean infrastructureFailure; + + /** + * Whenever a new exit code is created, it is registered (to prevent exit codes with identical + * numeric codes from being created). However, there are some exit codes in this file that have + * duplicate numeric codes, so these are not registered. + */ + private ExitCode(int exitCode, String name, boolean infrastructureFailure, boolean register) { + this.numericExitCode = exitCode; + this.name = name; + this.infrastructureFailure = infrastructureFailure; + if (register) { + ExitCode.register(this); + } + } + + @Override + public int hashCode() { + return Objects.hashCode(numericExitCode, name, infrastructureFailure); + } + + @Override + public boolean equals(Object object) { + if (object instanceof ExitCode) { + ExitCode that = (ExitCode) object; + return this.numericExitCode == that.numericExitCode + && this.name.equals(that.name) + && this.infrastructureFailure == that.infrastructureFailure; + } + return false; + } + + /** + * Returns the human-readable name for this exit code. Not guaranteed to be stable, use the + * numeric exit code for that. + */ + @Override + public String toString() { + return name; + } + + /** + * Returns the error's int value. + */ + public int getNumericExitCode() { + return numericExitCode; + } + + /** + * Returns the human-readable name. + */ + public String name() { + return name; + } + + /** + * Returns true if the current exit code represents a failure of Blaze infrastructure, + * vs. a build failure. + */ + public boolean isInfrastructureFailure() { + return infrastructureFailure; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/FileType.java b/src/main/java/com/google/devtools/build/lib/util/FileType.java new file mode 100644 index 0000000..c91b17b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/FileType.java
@@ -0,0 +1,278 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.annotation.concurrent.Immutable; + +/** + * A base class for FileType matchers. + */ +@Immutable +public abstract class FileType implements Predicate<String> { + // A special file type + public static final FileType NO_EXTENSION = new FileType() { + @Override + public boolean apply(String filename) { + return filename.lastIndexOf('.') == -1; + } + }; + + public static FileType of(final String ext) { + return new FileType() { + @Override + public boolean apply(String filename) { + return filename.endsWith(ext); + } + @Override + public List<String> getExtensions() { + return ImmutableList.of(ext); + } + }; + } + + public static FileType of(final Iterable<String> extensions) { + return new FileType() { + @Override + public boolean apply(String filename) { + for (String ext : extensions) { + if (filename.endsWith(ext)) { + return true; + } + } + return false; + } + @Override + public List<String> getExtensions() { + return ImmutableList.copyOf(extensions); + } + }; + } + + public static FileType of(final String... extensions) { + return of(Arrays.asList(extensions)); + } + + @Override + public String toString() { + return getExtensions().toString(); + } + + /** + * Returns true if the the filename matches. The filename should be a basename (the filename + * component without a path) for performance reasons. + */ + @Override + public abstract boolean apply(String filename); + + /** + * Get a list of filename extensions this matcher handles. The first entry in the list (if + * available) is the primary extension that code can use to construct output file names. + * The list can be empty for some matchers. + * + * @return a list of filename extensions + */ + public List<String> getExtensions() { + return ImmutableList.of(); + } + + /** Return true if a file name is matched by the FileType */ + public boolean matches(String filename) { + int slashIndex = filename.lastIndexOf('/'); + if (slashIndex != -1) { + filename = filename.substring(slashIndex + 1); + } + return apply(filename); + } + + /** Return true if a file referred by path is matched by the FileType */ + public boolean matches(Path path) { + return apply(path.getBaseName()); + } + + /** Return true if a file referred by fragment is matched by the FileType */ + public boolean matches(PathFragment fragment) { + return apply(fragment.getBaseName()); + } + + // Check FileTypes + + /** + * An interface for entities that have a filename. + */ + public interface HasFilename { + /** + * Returns the filename of this entity. + */ + String getFilename(); + } + + /** + * Checks whether an Iterable<? extends HasFileType> contains any of the specified file types. + * + * <p>At least one FileType must be specified. + */ + public static <T extends HasFilename> boolean contains(final Iterable<T> items, + FileType... fileTypes) { + Preconditions.checkState(fileTypes.length > 0, "Must specify at least one file type"); + final FileTypeSet fileTypeSet = FileTypeSet.of(fileTypes); + for (T item : items) { + if (fileTypeSet.matches(item.getFilename())) { + return true; + } + } + return false; + } + + /** + * Checks whether a HasFileType is any of the specified file types. + * + * <p>At least one FileType must be specified. + */ + public static <T extends HasFilename> boolean contains(T item, FileType... fileTypes) { + return FileTypeSet.of(fileTypes).matches(item.getFilename()); + } + + + private static <T extends HasFilename> Predicate<T> typeMatchingPredicateFor( + final FileType matchingType) { + return new Predicate<T>() { + @Override + public boolean apply(T item) { + return matchingType.matches(item.getFilename()); + } + }; + } + + private static <T extends HasFilename> Predicate<T> typeMatchingPredicateFor( + final FileTypeSet matchingTypes) { + return new Predicate<T>() { + @Override + public boolean apply(T item) { + return matchingTypes.matches(item.getFilename()); + } + }; + } + + private static <T extends HasFilename> Predicate<T> typeMatchingPredicateFrom( + final Predicate<String> fileTypePredicate) { + return new Predicate<T>() { + @Override + public boolean apply(T item) { + return fileTypePredicate.apply(item.getFilename()); + } + }; + } + + /** + * A filter for Iterable<? extends HasFileType> that returns only those whose FileType matches the + * specified Predicate. + */ + public static <T extends HasFilename> Iterable<T> filter(final Iterable<T> items, + final Predicate<String> predicate) { + return Iterables.filter(items, typeMatchingPredicateFrom(predicate)); + } + + /** + * A filter for Iterable<? extends HasFileType> that returns only those of the specified file + * types. + */ + public static <T extends HasFilename> Iterable<T> filter(final Iterable<T> items, + FileType... fileTypes) { + return filter(items, FileTypeSet.of(fileTypes)); + } + + /** + * A filter for Iterable<? extends HasFileType> that returns only those of the specified file + * types. + */ + public static <T extends HasFilename> Iterable<T> filter(final Iterable<T> items, + FileTypeSet fileTypes) { + return Iterables.filter(items, typeMatchingPredicateFor(fileTypes)); + } + + /** + * A filter for Iterable<? extends HasFileType> that returns only those of the specified file + * type. + */ + public static <T extends HasFilename> Iterable<T> filter(final Iterable<T> items, + FileType fileType) { + return Iterables.filter(items, typeMatchingPredicateFor(fileType)); + } + + /** + * A filter for Iterable<? extends HasFileType> that returns everything except the specified file + * type. + */ + public static <T extends HasFilename> Iterable<T> except(final Iterable<T> items, + FileType fileType) { + return Iterables.filter(items, Predicates.not(typeMatchingPredicateFor(fileType))); + } + + + /** + * A filter for List<? extends HasFileType> that returns only those of the specified file types. + * The result is a mutable list, computed eagerly; see {@link #filter} for a lazy variant. + */ + public static <T extends HasFilename> List<T> filterList(final Iterable<T> items, + FileType... fileTypes) { + if (fileTypes.length > 0) { + return filterList(items, FileTypeSet.of(fileTypes)); + } else { + return new ArrayList<>(); + } + } + + /** + * A filter for List<? extends HasFileType> that returns only those of the specified file type. + * The result is a mutable list, computed eagerly. + */ + public static <T extends HasFilename> List<T> filterList(final Iterable<T> items, + final FileType fileType) { + List<T> result = new ArrayList<>(); + for (T item : items) { + if (fileType.matches(item.getFilename())) { + result.add(item); + } + } + return result; + } + + /** + * A filter for List<? extends HasFileType> that returns only those of the specified file types. + * The result is a mutable list, computed eagerly. + */ + public static <T extends HasFilename> List<T> filterList(final Iterable<T> items, + final FileTypeSet fileTypeSet) { + List<T> result = new ArrayList<>(); + for (T item : items) { + if (fileTypeSet.matches(item.getFilename())) { + result.add(item); + } + } + return result; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/FileTypeSet.java b/src/main/java/com/google/devtools/build/lib/util/FileTypeSet.java new file mode 100644 index 0000000..694e877 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/FileTypeSet.java
@@ -0,0 +1,139 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; + +import java.util.ArrayList; +import java.util.List; + +import javax.annotation.concurrent.Immutable; + +/** + * A set of FileTypes for grouped matching. + */ +@Immutable +public class FileTypeSet implements Predicate<String> { + private final ImmutableSet<FileType> types; + + /** A set that matches all files. */ + public static final FileTypeSet ANY_FILE = + new FileTypeSet() { + @Override + public String toString() { + return "any files"; + } + @Override + public boolean matches(String filename) { + return true; + } + @Override + public List<String> getExtensions() { + return ImmutableList.<String>of(); + } + }; + + /** A predicate that matches no files. */ + public static final FileTypeSet NO_FILE = + new FileTypeSet(ImmutableList.<FileType>of()) { + @Override + public String toString() { + return "no files"; + } + @Override + public boolean matches(String filename) { + return false; + } + }; + + private FileTypeSet() { + this.types = null; + } + + private FileTypeSet(FileType... fileTypes) { + this.types = ImmutableSet.copyOf(fileTypes); + } + + private FileTypeSet(Iterable<FileType> fileTypes) { + this.types = ImmutableSet.copyOf(fileTypes); + } + + /** + * Returns a set that matches only the provided {@code fileTypes}. + * + * <p>If {@code fileTypes} is empty, the returned predicate will match no files. + */ + public static FileTypeSet of(FileType... fileTypes) { + if (fileTypes.length == 0) { + return FileTypeSet.NO_FILE; + } else { + return new FileTypeSet(fileTypes); + } + } + + /** + * Returns a set that matches only the provided {@code fileTypes}. + * + * <p>If {@code fileTypes} is empty, the returned predicate will match no files. + */ + public static FileTypeSet of(Iterable<FileType> fileTypes) { + if (Iterables.isEmpty(fileTypes)) { + return FileTypeSet.NO_FILE; + } else { + return new FileTypeSet(fileTypes); + } + } + + /** Returns true if the filename can be matched by any FileType in this set. */ + public boolean matches(String filename) { + int slashIndex = filename.lastIndexOf('/'); + if (slashIndex != -1) { + filename = filename.substring(slashIndex + 1); + } + for (FileType type : types) { + if (type.apply(filename)) { + return true; + } + } + return false; + } + + /** Returns true if this predicate matches nothing. */ + public boolean isNone() { + return this == FileTypeSet.NO_FILE; + } + + @Override + public boolean apply(String filename) { + return matches(filename); + } + + /** Returns the list of possible file extensions for this file type. Can be empty. */ + public List<String> getExtensions() { + List<String> extensions = new ArrayList<>(); + for (FileType type : types) { + extensions.addAll(type.getExtensions()); + } + return extensions; + } + + @Override + public String toString() { + return StringUtil.joinEnglishList(getExtensions()); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java b/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java new file mode 100644 index 0000000..e4c0876 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java
@@ -0,0 +1,319 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Map; +import java.util.UUID; + +/** + * Simplified wrapper for MD5 message digests. See also + * com.google.math.crypto.MD5HMAC for a similar interface. + * + * @see java.security.MessageDigest + */ +public final class Fingerprint { + + private final MessageDigest md; + + /** + * Creates and initializes a new MD5 object; if this fails, Java must be + * installed incorrectly. + */ + public Fingerprint() { + try { + md = MessageDigest.getInstance("md5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("MD5 not available"); + } + } + + /** + * Completes the hash computation by doing final operations, e.g., padding. + * + * <p>This method has the side-effect of resetting the underlying digest computer. + * + * @return the MD5 digest as a 16-byte array + * @see java.security.MessageDigest#digest() + */ + public byte[] digestAndReset() { + return md.digest(); + } + + /** + * Completes the hash computation and returns the digest as a string. + * + * <p>This method has the side-effect of resetting the underlying digest computer. + * + * @return the MD5 digest as a 32-character string of hexadecimal digits + * @see com.google.math.crypto.MD5HMAC#toString() + */ + public String hexDigestAndReset() { + return hexDigest(digestAndReset()); + } + + /** + * Returns a string representation of an MD5 digest. + * + * @param digest the MD5 digest, perhaps from a previous call to digest + * @return the digest as a 32-character string of hexadecimal digits + */ + public static String hexDigest(byte[] digest) { + StringBuilder b = new StringBuilder(32); + for (int i = 0; i < digest.length; i++) { + int n = digest[i]; + b.append("0123456789abcdef".charAt((n >> 4) & 0xF)); + b.append("0123456789abcdef".charAt(n & 0xF)); + } + return b.toString(); + } + + /** + * Override of Object.toString to return a string for the MD5 digest without + * finalizing the digest computation. Calling hexDigest() instead will + * finalize the digest computation. + * + * @return the string returned by hexDigest() + */ + @Override + public String toString() { + try { + // MD5 does support cloning, so this should not fail + return hexDigest(((MessageDigest) md.clone()).digest()); + } catch (CloneNotSupportedException e) { + // MessageDigest does not support cloning, + // so just return the toString() on the MessageDigest. + return md.toString(); + } + } + + /** + * Updates the digest with 0 or more bytes. + * + * @param input the array of bytes with which to update the digest + * @see java.security.MessageDigest#update(byte[]) + */ + public Fingerprint addBytes(byte[] input) { + md.update(input); + return this; + } + + /** + * Updates the digest with the specified number of bytes starting at offset. + * + * @param input the array of bytes with which to update the digest + * @param offset the offset into the array + * @param len the number of bytes to use + * @see java.security.MessageDigest#update(byte[], int, int) + */ + public Fingerprint addBytes(byte[] input, int offset, int len) { + md.update(input, offset, len); + return this; + } + + /** + * Updates the digest with a boolean value. + */ + public Fingerprint addBoolean(boolean input) { + addBytes(new byte[] { (byte) (input ? 1 : 0) }); + return this; + } + + /** + * Updates the digest with the little-endian bytes of a given int value. + * + * @param input the integer with which to update the digest + */ + public Fingerprint addInt(int input) { + md.update(new byte[] { + (byte) input, + (byte) (input >> 8), + (byte) (input >> 16), + (byte) (input >> 24), + }); + + return this; + } + + /** + * Updates the digest with the little-endian bytes of a given long value. + * + * @param input the long with which to update the digest + */ + public Fingerprint addLong(long input) { + md.update(new byte[]{ + (byte) input, + (byte) (input >> 8), + (byte) (input >> 16), + (byte) (input >> 24), + (byte) (input >> 32), + (byte) (input >> 40), + (byte) (input >> 48), + (byte) (input >> 56), + }); + + return this; + } + + /** + * Updates the digest with a UUID. + * + * @param uuid the UUID with which to update the digest. Must not be null. + */ + public Fingerprint addUUID(UUID uuid) { + addLong(uuid.getLeastSignificantBits()); + addLong(uuid.getMostSignificantBits()); + return this; + } + + /** + * Updates the digest with a String using its length plus its UTF8 encoded bytes. + * + * @param input the String with which to update the digest + * @see java.security.MessageDigest#update(byte[]) + */ + public Fingerprint addString(String input) { + byte[] bytes = input.getBytes(UTF_8); + addInt(bytes.length); + md.update(bytes); + return this; + } + + /** + * Updates the digest with a String using its length and content. + * + * @param input the String with which to update the digest + * @see java.security.MessageDigest#update(byte[]) + */ + public Fingerprint addStringLatin1(String input) { + addInt(input.length()); + byte[] bytes = new byte[input.length()]; + for (int i = 0; i < input.length(); i++) { + bytes[i] = (byte) input.charAt(i); + } + md.update(bytes); + return this; + } + + /** + * Updates the digest with a Path. + * + * @param input the Path with which to update the digest. + */ + public Fingerprint addPath(Path input) { + addStringLatin1(input.getPathString()); + return this; + } + + /** + * Updates the digest with a Path. + * + * @param input the Path with which to update the digest. + */ + public Fingerprint addPath(PathFragment input) { + addStringLatin1(input.getPathString()); + return this; + } + + /** + * Updates the digest with inputs by iterating over them and invoking + * {@code #addString(String)} on each element. + * + * @param inputs the inputs with which to update the digest + */ + public Fingerprint addStrings(Iterable<String> inputs) { + addInt(Iterables.size(inputs)); + for (String input : inputs) { + addString(input); + } + + return this; + } + + /** + * Updates the digest with inputs by iterating over them and invoking + * {@code #addString(String)} on each element. + * + * @param inputs the inputs with which to update the digest + */ + public Fingerprint addStrings(String... inputs) { + addInt(inputs.length); + for (String input : inputs) { + addString(input); + } + + return this; + } + + /** + * Updates the digest with inputs which are pairs in a map, by iterating over + * the map entries and invoking {@code #addString(String)} on each key and + * value. + * + * @param inputs the inputs in a map with which to update the digest + */ + public Fingerprint addStringMap(Map<String, String> inputs) { + addInt(inputs.size()); + for (Map.Entry<String, String> entry : inputs.entrySet()) { + addString(entry.getKey()); + addString(entry.getValue()); + } + + return this; + } + + /** + * Updates the digest with a list of paths by iterating over them and + * invoking {@link #addPath(PathFragment)} on each element. + * + * @param inputs the paths with which to update the digest + */ + public Fingerprint addPaths(Iterable<PathFragment> inputs) { + addInt(Iterables.size(inputs)); + for (PathFragment path : inputs) { + addPath(path); + } + + return this; + } + + /** + * Reset the Fingerprint for additional use as though previous digesting had not been done. + */ + public void reset() { + md.reset(); + } + + // -------- Convenience methods ---------------------------- + + /** + * Computes the hex digest from a String using UTF8 encoding and returning + * the hexDigest(). + * + * @param input the String from which to compute the digest + */ + public static String md5Digest(String input) { + Fingerprint f = new Fingerprint(); + f.addBytes(input.getBytes(UTF_8)); + return f.hexDigestAndReset(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/GroupedList.java b/src/main/java/com/google/devtools/build/lib/util/GroupedList.java new file mode 100644 index 0000000..2bf956d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/GroupedList.java
@@ -0,0 +1,344 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.collect.CompactHashSet; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + * Encapsulates a list of lists. Is intended to be used in "batch" mode -- to set the value of a + * GroupedList, users should first construct a {@link GroupedListHelper}, add elements to it, and + * then {@link #append} the helper to a new GroupedList instance. The generic type T <i>must not</i> + * be a {@link List}. + * + * <p>Despite the "list" name, it is an error for the same element to appear multiple times in the + * list. Users are responsible for not trying to add the same element to a GroupedList twice. + */ +public class GroupedList<T> implements Iterable<Iterable<T>> { + // Total number of items in the list. At least elements.size(), but might be larger if there are + // any nested lists. + private int size = 0; + // Items in this GroupedList. Each element is either of type T or List<T>. + // Non-final only for #remove. + private List<Object> elements; + + public GroupedList() { + // We optimize for small lists. + this.elements = new ArrayList<>(1); + } + + // Only for use when uncompressing a GroupedList. + private GroupedList(int size, List<Object> elements) { + this.size = size; + this.elements = new ArrayList<>(elements); + } + + /** Appends the list constructed in helper to this list. */ + public void append(GroupedListHelper<T> helper) { + Preconditions.checkState(helper.currentGroup == null, "%s %s", this, helper); + // Do a check to make sure we don't have lists here. Note that if helper.elements is empty, + // Iterables.getFirst will return null, and null is not instanceof List. + Preconditions.checkState(!(Iterables.getFirst(helper.elements, null) instanceof List), + "Cannot make grouped list of lists: %s", helper); + elements.addAll(helper.groupedList); + size += helper.size(); + } + + /** + * Removes the elements in toRemove from this list. Takes time proportional to the size of the + * list, so should not be called often. + */ + public void remove(Set<T> toRemove) { + elements = remove(elements, toRemove); + size -= toRemove.size(); + } + + /** Returns the number of elements in this list. */ + public int size() { + return size; + } + + /** Returns true if this list contains no elements. */ + public boolean isEmpty() { + return elements.isEmpty(); + } + + private static final Object EMPTY_LIST = new Object(); + + public Object compress() { + switch (size()) { + case 0: + return EMPTY_LIST; + case 1: + return Iterables.getOnlyElement(elements); + default: + return elements.toArray(); + } + } + + @SuppressWarnings("unchecked") + public Set<T> toSet() { + ImmutableSet.Builder<T> builder = ImmutableSet.builder(); + for (Object obj : elements) { + if (obj instanceof List) { + builder.addAll((List<T>) obj); + } else { + builder.add((T) obj); + } + } + return builder.build(); + } + + private static int sizeOf(Object obj) { + return obj instanceof List ? ((List<?>) obj).size() : 1; + } + + public static <E> GroupedList<E> create(Object compressed) { + if (compressed == EMPTY_LIST) { + return new GroupedList<>(); + } + if (compressed.getClass().isArray()) { + List<Object> elements = new ArrayList<>(); + int size = 0; + for (Object item : (Object[]) compressed) { + size += sizeOf(item); + elements.add(item); + } + return new GroupedList<>(size, elements); + } + // Just a single element. + return new GroupedList<>(1, ImmutableList.<Object>of(compressed)); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (this.getClass() != other.getClass()) { + return false; + } + GroupedList<?> that = (GroupedList<?>) other; + return elements.equals(that.elements); + } + + @Override + @SuppressWarnings("deprecation") + public String toString() { + return Objects.toStringHelper(this) + .add("elements", elements) + .add("size", size).toString(); + + } + + /** + * Iterator that returns the next group in elements for each call to {@link #next}. A custom + * iterator is needed here because, to optimize memory, we store single-element lists as elements + * internally, and so they must be wrapped before they're returned. + */ + private class GroupedIterator implements Iterator<Iterable<T>> { + private final Iterator<Object> iter = elements.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @SuppressWarnings("unchecked") // Cast of Object to List<T> or T. + @Override + public Iterable<T> next() { + Object obj = iter.next(); + if (obj instanceof List) { + return (List<T>) obj; + } + return ImmutableList.of((T) obj); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } + + @Override + public Iterator<Iterable<T>> iterator() { + return new GroupedIterator(); + } + + /** + * Removes everything in toRemove from the list of lists, elements. Called both by GroupedList and + * GroupedListHelper. + */ + private static <E> List<Object> remove(List<Object> elements, Set<E> toRemove) { + int removedCount = 0; + // elements.size is an upper bound of the needed size. Since normally removal happens just + // before the list is finished and compressed, optimizing this size isn't a concern. + List<Object> newElements = new ArrayList<>(elements.size()); + for (Object obj : elements) { + if (obj instanceof List) { + ImmutableList.Builder<E> newGroup = new ImmutableList.Builder<>(); + @SuppressWarnings("unchecked") + List<E> oldGroup = (List<E>) obj; + for (E elt : oldGroup) { + if (toRemove.contains(elt)) { + removedCount++; + } else { + newGroup.add(elt); + } + } + ImmutableList<E> group = newGroup.build(); + addItem(group, newElements); + } else { + if (toRemove.contains(obj)) { + removedCount++; + } else { + newElements.add(obj); + } + } + } + Preconditions.checkState(removedCount == toRemove.size(), + "%s %s %s %s", removedCount, removedCount, elements, newElements); + return newElements; + } + + private static void addItem(Collection<?> item, List<Object> elements) { + switch (item.size()) { + case 0: + return; + case 1: + elements.add(Iterables.getOnlyElement(item)); + return; + default: + elements.add(ImmutableList.copyOf(item)); + } + } + + /** + * Builder-like object for GroupedLists. An already-existing grouped list is appended to by + * constructing a helper, mutating it, and then appending that helper to the grouped list. + */ + public static class GroupedListHelper<E> implements Iterable<E> { + // Non-final only for removal. + private List<Object> groupedList; + private List<E> currentGroup = null; + private final Set<E> elements = CompactHashSet.create(); + + private GroupedListHelper(GroupedList<E> groupedList) { + this.groupedList = new ArrayList<>(groupedList.elements); + for (Iterable<E> group : groupedList) { + Iterables.addAll(elements, group); + } + } + + public GroupedListHelper() { + // Optimize for short lists. + groupedList = new ArrayList<>(1); + } + + /** + * Add an element to this list. If in a group, will be added to the current group. Otherwise, + * goes in a group of its own. + */ + public void add(E elt) { + Preconditions.checkState(elements.add(elt), "%s %s", elt, this); + if (currentGroup == null) { + groupedList.add(elt); + } else { + currentGroup.add(elt); + } + } + + /** + * Remove all elements of toRemove from this list. It is a fatal error if any elements of + * toRemove are not present. Takes time proportional to the size of the list, so should not be + * called often. + */ + public void remove(Set<E> toRemove) { + groupedList = GroupedList.remove(groupedList, toRemove); + int oldSize = size(); + elements.removeAll(toRemove); + Preconditions.checkState(oldSize == size() + toRemove.size(), + "%s %s %s", oldSize, toRemove, this); + } + + /** + * Starts a group. All elements added until {@link #endGroup} will be in the same group. Each + * call of {@link #startGroup} must be paired with a following {@link #endGroup} call. + */ + public void startGroup() { + Preconditions.checkState(currentGroup == null, this); + currentGroup = new ArrayList<>(); + } + + private void addList(Collection<E> group) { + addItem(group, groupedList); + } + + /** Ends a group started with {@link #startGroup}. */ + public void endGroup() { + Preconditions.checkNotNull(currentGroup); + addList(currentGroup); + currentGroup = null; + } + + /** Returns true if elt is present in the list. */ + public boolean contains(E elt) { + return elements.contains(elt); + } + + private int size() { + return elements.size(); + } + + /** Returns true if list is empty. */ + public boolean isEmpty() { + return elements.isEmpty(); + } + + @Override + public Iterator<E> iterator() { + return elements.iterator(); + } + + /** Create a GroupedListHelper from a collection of elements, all put in the same group.*/ + public static <F> GroupedListHelper<F> create(Collection<F> elements) { + GroupedListHelper<F> helper = new GroupedListHelper<>(); + helper.addList(elements); + helper.elements.addAll(elements); + Preconditions.checkState(helper.elements.size() == elements.size(), + "%s %s", helper, elements); + return helper; + } + + @Override + @SuppressWarnings("deprecation") + public String toString() { + return Objects.toStringHelper(this) + .add("groupedList", groupedList) + .add("elements", elements) + .add("currentGroup", currentGroup).toString(); + + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/IncludeScanningUtil.java b/src/main/java/com/google/devtools/build/lib/util/IncludeScanningUtil.java new file mode 100644 index 0000000..24f55e4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/IncludeScanningUtil.java
@@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Static utilities for include scanning. + */ +public class IncludeScanningUtil { + private IncludeScanningUtil() { + } + + private static final String INCLUDES_SUFFIX = ".includes"; + public static final PathFragment GREPPED_INCLUDES = + new PathFragment(Constants.PRODUCT_NAME + "-out/_grepped_includes"); + + /** + * Returns the exec-root relative output path for grepped includes. + * + * @param srcExecPath the exec-root relative path of the source file. + */ + public static PathFragment getExecRootRelativeOutputPath(PathFragment srcExecPath) { + return GREPPED_INCLUDES.getRelative(getRootRelativeOutputPath(srcExecPath)); + } + + /** + * Returns the root relative output path for grepped includes. + * + * @param srcExecPath the exec-root relative path of the source file. + */ + public static PathFragment getRootRelativeOutputPath(PathFragment srcExecPath) { + return srcExecPath.replaceName(srcExecPath.getBaseName() + INCLUDES_SUFFIX); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/JavaClock.java b/src/main/java/com/google/devtools/build/lib/util/JavaClock.java new file mode 100644 index 0000000..bdd1116 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/JavaClock.java
@@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +/** + * Class provides a simple clock implementation used by the tool. By default it uses {@link System} + * class. + */ +public class JavaClock implements Clock { + + public JavaClock() { + } + + @Override + public long currentTimeMillis() { + return System.currentTimeMillis(); + } + + @Override + public long nanoTime() { + // Note that some JVM implementations of System#nanoTime don't yield a non-decreasing + // sequence of values. + return System.nanoTime(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/LazyString.java b/src/main/java/com/google/devtools/build/lib/util/LazyString.java new file mode 100644 index 0000000..0e037b2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/LazyString.java
@@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +/** + * This class serves as a base implementation for a {@code CharSequence} + * that delay string construction (mostly till the execution phase). + * + * They are not full implementations, they lack {@code #charAt(int)} and + * {@code #subSequence(int, int)}. + */ +public abstract class LazyString implements CharSequence { + + @Override + public char charAt(int index) { + throw new UnsupportedOperationException(); + } + + @Override + public int length() { + return toString().length(); + } + + @Override + public CharSequence subSequence(int start, int end) { + throw new UnsupportedOperationException(); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/LoggingUtil.java b/src/main/java/com/google/devtools/build/lib/util/LoggingUtil.java new file mode 100644 index 0000000..5170727 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/LoggingUtil.java
@@ -0,0 +1,87 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.Uninterruptibles; +import com.google.devtools.build.lib.concurrent.ThreadSafety; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +/** + * Logging utilities for sending log messages to a remote service. Log messages + * will not be output anywhere else, including the terminal and blaze clients. + */ +@ThreadSafety.ThreadSafe +public final class LoggingUtil { + // TODO(bazel-team): this class is a thin wrapper around Logger and could probably be discarded. + private static Future<Logger> remoteLogger; + + /** + * Installs the remote logger. + * + * <p>This can only be called once, and the caller should not keep the + * reference to the logger. + * + * @param logger The logger future. Must have already started. + */ + public static synchronized void installRemoteLogger(Future<Logger> logger) { + Preconditions.checkState(remoteLogger == null); + remoteLogger = logger; + } + + /** Returns the installed logger, or null if none is installed. */ + public static synchronized Logger getRemoteLogger() { + try { + return (remoteLogger == null) ? null : Uninterruptibles.getUninterruptibly(remoteLogger); + } catch (ExecutionException e) { + throw new RuntimeException("Unexpected error initializing remote logging", e); + } + } + + /** + * @see #logToRemote(Level, String, Throwable, String...). + */ + public static void logToRemote(Level level, String msg, Throwable trace) { + Logger logger = getRemoteLogger(); + if (logger != null) { + logger.log(level, msg, trace); + } + } + + /** + * Log a message to the remote backend. This is done out of thread, so this + * method is non-blocking. + * + * @param level The severity level. Non null. + * @param msg The log message. Non null. + * @param trace The stack trace. May be null. + * @param values Additional values to upload. + */ + public static void logToRemote(Level level, String msg, Throwable trace, + String... values) { + Logger logger = getRemoteLogger(); + if (logger != null) { + LogRecord logRecord = new LogRecord(level, msg); + logRecord.setThrown(trace); + logRecord.setParameters(values); + logger.log(logRecord); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/NetUtil.java b/src/main/java/com/google/devtools/build/lib/util/NetUtil.java new file mode 100644 index 0000000..498da77 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/NetUtil.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * Various utility methods for network related stuff. + */ +public final class NetUtil { + + private NetUtil() { + } + + /** + * Returns the short hostname or <code>unknown</code> if the host name could + * not be determined. + */ + public static String findShortHostName() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + return "unknown"; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/OS.java b/src/main/java/com/google/devtools/build/lib/util/OS.java new file mode 100644 index 0000000..b19bd7e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/OS.java
@@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +/** + * An operating system. + */ +public enum OS { + DARWIN, + LINUX, + WINDOWS, + UNKNOWN; + + /** + * The current operating system. + */ + public static OS getCurrent() { + return HOST_SYSTEM; + } + // We inject a the OS name through blaze.os, so we can have + // some coverage for Windows specific code on Linux. + private static String getOsName() { + String override = System.getProperty("blaze.os"); + return override == null ? System.getProperty("os.name") : override; + } + + private static final OS HOST_SYSTEM = + "Mac OS X".equals(getOsName()) ? OS.DARWIN : ( + "Linux".equals(getOsName()) ? OS.LINUX : ( + getOsName().contains("Windows") ? OS.WINDOWS : OS.UNKNOWN)); +} +
diff --git a/src/main/java/com/google/devtools/build/lib/util/OptionsUtils.java b/src/main/java/com/google/devtools/build/lib/util/OptionsUtils.java new file mode 100644 index 0000000..11bf94f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/OptionsUtils.java
@@ -0,0 +1,154 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.OptionsParser.UnparsedOptionValueDescription; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Blaze-specific option utilities. + */ +public final class OptionsUtils { + + /** + * Returns a string representation of the non-hidden specified options; option values are + * shell-escaped. + */ + public static String asShellEscapedString(Iterable<UnparsedOptionValueDescription> optionsList) { + StringBuffer result = new StringBuffer(); + for (UnparsedOptionValueDescription option : optionsList) { + if (option.isHidden()) { + continue; + } + if (result.length() != 0) { + result.append(' '); + } + String value = option.getUnparsedValue(); + if (option.isBooleanOption()) { + boolean isEnabled = false; + try { + isEnabled = new Converters.BooleanConverter().convert(value); + } catch (OptionsParsingException e) { + throw new RuntimeException("Unexpected parsing exception", e); + } + result.append(isEnabled ? "--" : "--no").append(option.getName()); + } else { + result.append("--").append(option.getName()); + if (value != null) { // Can be null for Void options. + result.append("=").append(ShellEscaper.escapeString(value)); + } + } + } + return result.toString(); + } + + /** + * Returns a string representation of the non-hidden explicitly or implicitly + * specified options; option values are shell-escaped. + */ + public static String asShellEscapedString(OptionsProvider options) { + return asShellEscapedString(options.asListOfUnparsedOptions()); + } + + /** + * Returns a string representation of the non-hidden explicitly or implicitly + * specified options, filtering out any sensitive options; option values are + * shell-escaped. + */ + public static String asFilteredShellEscapedString(OptionsProvider options, + Iterable<UnparsedOptionValueDescription> optionsList) { + return asShellEscapedString(optionsList); + } + + /** + * Returns a string representation of the non-hidden explicitly or implicitly + * specified options, filtering out any sensitive options; option values are + * shell-escaped. + */ + public static String asFilteredShellEscapedString(OptionsProvider options) { + return asFilteredShellEscapedString(options, options.asListOfUnparsedOptions()); + } + + /** + * Converter from String to PathFragment. + */ + public static class PathFragmentConverter + implements Converter<PathFragment> { + + @Override + public PathFragment convert(String input) { + return new PathFragment(input); + } + + @Override + public String getTypeDescription() { + return "a path"; + } + } + + /** + * Converter from String to PathFragment. + * + * <p>Complains if the path is not absolute. + */ + public static class AbsolutePathFragmentConverter + implements Converter<PathFragment> { + + @Override + public PathFragment convert(String input) throws OptionsParsingException { + PathFragment pathFragment = new PathFragment(input); + if (!pathFragment.isAbsolute()) { + throw new OptionsParsingException("Expected absolute path, found " + input); + } + return pathFragment; + } + + @Override + public String getTypeDescription() { + return "an absolute path"; + } + } + + /** + * Converts from a colon-separated list of strings into a list of PathFragment instances. + */ + public static class PathFragmentListConverter + implements Converter<List<PathFragment>> { + + @Override + public List<PathFragment> convert(String input) { + List<PathFragment> list = new ArrayList<>(); + for (String piece : input.split(":")) { + if (!piece.equals("")) { + list.add(new PathFragment(piece)); + } + } + return Collections.unmodifiableList(list); + } + + @Override + public String getTypeDescription() { + return "a colon-separated list of paths"; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/OsUtils.java b/src/main/java/com/google/devtools/build/lib/util/OsUtils.java new file mode 100644 index 0000000..fa12f59 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/OsUtils.java
@@ -0,0 +1,74 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Operating system-specific utilities. + */ +public final class OsUtils { + + private static final String EXECUTABLE_EXTENSION = OS.getCurrent() == OS.WINDOWS ? ".exe" : ""; + + // Utility class. + private OsUtils() { + } + + /** + * Returns the extension used for executables on the current platform (.exe + * for Windows, empty string for others). + */ + public static String executableExtension() { + return EXECUTABLE_EXTENSION; + } + + /** + * Loads JNI libraries, if necessary under the current platform. + */ + public static void maybeForceJNI(PathFragment installBase) { + if (jniLibsAvailable()) { + forceJNI(installBase); + } + } + + private static boolean jniLibsAvailable() { + // JNI libraries work fine on Windows, but at the moment we are not using any. + return OS.getCurrent() != OS.WINDOWS; + } + + // Force JNI linking at a moment when we have 'installBase' handy, and print + // an informative error if it fails. + private static void forceJNI(PathFragment installBase) { + try { + ProcessUtils.getpid(); // force JNI initialization + } catch (UnsatisfiedLinkError t) { + System.err.println("JNI initialization failed: " + t.getMessage() + ". " + + "Possibly your installation has been corrupted; " + + "if this problem persists, try 'rm -fr " + installBase + "'."); + throw t; + } + } + + /** + * Returns the PID of the current process, or -1 if not available. + */ + public static int getpid() { + if (jniLibsAvailable()) { + return ProcessUtils.getpid(); + } + return -1; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/Pair.java b/src/main/java/com/google/devtools/build/lib/util/Pair.java new file mode 100644 index 0000000..a377c3c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/Pair.java
@@ -0,0 +1,122 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import com.google.common.base.Function; + +import java.util.Comparator; +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * An immutable, semantic-free ordered pair of nullable values. Avoid using it in public APIs. + */ +public final class Pair<A, B> { + + /** + * Creates a new pair containing the given elements in order. + */ + public static <A, B> Pair<A, B> of(@Nullable A first, @Nullable B second) { + return new Pair<A, B>(first, second); + } + + /** + * The first element of the pair. + */ + @Nullable + public final A first; + + /** + * The second element of the pair. + */ + @Nullable + public final B second; + + /** + * Constructor. It is usually easier to call {@link #of}. + */ + public Pair(@Nullable A first, @Nullable B second) { + this.first = first; + this.second = second; + } + + @Nullable + public A getFirst() { + return first; + } + + @Nullable + public B getSecond() { + return second; + } + + @Override + public String toString() { + return "(" + first + ", " + second + ")"; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Pair)) { + return false; + } + Pair<?, ?> p = (Pair<?, ?>) o; + return Objects.equals(first, p.first) && Objects.equals(second, p.second); + } + + @Override + public int hashCode() { + return Objects.hash(first, second); + } + + /** + * A function that maps to the first element in a pair. + */ + public static <A, B> Function<Pair<A, B>, A> firstFunction() { + return new Function<Pair<A, B>, A>() { + @Override + public A apply(Pair<A, B> pair) { + return pair.first; + } + }; + } + + /** + * A function that maps to the second element in a pair. + */ + public static <A, B> Function<Pair<A, B>, B> secondFunction() { + return new Function<Pair<A, B>, B>() { + @Override + public B apply(Pair<A, B> pair) { + return pair.second; + } + }; + } + + /** + * A comparator that compares pairs by comparing the first element. + */ + public static <T extends Comparable<T>, B> Comparator<Pair<T, B>> compareByFirst() { + return new Comparator<Pair<T, B>>() { + @Override + public int compare(Pair<T, B> o1, Pair<T, B> o2) { + return o1.first.compareTo(o2.first); + } + }; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/PathFragmentFilter.java b/src/main/java/com/google/devtools/build/lib/util/PathFragmentFilter.java new file mode 100644 index 0000000..34f6cd2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/PathFragmentFilter.java
@@ -0,0 +1,111 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Converter; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * Handles options that specify list of included/excluded directories. + * Validates whether path is included in that filter. + * + * Excluded directories always take precedence over included ones (path depth + * and order are not important). + */ +public class PathFragmentFilter implements Serializable { + private final List<PathFragment> inclusions; + private final List<PathFragment> exclusions; + + /** + * Converts from a colon-separated list of of paths with optional '-' prefix into the + * PathFragmentFilter: + * [-]path1[,[-]path2]... + * + * Order of paths is not important. Empty entries are ignored. '-' marks an excluded path. + */ + public static class PathFragmentFilterConverter implements Converter<PathFragmentFilter> { + + @Override + public PathFragmentFilter convert(String input) { + List<PathFragment> inclusionList = new ArrayList<>(); + List<PathFragment> exclusionList = new ArrayList<>(); + + for (String piece : Splitter.on(',').split(input)) { + if (piece.length() > 1 && piece.startsWith("-")) { + exclusionList.add(new PathFragment(piece.substring(1))); + } else if (piece.length() > 0) { + inclusionList.add(new PathFragment(piece)); + } + } + + // TODO(bazel-team): (2010) Both lists could be optimized not to include unnecessary + // entries - e.g. entry 'a/b/c' is not needed if 'a/b' is also specified and 'a/b' on + // inclusion list is not needed if 'a' or 'a/b' is on exclusion list. + return new PathFragmentFilter(inclusionList, exclusionList); + } + + @Override + public String getTypeDescription() { + return "a comma-separated list of paths with prefix '-' specifying excluded paths"; + } + + } + + /** + * Creates new PathFragmentFilter using provided inclusion and exclusion path lists. + */ + public PathFragmentFilter(List<PathFragment> inclusions, List<PathFragment> exclusions) { + this.inclusions = ImmutableList.copyOf(inclusions); + this.exclusions = ImmutableList.copyOf(exclusions); + } + + /** + * @return true iff path is included (it is not on the exclusion list and + * it is either on the inclusion list or inclusion list is empty). + */ + public boolean isIncluded(PathFragment path) { + for (PathFragment excludedPath : exclusions) { + if (path.startsWith(excludedPath)) { + return false; + } + } + for (PathFragment includedPath : inclusions) { + if (path.startsWith(includedPath)) { + return true; + } + } + return inclusions.isEmpty(); // If inclusion filter is not specified, path is included. + } + + @Override + public String toString() { + List<String> list = Lists.newArrayListWithExpectedSize(inclusions.size() + exclusions.size()); + for (PathFragment path : inclusions) { + list.add(path.getPathString()); + } + for (PathFragment path : exclusions) { + list.add("-" + path.getPathString()); + } + return Joiner.on(',').join(list); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/PersistentMap.java b/src/main/java/com/google/devtools/build/lib/util/PersistentMap.java new file mode 100644 index 0000000..7fd4b6d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/PersistentMap.java
@@ -0,0 +1,486 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.common.collect.ForwardingMap; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Hashtable; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * A map that is backed by persistent storage. It uses two files on disk for + * this: The first file contains all the entries and gets written when invoking + * the {@link #save()} method. The second file contains a journal of all entries + * that were added to or removed from the map since constructing the instance of + * the map or the last invocation of {@link #save()} and gets written after each + * update of the map although sub-classes are free to implement their own + * journal update strategy. + * + * <p><b>Ceci n'est pas un Map</b>. Strictly speaking, the {@link Map} + * interface doesn't permit the possibility of failure. This class uses + * persistence; persistence means I/O, and I/O means the possibility of + * failure. Therefore the semantics of this may deviate from the Map contract + * in failure cases. In particular, updates are not guaranteed to succeed. + * However, I/O failures are guaranteed to be reported upon the subsequent call + * to a method that throws {@code IOException} such as {@link #save}. + * + * <p>To populate the map entries using the previously persisted entries call + * {@link #load()} prior to invoking any other map operation. + * <p> + * Like {@link Hashtable} but unlike {@link HashMap}, this class does + * <em>not</em> allow <tt>null</tt> to be used as a key or a value. + * <p> + * IO failures during reading or writing the map entries to disk may result in + * {@link AssertionError} getting thrown from the failing method. + * <p> + * The implementation of the map is not synchronized. If access from multiple + * threads is required it must be synchronized using an external object. + * <p> + * The constructor allows passing in a version number that gets written to the + * files on disk and checked before reading from disk. Files with an + * incompatible version number will be ignored. This allows the client code to + * change the persistence format without polluting the file system name space. + */ +public abstract class PersistentMap<K, V> extends ForwardingMap<K, V> { + + private static final int MAGIC = 0x20071105; + + private static final int ENTRY_MAGIC = 0xfe; + + private final int version; + private final Path mapFile; + private final Path journalFile; + private final Map<K, V> journal; + private DataOutputStream journalOut; + + /** + * 'dirty' is true when the in-memory representation of the map is more recent + * than the on-disk representation. + */ + private boolean dirty; + + /** + * If non-null, contains the message from an {@code IOException} thrown by a + * previously failed write. This error is deferred until the next call to a + * method which is able to throw an exception. + */ + private String deferredIOFailure = null; + + /** + * 'loaded' is true when the in-memory representation is at least as recent as + * the on-disk representation. + */ + private boolean loaded; + + private final Map<K, V> delegate; + + /** + * Creates a new PersistentMap instance using the specified backing map. + * + * @param version the version tag. Changing the version tag allows updating + * the on disk format. The map will never read from a file that was + * written using a different version tag. + * @param map the backing map to use for this PersistentMap. + * @param mapFile the file to save the map entries to. + * @param journalFile the journal file to write entries between invocations of + * {@link #save()}. + */ + public PersistentMap(int version, Map<K, V> map, Path mapFile, Path journalFile) { + this.version = version; + journal = new LinkedHashMap<>(); + this.mapFile = mapFile; + this.journalFile = journalFile; + delegate = map; + } + + @Override protected Map<K, V> delegate() { + return delegate; + } + + @Override + public V put(K key, V value) { + if (key == null) { + throw new NullPointerException(); + } + if (value == null) { + throw new NullPointerException(); + } + V previous = delegate().put(key, value); + journal.put(key, value); + markAsDirty(); + return previous; + } + + /** + * Marks the map as dirty and potentially writes updated entries to the + * journal. + */ + private void markAsDirty() { + dirty = true; + if (updateJournal()) { + writeJournal(); + } + } + + /** + * Determines if the journal should be updated. The default implementation + * always returns 'true', but subclasses are free to override this to + * implement their own journal updating strategy. For example it is possible + * to implement an update at most every five seconds using the following code: + * + * <pre> + * private long nextUpdate; + * protected boolean updateJournal() { + * long time = System.currentTimeMillis(); + * if (time > nextUpdate) { + * nextUpdate = time + 5 * 1000; + * return true; + * } + * return false; + * } + * </pre> + */ + protected boolean updateJournal() { + return true; + } + + @Override + @SuppressWarnings("unchecked") + public V remove(Object object) { + V previous = delegate().remove(object); + if (previous != null) { + // we know that 'object' must be an instance of K, because the + // remove call succeeded, i.e. 'object' was mapped to 'previous'. + journal.put((K) object, null); // unchecked + markAsDirty(); + } + return previous; + } + + /** + * Updates the persistent journal by writing all entries to the + * {@link #journalOut} stream and clearing the in memory journal. + */ + private void writeJournal() { + try { + if (journalOut == null) { + journalOut = createMapFile(journalFile); + } + writeEntries(journalOut, journal); + journalOut.flush(); + journal.clear(); + } catch (IOException e) { + this.deferredIOFailure = e.getMessage() + " during journal append"; + } + } + + protected void forceFlush() { + if (dirty) { + writeJournal(); + } + } + + /** + * Load the previous written map entries from disk. + * + * @param failFast if true, throw IOException rather than silently ignoring. + * @throws IOException + */ + public void load(boolean failFast) throws IOException { + if (!loaded) { + loadEntries(mapFile, failFast); + if (journalFile.exists()) { + try { + loadEntries(journalFile, failFast); + } catch (IOException e) { + if (failFast) { + throw e; + } + //Else: ignore any errors reading the journal file as it may contain + //partial entries. + } + // Force the map to be dirty, so that we can save it to disk. + dirty = true; + save(/*fullSave=*/true); + } else { + dirty = false; + } + loaded = true; + } + } + + /** + * Load the previous written map entries from disk. + * + * @throws IOException + */ + public void load() throws IOException { + load(/*throwOnLoadFailure=*/false); + } + + @Override + public void clear() { + super.clear(); + markAsDirty(); + try { + save(); + } catch (IOException e) { + this.deferredIOFailure = e.getMessage() + " during map write"; + } + } + + /** + * Saves all the entries of this map to disk and deletes the journal file. + * + * @throws IOException if there was an I/O error during this call, or any previous call since the + * last save(). + */ + public long save() throws IOException { + return save(false); + } + + /** + * Saves all the entries of this map to disk and deletes the journal file. + * + * @param fullSave if true, always write the full cache to disk, without the + * journal. + * @throws IOException if there was an I/O error during this call, or any + * previous call since the last save(). + */ + private long save(boolean fullSave) throws IOException { + /* Report a previously failing I/O operation. */ + if (deferredIOFailure != null) { + try { + throw new IOException(deferredIOFailure); + } finally { + deferredIOFailure = null; + } + } + if (dirty) { + if (!fullSave && keepJournal()) { + forceFlush(); + journalOut.close(); + journalOut = null; + return journalSize() + cacheSize(); + } else { + dirty = false; + Path mapTemp = + mapFile.getRelative(FileSystemUtils.replaceExtension(mapFile.asFragment(), ".tmp")); + try { + saveEntries(delegate(), mapTemp); + mapTemp.renameTo(mapFile); + } finally { + mapTemp.delete(); + } + clearJournal(); + journalFile.delete(); + return cacheSize(); + } + } else { + return cacheSize(); + } + } + + protected final long journalSize() throws IOException { + return journalFile.exists() ? journalFile.getFileSize() : 0; + } + + protected final long cacheSize() throws IOException { + return mapFile.exists() ? mapFile.getFileSize() : 0; + } + + /** + * If true, keep the journal during the save(). The journal is flushed, but + * the map file is not touched. This may be useful in cases where the journal + * is much smaller than the map. + */ + protected boolean keepJournal() { + return false; + } + + private void clearJournal() throws IOException { + journal.clear(); + if (journalOut != null) { + journalOut.close(); + journalOut = null; + } + } + + private void loadEntries(Path mapFile, boolean failFast) throws IOException { + if (!mapFile.exists()) { + return; + } + DataInputStream in = + new DataInputStream(new BufferedInputStream(mapFile.getInputStream())); + try { + long fileSize = mapFile.getFileSize(); + if (fileSize < (16)) { + if (failFast) { + throw new IOException(mapFile + " is too short: Only " + fileSize + " bytes"); + } else { + return; + } + } + if (in.readLong() != MAGIC) { // not a PersistentMap + if (failFast) { + throw new IOException("Unexpected format"); + } + return; + } + if (in.readLong() != version) { // PersistentMap version incompatible + if (failFast) { + throw new IOException("Unexpected format"); + } + return; + } + readEntries(in, failFast); + } finally { + in.close(); + } + } + + /** + * Saves the entries in the specified map into the specified file. + * + * @param map the map to be written into the file. + * @param mapFile the file the map is written to. + * @throws IOException + */ + private void saveEntries(Map<K, V> map, Path mapFile) throws IOException { + DataOutputStream out = createMapFile(mapFile); + writeEntries(out, map); + out.close(); + } + + /** + * Creates the specified file and returns the DataOuputStream suitable for writing entries. + * + * @param mapFile the file the map is written to. + * @return the DataOutputStream that was can be used for saving the map to the file. + * @throws IOException + */ + private DataOutputStream createMapFile(Path mapFile) throws IOException { + FileSystemUtils.createDirectoryAndParents(mapFile.getParentDirectory()); + DataOutputStream out = + new DataOutputStream(new BufferedOutputStream(mapFile.getOutputStream())); + out.writeLong(MAGIC); + out.writeLong(version); + return out; + } + + /** + * Writes the Map entries to the specified DataOutputStream. + * + * @param out the DataOutputStream to write the Map entries to. + * @param map the Map containing the entries to be written to the + * DataOutputStream. + * @throws IOException + */ + private void writeEntries(DataOutputStream out, Map<K, V> map) + throws IOException { + for (Map.Entry<K, V> entry : map.entrySet()) { + out.writeByte(ENTRY_MAGIC); + writeKey(entry.getKey(), out); + V value = entry.getValue(); + boolean isEntry = (value != null); + out.writeBoolean(isEntry); + if (isEntry) { + writeValue(value, out); + } + } + } + + /** + * Reads the Map entries from the specified DataInputStream. + * + * @param failFast if true, throw IOException if entries are in an unexpected + * format. + * @param in the DataInputStream to read the Map entries from. + * @throws IOException + */ + private void readEntries(DataInputStream in, boolean failFast) throws IOException { + Map<K, V> map = delegate(); + while (hasEntries(in, failFast)) { + K key = readKey(in); + boolean isEntry = in.readBoolean(); + if (isEntry) { + V value = readValue(in); + map.put(key, value); + } else { + map.remove(key); + } + } + } + + private boolean hasEntries(DataInputStream in, boolean failFast) throws IOException { + if (in.available() <= 0) { + return false; + } else if (!(in.readUnsignedByte() == ENTRY_MAGIC)) { + if (failFast) { + throw new IOException("Corrupted entry separator"); + } else { + return false; + } + } + return true; + } + + /** + * Writes a key of this map into the specified DataOutputStream. + * + * @param key the key to write to the DataOutputStream. + * @param out the DataOutputStream to write the entry to. + * @throws IOException + */ + protected abstract void writeKey(K key, DataOutputStream out) + throws IOException; + + /** + * Writes a value of this map into the specified DataOutputStream. + * + * @param value the value to write to the DataOutputStream. + * @param out the DataOutputStream to write the entry to. + * @throws IOException + */ + protected abstract void writeValue(V value, DataOutputStream out) + throws IOException; + + /** + * Reads an entry of this map from the specified DataInputStream. + * + * @param in the DataOutputStream to read the entry from. + * @return the entry that was read from the DataInputStream. + * @throws IOException + */ + protected abstract K readKey(DataInputStream in) throws IOException; + + /** + * Reads an entry of this map from the specified DataInputStream. + * + * @param in the DataOutputStream to read the entry from. + * @return the entry that was read from the DataInputStream. + * @throws IOException + */ + protected abstract V readValue(DataInputStream in) throws IOException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ProcMeminfoParser.java b/src/main/java/com/google/devtools/build/lib/util/ProcMeminfoParser.java new file mode 100644 index 0000000..44c1112 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/ProcMeminfoParser.java
@@ -0,0 +1,121 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.CharMatcher; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Map; + +/** + * Parse and return information from /proc/meminfo. + */ +public class ProcMeminfoParser { + + public static final String FILE = "/proc/meminfo"; + + private final Map<String, Long> memInfo; + + /** + * Populates memory information by reading /proc/meminfo. + * @throws IOException if reading the file failed. + */ + public ProcMeminfoParser() throws IOException { + this(FILE); + } + + @VisibleForTesting + public ProcMeminfoParser(String fileName) throws IOException { + List<String> lines = Files.readLines(new File(fileName), Charset.defaultCharset()); + ImmutableMap.Builder<String, Long> builder = ImmutableMap.builder(); + for (String line : lines) { + int colon = line.indexOf(":"); + String keyword = line.substring(0, colon); + String valString = line.substring(colon + 1); + try { + long val = Long.parseLong(CharMatcher.inRange('0', '9').retainFrom(valString)); + builder.put(keyword, val); + } catch (NumberFormatException e) { + // Ignore: we'll fail later if somebody tries to capture this value. + } + } + memInfo = builder.build(); + } + + /** + * Gets a named field in KB. + */ + public long getRamKb(String keyword) { + Long val = memInfo.get(keyword); + if (val == null) { + throw new IllegalArgumentException("Can't locate " + keyword + " in the /proc/meminfo"); + } + return val; + } + + /** + * Return the total physical memory. + */ + public long getTotalKb() { + return getRamKb("MemTotal"); + } + + /** + * Return the inactive memory. + */ + public long getInactiveKb() { + return getRamKb("Inactive"); + } + + /** + * Return the active memory. + */ + public long getActiveKb() { + return getRamKb("Active"); + } + + /** + * Return the slab memory. + */ + public long getSlabKb() { + return getRamKb("Slab"); + } + + /** + * Convert KB to MB. + */ + public static double kbToMb(long kb) { + return kb / 1E3; + } + + /** + * Calculates amount of free RAM from /proc/meminfo content by using + * MemTotal - (Active + 0.3*InActive + 0.8*Slab) formula. + * Assumption here is that we allow Blaze to use all memory except when + * used by active pages, 30% of the inactive pages (since they may become + * active at any time) and 80% of memory used by kernel slab heap (since we + * want to keep most of the slab heap in the memory but do not want it to + * consume all available free memory). + */ + public long getFreeRamKb() { + return getTotalKb() - getActiveKb() - (long)(getInactiveKb() * 0.3) - (long)(getSlabKb() * 0.8); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ProcessUtils.java b/src/main/java/com/google/devtools/build/lib/util/ProcessUtils.java new file mode 100644 index 0000000..ec68736 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/ProcessUtils.java
@@ -0,0 +1,86 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +/** + * OS Process related utilities. + * + * <p>Default implementation forwards all requests to + * {@link com.google.devtools.build.lib.unix.ProcessUtils}. The default implementation + * can be overridden by {@code #setImplementation(ProcessUtilsImpl)} method. + */ +@ThreadSafe +public final class ProcessUtils { + + /** + * Describes implementation to which all {@code ProcessUtils} requests are + * forwarded. + */ + public interface ProcessUtilsImpl { + /** @see ProcessUtils#getgid() */ + int getgid(); + + /** @see ProcessUtils#getpid() */ + int getpid(); + + /** @see ProcessUtils#getuid() */ + int getuid(); + } + + private volatile static ProcessUtilsImpl implementation = new ProcessUtilsImpl() { + + @Override + public int getgid() { + return com.google.devtools.build.lib.unix.ProcessUtils.getgid(); + } + + @Override + public int getpid() { + return com.google.devtools.build.lib.unix.ProcessUtils.getpid(); + } + + @Override + public int getuid() { + return com.google.devtools.build.lib.unix.ProcessUtils.getuid(); + } + }; + + private ProcessUtils() { + // prevent construction. + } + + /** + * @return the real group ID of the current process. + */ + public static int getgid() { + return implementation.getgid(); + } + + /** + * @return the process ID of this process. + */ + public static int getpid() { + return implementation.getpid(); + } + + /** + * @return the real user ID of the current process. + */ + public static int getuid() { + return implementation.getuid(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/RegexFilter.java b/src/main/java/com/google/devtools/build/lib/util/RegexFilter.java new file mode 100644 index 0000000..d7c6834 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/RegexFilter.java
@@ -0,0 +1,167 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.common.base.Joiner; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.OptionsParsingException; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * Handles options that specify list of included/excluded regex expressions. + * Validates whether string is included in that filter. + * + * String is considered to be included into the filter if it does not match + * any of the excluded regex expressions and if it matches at least one + * included regex expression. + */ +public class RegexFilter implements Serializable { + private final Pattern inclusionPattern; + private final Pattern exclusionPattern; + private final int hashCode; + + /** + * Converts from a colon-separated list of regex expressions with optional + * -/+ prefix into the RegexFilter. Colons prefixed with backslash are + * considered to be part of regex definition and not a delimiter between + * separate regex expressions. + * + * Order of expressions is not important. Empty entries are ignored. + * '-' marks an excluded expression. + */ + public static class RegexFilterConverter + implements Converter<RegexFilter> { + + @Override + public RegexFilter convert(String input) throws OptionsParsingException { + List<String> inclusionList = new ArrayList<>(); + List<String> exclusionList = new ArrayList<>(); + + for (String piece : input.split("(?<!\\\\),")) { // Split on ',' but not on '\,' + piece = piece.replace("\\,", ","); + boolean isExcluded = piece.startsWith("-"); + if (isExcluded || piece.startsWith("+")) { + piece = piece.substring(1); + } + if (piece.length() > 0) { + (isExcluded ? exclusionList : inclusionList).add(piece); + } + } + + try { + return new RegexFilter(inclusionList, exclusionList); + } catch (PatternSyntaxException e) { + throw new OptionsParsingException("Failed to build valid regular expression: " + + e.getMessage()); + } + } + + @Override + public String getTypeDescription() { + return "a comma-separated list of regex expressions with prefix '-' specifying" + + " excluded paths"; + } + + } + + /** + * Creates new RegexFilter using provided inclusion and exclusion path lists. + */ + public RegexFilter(List<String> inclusions, List<String> exclusions) { + inclusionPattern = convertRegexListToPattern(inclusions); + exclusionPattern = convertRegexListToPattern(exclusions); + hashCode = Objects.hash(inclusions, exclusions); + } + + /** + * Converts list of regex expressions into one compiled regex expression. + */ + private static Pattern convertRegexListToPattern(List<String> regexList) { + if (regexList.size() == 0) { + return null; + } + // Wrap each individual regex in the independent group, combine them using '|' and + // wrap in the non-capturing group. + return Pattern.compile("(?:(?>" + Joiner.on(")|(?>").join(regexList) + "))"); + } + + /** + * @return true iff given string is included (it is does not match exclusion + * pattern (if any) and matches inclusionPatter (if any). + */ + public boolean isIncluded(String value) { + if (exclusionPattern != null && exclusionPattern.matcher(value).find()) { + return false; + } + if (inclusionPattern == null) { + return true; + } + return inclusionPattern.matcher(value).find(); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + if (inclusionPattern != null) { + builder.append(inclusionPattern.pattern().replace(",", "\\,")); + if (exclusionPattern != null) { + builder.append(","); + } + } + if (exclusionPattern != null) { + builder.append("-"); + builder.append(exclusionPattern.pattern().replace(",", "\\,")); + } + return builder.toString(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof RegexFilter)) { + return false; + } + + RegexFilter otherFilter = (RegexFilter) other; + if (this.exclusionPattern == null ^ otherFilter.exclusionPattern == null) { + return false; + } + if (this.inclusionPattern == null ^ otherFilter.inclusionPattern == null) { + return false; + } + if (this.exclusionPattern != null && !this.exclusionPattern.pattern().equals( + otherFilter.exclusionPattern.pattern())) { + return false; + } + if (this.inclusionPattern != null && !this.inclusionPattern.pattern().equals( + otherFilter.inclusionPattern.pattern())) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return hashCode; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ResourceFileLoader.java b/src/main/java/com/google/devtools/build/lib/util/ResourceFileLoader.java new file mode 100644 index 0000000..ce26c6c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/ResourceFileLoader.java
@@ -0,0 +1,57 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.io.ByteStreams; + +import java.io.IOException; +import java.io.InputStream; + +/** + * A little utility to load resources (property files) from jars or + * the classpath. Recommended for longer texts that do not fit nicely into + * a piece of Java code - e.g. a template for a lengthy email. + */ +public final class ResourceFileLoader { + + private ResourceFileLoader() {} + + /** + * Loads a text resource that is located in a directory on the Java classpath that + * corresponds to the package of <code>relativeToClass</code> using UTF8 encoding. + * E.g. + * <code>loadResource(Class.forName("com.google.foo.Foo", "bar.txt"))</code> + * will look for <code>com/google/foo/bar.txt</code> in the classpath. + */ + public static String loadResource(Class<?> relativeToClass, String resourceName) + throws IOException { + ClassLoader loader = relativeToClass.getClassLoader(); + // TODO(bazel-team): use relativeToClass.getPackage().getName(). + String className = relativeToClass.getName(); + String packageName = className.substring(0, className.lastIndexOf('.')); + String path = packageName.replace('.', '/'); + String resource = path + '/' + resourceName; + InputStream stream = loader.getResourceAsStream(resource); + if (stream == null) { + throw new IOException(resourceName + " not found."); + } + try { + return new String(ByteStreams.toByteArray(stream), UTF_8); + } finally { + stream.close(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ResourceUsage.java b/src/main/java/com/google/devtools/build/lib/util/ResourceUsage.java new file mode 100644 index 0000000..55807f2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/ResourceUsage.java
@@ -0,0 +1,353 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import static java.nio.charset.StandardCharsets.US_ASCII; + +import com.google.common.base.CharMatcher; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; +import com.google.common.io.Files; + +import com.sun.management.OperatingSystemMXBean; + +import java.io.File; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.util.Iterator; + +/** + * Provides methods to measure the current resource usage of the current + * process. Also provides some convenience methods to obtain several system + * characteristics, like number of processors , total memory, etc. + */ +public final class ResourceUsage { + + /* + * Use com.sun.management.OperatingSystemMXBean instead of + * java.lang.management.OperatingSystemMXBean because the latter does not + * support getTotalPhysicalMemorySize() and getFreePhysicalMemorySize(). + */ + private static final OperatingSystemMXBean OS_BEAN = + (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); + + private static final MemoryMXBean MEM_BEAN = ManagementFactory.getMemoryMXBean(); + private static final Splitter WHITESPACE_SPLITTER = Splitter.on(CharMatcher.WHITESPACE); + + /** + * Calculates an estimate of the current total CPU usage and the CPU usage of + * the process in percent measured from the two given measurements. The + * returned CPU usages rea average values for the time between the two + * measurements. The returned array contains the total CPU usage at index 0 + * and the CPU usage of the measured process at index 1. + */ + public static float[] calculateCurrentCpuUsage(Measurement oldMeasurement, + Measurement newMeasurement) { + if (oldMeasurement == null) { + return new float[2]; + } + long idleJiffies = + newMeasurement.getTotalCpuIdleTimeInJiffies() + - oldMeasurement.getTotalCpuIdleTimeInJiffies(); + long oldProcessJiffies = + oldMeasurement.getCpuUtilizationInJiffies()[0] + + oldMeasurement.getCpuUtilizationInJiffies()[1]; + long newProcessJiffies = + newMeasurement.getCpuUtilizationInJiffies()[0] + + newMeasurement.getCpuUtilizationInJiffies()[1]; + long processJiffies = newProcessJiffies - oldProcessJiffies; + long elapsedTimeJiffies = + newMeasurement.getTimeInJiffies() - oldMeasurement.getTimeInJiffies(); + int processors = getAvailableProcessors(); + // TODO(bazel-team): Sometimes smaller then zero. Not sure why. + double totalUsage = Math.max(0, 1.0D - (double) idleJiffies / elapsedTimeJiffies / processors); + double usage = Math.max(0, (double) processJiffies / elapsedTimeJiffies / processors); + return new float[] {(float) totalUsage * 100, (float) usage * 100}; + } + + private ResourceUsage() { + } + + /** + * Returns the number of processors available to the Java virtual machine. + */ + public static int getAvailableProcessors() { + return OS_BEAN.getAvailableProcessors(); + } + + /** + * Returns the total physical memory in bytes. + */ + public static long getTotalPhysicalMemorySize() { + return OS_BEAN.getTotalPhysicalMemorySize(); + } + + /** + * Returns the operating system architecture. + */ + public static String getOsArchitecture() { + return OS_BEAN.getArch(); + } + + /** + * Returns the operating system name. + */ + public static String getOsName() { + return OS_BEAN.getName(); + } + + /** + * Returns the operating system version. + */ + public static String getOsVersion() { + return OS_BEAN.getVersion(); + } + + /** + * Returns the initial size of heap memory in bytes. + * + * @see MemoryMXBean#getHeapMemoryUsage() + */ + public static long getHeapMemoryInit() { + return MEM_BEAN.getHeapMemoryUsage().getInit(); + } + + /** + * Returns the initial size of non heap memory in bytes. + * + * @see MemoryMXBean#getNonHeapMemoryUsage() + */ + public static long getNonHeapMemoryInit() { + return MEM_BEAN.getNonHeapMemoryUsage().getInit(); + } + + /** + * Returns the maximum size of heap memory in bytes. + * + * @see MemoryMXBean#getHeapMemoryUsage() + */ + public static long getHeapMemoryMax() { + return MEM_BEAN.getHeapMemoryUsage().getMax(); + } + + /** + * Returns the maximum size of non heap memory in bytes. + * + * @see MemoryMXBean#getNonHeapMemoryUsage() + */ + public static long getNonHeapMemoryMax() { + return MEM_BEAN.getNonHeapMemoryUsage().getMax(); + } + + /** + * Returns a measurement of the current resource usage of the current process. + */ + public static Measurement measureCurrentResourceUsage() { + return measureCurrentResourceUsage("self"); + } + + /** + * Returns a measurement of the current resource usage of the process with the + * given process id. + * + * @param processId the process id or <code>self</code> for the current + * process. + */ + public static Measurement measureCurrentResourceUsage(String processId) { + return new Measurement(MEM_BEAN.getHeapMemoryUsage().getUsed(), MEM_BEAN.getHeapMemoryUsage() + .getCommitted(), MEM_BEAN.getNonHeapMemoryUsage().getUsed(), MEM_BEAN + .getNonHeapMemoryUsage().getCommitted(), (float) OS_BEAN.getSystemLoadAverage(), OS_BEAN + .getFreePhysicalMemorySize(), getCurrentTotalIdleTimeInJiffies(), + getCurrentCpuUtilizationInJiffies(processId)); + } + + /** + * Returns the current total idle time of the processors since system boot. + * Reads /proc/stat to obtain this information. + */ + private static long getCurrentTotalIdleTimeInJiffies() { + try { + File file = new File("/proc/stat"); + String content = Files.toString(file, US_ASCII); + String value = Iterables.get(WHITESPACE_SPLITTER.split(content), 5); + return Long.parseLong(value); + } catch (NumberFormatException | IOException e) { + return 0L; + } + } + + /** + * Returns the current cpu utilization of the current process with the given + * id in jiffies. The returned array contains the following information: The + * 1st entry is the number of jiffies that the process has executed in user + * mode, and the 2nd entry is the number of jiffies that the process has + * executed in kernel mode. Reads /proc/self/stat to obtain this information. + * + * @param processId the process id or <code>self</code> for the current + * process. + */ + private static long[] getCurrentCpuUtilizationInJiffies(String processId) { + try { + File file = new File("/proc/" + processId + "/stat"); + if (file.isDirectory()) { + return new long[2]; + } + Iterator<String> stat = WHITESPACE_SPLITTER.split( + Files.toString(file, US_ASCII)).iterator(); + for (int i = 0; i < 13; ++i) { + stat.next(); + } + long token13 = Long.parseLong(stat.next()); + long token14 = Long.parseLong(stat.next()); + return new long[] { token13, token14 }; + } catch (NumberFormatException e) { + return new long[2]; + } catch (IOException e) { + return new long[2]; + } + } + + /** + * A snapshot of the resource usage of the current process at a point in time. + */ + public static class Measurement { + + private final long timeInNanos; + private final long heapMemoryUsed; + private final long heapMemoryCommitted; + private final long nonHeapMemoryUsed; + private final long nonHeapMemoryCommitted; + private final float loadAverageLastMinute; + private final long freePhysicalMemory; + private final long totalCpuIdleTimeInJiffies; + private final long[] cpuUtilizationInJiffies; + + public Measurement(long heapMemoryUsed, long heapMemoryCommitted, long nonHeapMemoryUsed, + long nonHeapMemoryCommitted, float loadAverageLastMinute, long freePhysicalMemory, + long totalCpuIdleTimeInJiffies, long[] cpuUtilizationInJiffies) { + super(); + timeInNanos = System.nanoTime(); + this.heapMemoryUsed = heapMemoryUsed; + this.heapMemoryCommitted = heapMemoryCommitted; + this.nonHeapMemoryUsed = nonHeapMemoryUsed; + this.nonHeapMemoryCommitted = nonHeapMemoryCommitted; + this.loadAverageLastMinute = loadAverageLastMinute; + this.freePhysicalMemory = freePhysicalMemory; + this.totalCpuIdleTimeInJiffies = totalCpuIdleTimeInJiffies; + this.cpuUtilizationInJiffies = cpuUtilizationInJiffies; + } + + /** + * Returns the time of the measurement in jiffies. + */ + public long getTimeInJiffies() { + return timeInNanos / 10000000; + } + + /** + * Returns the time of the measurement in ms. + */ + public long getTimeInMs() { + return timeInNanos / 1000000; + } + + /** + * Returns the amount of used heap memory in bytes at the time of + * measurement. + * + * @see MemoryMXBean#getHeapMemoryUsage() + */ + public long getHeapMemoryUsed() { + return heapMemoryUsed; + } + + /** + * Returns the amount of used non heap memory in bytes at the time of + * measurement. + * + * @see MemoryMXBean#getNonHeapMemoryUsage() + */ + public long getHeapMemoryCommitted() { + return heapMemoryCommitted; + } + + /** + * Returns the amount of memory in bytes that is committed for the Java + * virtual machine to use for the heap at the time of measurement. + * + * @see MemoryMXBean#getHeapMemoryUsage() + */ + public long getNonHeapMemoryUsed() { + return nonHeapMemoryUsed; + } + + /** + * Returns the amount of memory in bytes that is committed for the Java + * virtual machine to use for non heap memory at the time of measurement. + * + * @see MemoryMXBean#getNonHeapMemoryUsage() + */ + public long getNonHeapMemoryCommitted() { + return nonHeapMemoryCommitted; + } + + /** + * Returns the system load average for the last minute at the time of + * measurement. + * + * @see OperatingSystemMXBean#getSystemLoadAverage() + */ + public float getLoadAverageLastMinute() { + return loadAverageLastMinute; + } + + /** + * Returns the free physical memmory in bytes at the time of measurement. + */ + public long getFreePhysicalMemory() { + return freePhysicalMemory; + } + + /** + * Returns the current total cpu idle since system boot in jiffies. + */ + public long getTotalCpuIdleTimeInJiffies() { + return totalCpuIdleTimeInJiffies; + } + + /** + * Returns the current cpu utilization of the current process in jiffies. + * The returned array contains the following information: The 1st entry is + * the number of jiffies that the process has executed in user mode, and the + * 2nd entry is the number of jiffies that the process has executed in + * kernel mode. Reads /proc/self/stat to obtain this information. + */ + public long[] getCpuUtilizationInJiffies() { + return cpuUtilizationInJiffies; + } + + /** + * Returns the current cpu utilization of the current process in ms. The + * returned array contains the following information: The 1st entry is the + * number of ms that the process has executed in user mode, and the 2nd + * entry is the number of ms that the process has executed in kernel mode. + * Reads /proc/self/stat to obtain this information. + */ + public long[] getCpuUtilizationInMs() { + return new long[] {cpuUtilizationInJiffies[0] * 10, cpuUtilizationInJiffies[1] * 10}; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ShellEscaper.java b/src/main/java/com/google/devtools/build/lib/util/ShellEscaper.java new file mode 100644 index 0000000..fd23443 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/ShellEscaper.java
@@ -0,0 +1,202 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.common.base.CharMatcher; +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.collect.Iterables; +import com.google.common.escape.CharEscaperBuilder; +import com.google.common.escape.Escaper; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.io.IOException; + +/** + * Utility class to escape strings for use with shell commands. + * + * <p>Escaped strings may safely be inserted into shell commands. Escaping is + * only done if necessary. Strings containing only shell-neutral characters + * will not be escaped. + * + * <p>This is a replacement for {@code ShellUtils.shellEscape(String)} and + * {@code ShellUtils.prettyPrintArgv(java.util.List)} (see + * {@link com.google.devtools.build.lib.shell.ShellUtils}). Its advantage is the use + * of standard building blocks from the {@code com.google.common.base} + * package, such as {@link Joiner} and {@link CharMatcher}, making this class + * more efficient and reliable than {@code ShellUtils}. + * + * <p>The behavior is slightly different though: this implementation will + * defensively escape non-ASCII letters and digits, whereas + * {@code shellEscape} does not. + */ +@Immutable +public final class ShellEscaper extends Escaper { + // Note: extending Escaper may seem desirable, but is in fact harmful. + // The class would then need to implement escape(Appendable), returning an Appendable + // that escapes everything it receives. In case of shell escaping, we most often join + // string parts on spaces, using a Joiner. Spaces are escaped characters. Using the + // Appendable returned by escape(Appendable) would escape these spaces too, which + // is unwanted. + + public static final ShellEscaper INSTANCE = new ShellEscaper(); + + private static final Function<String, String> AS_FUNCTION = INSTANCE.asFunction(); + + private static final Joiner SPACE_JOINER = Joiner.on(' '); + private static final Escaper STRONGQUOTE_ESCAPER = + new CharEscaperBuilder().addEscape('\'', "'\\''").toEscaper(); + private static final CharMatcher SAFECHAR_MATCHER = + CharMatcher.anyOf("@%-_+:,./") + .or(CharMatcher.inRange('0', '9')) // We can't use CharMatcher.JAVA_LETTER_OR_DIGIT, + .or(CharMatcher.inRange('a', 'z')) // that would also accept non-ASCII digits and + .or(CharMatcher.inRange('A', 'Z')); // letters. + + /** + * Escapes a string by adding strong (single) quotes around it if necessary. + * + * <p>A string is not escaped iff it only contains safe characters. + * The following characters are safe: + * <ul> + * <li>ASCII letters and digits: [a-zA-Z0-9] + * <li>shell-neutral characters: at symbol (@), percent symbol (%), + * dash/minus sign (-), underscore (_), plus sign (+), colon (:), + * comma(,), period (.) and slash (/). + * </ul> + * + * <p>A string is escaped iff it contains at least one non-safe character. + * Escaped strings are created by replacing every occurrence of single + * quotes with the string '\'' and enclosing the result in a pair of + * single quotes. + * + * <p>Examples: + * <ul> + * <li>"{@code foo}" becomes "{@code foo}" (remains the same) + * <li>"{@code +bar}" becomes "{@code +bar}" (remains the same) + * <li>"" becomes "{@code''}" (empty string becomes a pair of strong quotes) + * <li>"{@code $BAZ}" becomes "{@code '$BAZ'}" + * <li>"{@code quote'd}" becomes "{@code 'quote'\''d'}" + * </ul> + */ + @Override + public String escape(String unescaped) { + final String s = unescaped.toString(); + if (s.isEmpty()) { + // Empty string is a special case: needs to be quoted to ensure that it + // gets treated as a separate argument. + return "''"; + } else { + return SAFECHAR_MATCHER.matchesAllOf(s) + ? s + : "'" + STRONGQUOTE_ESCAPER.escape(s) + "'"; + } + } + + public static String escapeString(String unescaped) { + return INSTANCE.escape(unescaped); + } + + /** + * Transforms the input {@code Iterable} of unescaped strings to an + * {@code Iterable} of escaped ones. The escaping is done lazily. + */ + public static Iterable<String> escapeAll(Iterable<? extends String> unescaped) { + return Iterables.transform(unescaped, AS_FUNCTION); + } + + /** + * Escapes all strings in {@code argv} individually and joins them on + * single spaces into {@code out}. The result is appended directly into + * {@code out}, without adding a separator. + * + * <p>This method works as if by invoking + * {@link #escapeJoinAll(Appendable, Iterable, Joiner)} with + * {@code Joiner.on(' ')}. + * + * @param out what the result will be appended to + * @param argv the strings to escape and join + * @return the same reference as {@code out}, now containing the the + * joined, escaped fragments + * @throws IOException if an I/O error occurs while appending + */ + public static Appendable escapeJoinAll(Appendable out, Iterable<? extends String> argv) + throws IOException { + return SPACE_JOINER.appendTo(out, escapeAll(argv)); + } + + /** + * Escapes all strings in {@code argv} individually and joins them into + * {@code out} using the specified {@link Joiner}. The result is appended + * directly into {@code out}, without adding a separator. + * + * <p>The resulting strings are the same as if escaped one by one using + * {@link #escapeString(String)}. + * + * <p>Example: if the joiner is {@code Joiner.on('|')}, then the input + * {@code ["abc", "de'f"]} will be escaped as "{@code abc|'de'\''f'}". + * If {@code out} initially contains "{@code 123}", then the returned + * {@code Appendable} will contain "{@code 123abc|'de'\''f'}". + * + * @param out what the result will be appended to + * @param argv the strings to escape and join + * @param joiner the {@link Joiner} to use to join the escaped strings + * @return the same reference as {@code out}, now containing the the + * joined, escaped fragments + * @throws IOException if an I/O error occurs while appending + */ + public static Appendable escapeJoinAll(Appendable out, Iterable<? extends String> argv, + Joiner joiner) throws IOException { + return joiner.appendTo(out, escapeAll(argv)); + } + + /** + * Escapes all strings in {@code argv} individually and joins them on + * single spaces, then returns the resulting string. + * + * <p>This method works as if by invoking + * {@link #escapeJoinAll(Iterable, Joiner)} with {@code Joiner.on(' ')}. + * + * <p>Example: {@code ["abc", "de'f"]} will be escaped and joined as + * "abc 'de'\''f'". + * + * @param argv the strings to escape and join + * @return the string of escaped and joined input elements + */ + public static String escapeJoinAll(Iterable<? extends String> argv) { + return SPACE_JOINER.join(escapeAll(argv)); + } + + /** + * Escapes all strings in {@code argv} individually and joins them using + * the specified {@link Joiner}, then returns the resulting string. + * + * <p>The resulting strings are the same as if escaped one by one using + * {@link #escapeString(String)}. + * + * <p>Example: if the joiner is {@code Joiner.on('|')}, then the input + * {@code ["abc", "de'f"]} will be escaped and joined as "abc|'de'\''f'". + * + * @param argv the strings to escape and join + * @param joiner the {@link Joiner} to use to join the escaped strings + * @return the string of escaped and joined input elements + */ + public static String escapeJoinAll(Iterable<? extends String> argv, Joiner joiner) { + return joiner.join(escapeAll(argv)); + } + + private ShellEscaper() { + // Utility class - do not instantiate. + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/StringCanonicalizer.java b/src/main/java/com/google/devtools/build/lib/util/StringCanonicalizer.java new file mode 100644 index 0000000..7bdbe7e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/StringCanonicalizer.java
@@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; + +/** + * Static singleton holder for the string interning pool. Doesn't use {@link String#intern} + * because that consumes permgen space. + */ +public final class StringCanonicalizer { + + private static final Interner<String> interner = Interners.newWeakInterner(); + + private StringCanonicalizer() { + } + + /** + * Interns a String. + */ + public final static String intern(String arg) { + return interner.intern(arg); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/StringIndexer.java b/src/main/java/com/google/devtools/build/lib/util/StringIndexer.java new file mode 100644 index 0000000..cf345d2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/StringIndexer.java
@@ -0,0 +1,61 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +/** + * An object that provides bidirectional String <-> unique integer mapping. + */ +public interface StringIndexer { + + /** + * Removes all mappings. + */ + public void clear(); + + /** + * @return some measure of the size of the index. + */ + public int size(); + + /** + * Creates new mapping for the given string if necessary and returns + * string index. Also, as a side effect, zero or more additional mappings + * may be created for various prefixes of the given string. + * + * @return a unique index. + */ + public int getOrCreateIndex(String s); + + /** + * @return a unique index for the given string or -1 if string + * was not added. + */ + public int getIndex(String s); + + /** + * Creates mapping for the given string if necessary. + * Also, as a side effect, zero or more additional mappings may be + * created for various prefixes of the given string. + * + * @return true if new mapping was created, false if mapping already existed. + */ + public boolean addString(String s); + + /** + * @return string associated with the given index or null if + * mapping does not exist. + */ + public String getStringForIndex(int i); + +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/StringTrie.java b/src/main/java/com/google/devtools/build/lib/util/StringTrie.java new file mode 100644 index 0000000..4744c7e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/StringTrie.java
@@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import com.google.common.base.Preconditions; + + +/** + * A trie that operates on path segments of an input string instead of individual characters. + * + * <p>Only accepts strings that contain only low-ASCII characters (0-127) + * + * @param <T> the type of the values + */ +public class StringTrie<T> { + private static final int RANGE = 128; + + @SuppressWarnings("unchecked") + private static class Node<T> { + private Node() { + children = new Node[RANGE]; + } + + private T value; + private Node<T> children[]; + } + + private final Node<T> root; + + public StringTrie() { + root = new Node<T>(); + } + + /** + * Puts a value in the trie. + */ + public void put(CharSequence key, T value) { + Node<T> current = root; + + for (int i = 0; i < key.length(); i++) { + char ch = key.charAt(i); + Preconditions.checkState(ch < RANGE); + Node<T> next = current.children[ch]; + if (next == null) { + next = new Node<T>(); + current.children[ch] = next; + } + + current = next; + } + + current.value = value; + } + + /** + * Gets a value from the trie. If there is an entry with the same key, that will be returned, + * otherwise, the value corresponding to the key that matches the longest prefix of the input. + */ + public T get(String key) { + Node<T> current = root; + T lastValue = current.value; + + for (int i = 0; i < key.length(); i++) { + char ch = key.charAt(i); + Preconditions.checkState(ch < RANGE); + + current = current.children[ch]; + if (current == null) { + break; + } + + if (current.value != null) { + lastValue = current.value; + } + } + + return lastValue; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/StringUtil.java b/src/main/java/com/google/devtools/build/lib/util/StringUtil.java new file mode 100644 index 0000000..40f7ec1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/StringUtil.java
@@ -0,0 +1,175 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +/** + * Various utility methods operating on strings. + */ +public class StringUtil { + /** + * Creates a comma-separated list of words as in English. + * + * <p>Example: ["a", "b", "c"] -> "a, b or c". + */ + public static String joinEnglishList(Iterable<?> choices) { + return joinEnglishList(choices, "or", ""); + } + + /** + * Creates a comma-separated list of words as in English with the given last-separator. + * + * <p>Example with lastSeparator="then": ["a", "b", "c"] -> "a, b then c". + */ + public static String joinEnglishList(Iterable<?> choices, String lastSeparator) { + return joinEnglishList(choices, lastSeparator, ""); + } + + /** + * Creates a comma-separated list of words as in English with the given last-separator and quotes. + * + * <p>Example with lastSeparator="then", quote="'": ["a", "b", "c"] -> "'a', 'b' then 'c'". + */ + public static String joinEnglishList(Iterable<?> choices, String lastSeparator, String quote) { + StringBuilder buf = new StringBuilder(); + for (Iterator<?> ii = choices.iterator(); ii.hasNext(); ) { + Object choice = ii.next(); + if (buf.length() > 0) { + buf.append(ii.hasNext() ? "," : " " + lastSeparator); + buf.append(" "); + } + buf.append(quote).append(choice).append(quote); + } + return buf.length() == 0 ? "nothing" : buf.toString(); + } + + /** + * Split a single space-separated string into a List of values. + * + * <p>Individual values are canonicalized such that within and + * across calls to this method, equal values point to the same + * object. + * + * <p>If the input is null, return an empty list. + * + * @param in space-separated list of values, eg "value1 value2". + */ + public static List<String> splitAndInternString(String in) { + List<String> result = new ArrayList<>(); + if (in == null) { + return result; + } + for (String val : Splitter.on(" ").omitEmptyStrings().split(in)) { + // Note that splitter returns a substring(), effectively + // retaining the entire "in" String. Make an explicit copy here + // to avoid that memory pitfall. Further, because there may be + // many concurrent submissions that touch the same files, + // attempt to use a single reference for equal strings via the + // deduplicator. + result.add(StringCanonicalizer.intern(new String(val))); + } + return result; + } + + /** + * Lists items up to a given limit, then prints how many were omitted. + */ + public static StringBuilder listItemsWithLimit(StringBuilder appendTo, int limit, + Collection<?> items) { + Preconditions.checkState(limit > 0); + Joiner.on(", ").appendTo(appendTo, Iterables.limit(items, limit)); + if (items.size() > limit) { + appendTo.append(" ...(omitting ") + .append(items.size() - limit) + .append(" more item(s))"); + } + return appendTo; + } + + /** + * Returns the ordinal representation of the number. + */ + public static String ordinal(int number) { + switch (number) { + case 1: + return "1st"; + case 2: + return "2nd"; + case 3: + return "3rd"; + default: + return number + "th"; + } + } + + /** + * Appends a prefix and a suffix to each of the Strings. + */ + public static Iterable<String> append(Iterable<String> values, final String prefix, + final String suffix) { + return Iterables.transform(values, new Function<String, String>() { + @Override + public String apply(String input) { + return prefix + input + suffix; + } + }); + } + + /** + * Indents the specified string by the given number of characters. + * + * <p>The beginning of the string before the first newline is not indented. + */ + public static String indent(String input, int depth) { + StringBuilder prefix = new StringBuilder(); + prefix.append("\n"); + for (int i = 0; i < depth; i++) { + prefix.append(" "); + } + + return input.replace("\n", prefix); + } + + /** + * Strips a suffix from a string. If the string does not end with the suffix, returns null. + */ + public static String stripSuffix(String input, String suffix) { + return input.endsWith(suffix) + ? input.substring(0, input.length() - suffix.length()) + : null; + } + + /** + * Capitalizes the first character of a string. + */ + public static String capitalize(String input) { + if (input.isEmpty()) { + return input; + } + + char first = input.charAt(0); + char capitalized = Character.toUpperCase(first); + return first == capitalized ? input : capitalized + input.substring(1); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/StringUtilities.java b/src/main/java/com/google/devtools/build/lib/util/StringUtilities.java new file mode 100644 index 0000000..9ac1d35 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/StringUtilities.java
@@ -0,0 +1,207 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.escape.CharEscaperBuilder; +import com.google.common.escape.Escaper; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * Various utility methods operating on strings. + */ +public class StringUtilities { + + private static final Joiner NEWLINE_JOINER = Joiner.on('\n'); + + private static final Escaper KEY_ESCAPER = new CharEscaperBuilder() + .addEscape('!', "!!") + .addEscape('<', "!<") + .addEscape('>', "!>") + .toEscaper(); + + private static final Escaper CONTROL_CHAR_ESCAPER = new CharEscaperBuilder() + .addEscape('\r', "\\r") + .addEscapes(new char[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, /*13=\r*/ + 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 127}, "<?>") + .toEscaper(); + + /** + * Java doesn't have multiline string literals, so having to join a bunch + * of lines is a very common problem. So, here's a static method that we + * can static import in such situations. + */ + public static String joinLines(String... lines) { + return NEWLINE_JOINER.join(lines); + } + + /** + * A corollary to {@link #joinLines(String[])} for collections. + */ + public static String joinLines(Collection<String> lines) { + return NEWLINE_JOINER.join(lines); + } + + /** + * combineKeys(x1, ..., xn): + * Computes a string that encodes the sequence + * x1, ..., xn. Distinct sequences map to distinct strings. + * + * The encoding is intended to be vaguely human-readable. + */ + public static String combineKeys(Iterable<String> parts) { + final StringBuilder buf = new StringBuilder(128); + for (String part : parts) { + // We enclose each part in angle brackets to separate them. Some + // trickiness is required to ensure that the result is unique (distinct + // sequences map to distinct strings): we escape any angle bracket + // characters in the parts by preceding them with an escape character + // (we use "!") and we also need to escape any escape characters. + buf.append('<'); + buf.append(KEY_ESCAPER.escape(part)); + buf.append('>'); + } + return buf.toString(); + } + + /** + * combineKeys(x1, ..., xn): + * Computes a string that encodes the sequence + * x1, ..., xn. Distinct sequences map to distinct strings. + * + * The encoding is intended to be vaguely human-readable. + */ + public static String combineKeys(String... parts) { + return combineKeys(ImmutableList.copyOf(parts)); + } + + /** + * Replaces all occurrences of 'literal' in 'input' with 'replacement'. + * Like {@link String#replaceAll(String, String)} but for literal Strings + * instead of regular expression patterns. + * + * @param input the input String + * @param literal the literal String to replace in 'input'. + * @param replacement the replacement String to replace 'literal' in 'input'. + * @return the 'input' String with all occurrences of 'literal' replaced with + * 'replacement'. + */ + public static String replaceAllLiteral(String input, String literal, + String replacement) { + int literalLength = literal.length(); + if (literalLength == 0) { + return input; + } + StringBuilder result = new StringBuilder( + input.length() + replacement.length()); + int start = 0; + int index = 0; + + while ((index = input.indexOf(literal, start)) >= 0) { + result.append(input.substring(start, index)); + result.append(replacement); + start = index + literalLength; + } + result.append(input.substring(start)); + return result.toString(); + } + + /** + * Creates a simple key-value table of the form + * + * <pre> + * key: some value + * another key: some other value + * yet another key: and so on ... + * </pre> + * + * The return value will not include a final {@code "\n"}. + */ + public static String layoutTable(Map<String, String> data) { + List<String> tableLines = new ArrayList<>(); + for (Map.Entry<String, String> entry : data.entrySet()) { + tableLines.add(entry.getKey() + ": " + entry.getValue()); + } + return NEWLINE_JOINER.join(tableLines); + } + + /** + * Returns an easy-to-read string approximation of a number of bytes, + * e.g. "21MB". Note, these are IEEE units, i.e. decimal not binary powers. + */ + public static String prettyPrintBytes(long bytes) { + if (bytes < 1E4) { // up to 10KB + return bytes + "B"; + } else if (bytes < 1E7) { // up to 10MB + return ((int) (bytes / 1E3)) + "KB"; + } else if (bytes < 1E11) { // up to 100GB + return ((int) (bytes / 1E6)) + "MB"; + } else { + return ((int) (bytes / 1E9)) + "GB"; + } + } + + /** + * Returns true if 'source' contains 'target' as a sub-array. + */ + public static boolean containsSubarray(char[] source, char[] target) { + if (target.length > source.length) { + return false; + } + for (int i = 0; i < source.length - target.length + 1; i++) { + boolean matches = true; + for (int j = 0; j < target.length; j++) { + if (source[i + j] != target[j]) { + matches = false; + break; + } + } + if (matches) { + return true; + } + } + return false; + } + + /** + * Replace control characters with visible strings. + * @return the sanitized string. + */ + public static String sanitizeControlChars(String message) { + return CONTROL_CHAR_ESCAPER.escape(message); + } + + /** + * Converts a Java style function name to a Python style function name the following way: + * every upper case character gets replaced with an underscore and its lower case counterpart. + * <p>E.g. fooBar --> foo_bar + */ + public static String toPythonStyleFunctionName(String javaStyleFunctionName) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < javaStyleFunctionName.length(); i++) { + char c = javaStyleFunctionName.charAt(i); + if (Character.isUpperCase(c)) { + sb.append('_').append(Character.toLowerCase(c)); + } else { + sb.append(c); + } + } + return sb.toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/ThreadUtils.java b/src/main/java/com/google/devtools/build/lib/util/ThreadUtils.java new file mode 100644 index 0000000..7b8ebed --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/ThreadUtils.java
@@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + + +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Utility methods relating to threads and stack traces. + */ +public class ThreadUtils { + private static final Logger LOG = Logger.getLogger(ThreadUtils.class.getName()); + + private ThreadUtils() { + } + + /** Write a thread dump to the blaze.INFO log if interrupt took too long. */ + public static void warnAboutSlowInterrupt() { + LOG.warning("Interrupt took too long. Dumping thread state."); + for (Map.Entry <Thread, StackTraceElement[]> e : Thread.getAllStackTraces().entrySet()) { + Thread t = e.getKey(); + LOG.warning("\"" + t.getName() + "\"" + " " + + " Thread id=" + t.getId() + " " + t.getState()); + for (StackTraceElement line : e.getValue()) { + LOG.warning("\t" + line); + } + LOG.warning(""); + } + LoggingUtil.logToRemote(Level.WARNING, "Slow interrupt", new SlowInterruptException()); + } + + private static final class SlowInterruptException extends RuntimeException { + public SlowInterruptException() { + super("Slow interruption..."); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/TimeUtilities.java b/src/main/java/com/google/devtools/build/lib/util/TimeUtilities.java new file mode 100644 index 0000000..689744a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/TimeUtilities.java
@@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +/** + * Various utility methods operating on time values. + */ +public class TimeUtilities { + + private TimeUtilities() { + } + + /** + * Converts time to the user-friendly string representation. + * + * @param timeInNs The length of time in nanoseconds. + */ + public static String prettyTime(long timeInNs) { + double ms = timeInNs / 1000000.0; + if (ms < 10.0) { + return String.format("%.2f ms", ms); + } else if (ms < 100.0) { + return String.format("%.1f ms", ms); + } else if (ms < 1000.0) { + return String.format("%.0f ms", ms); + } + return String.format("%.3f s", ms / 1000.0); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/UserUtils.java b/src/main/java/com/google/devtools/build/lib/util/UserUtils.java new file mode 100644 index 0000000..93e2a66 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/UserUtils.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import com.google.common.base.Strings; + +import java.util.Map; + +/** + * User information utility methods. + */ +public final class UserUtils { + + private static final String ORIGINATING_USER_KEY = "BLAZE_ORIGINATING_USER"; + + private UserUtils() { + // prohibit instantiation + } + + private static class Holder { + static final String userName = System.getProperty("user.name"); + } + + /** + * Returns the user name as provided by system property 'user.name'. + */ + public static String getUserName() { + return Holder.userName; + } + + /** + * Returns the originating user for this build from the command-line or the environment. + */ + public static String getOriginatingUser(String originatingUser, + Map<String, String> clientEnv) { + if (!Strings.isNullOrEmpty(originatingUser)) { + return originatingUser; + } + + if (!Strings.isNullOrEmpty(clientEnv.get(ORIGINATING_USER_KEY))) { + return clientEnv.get(ORIGINATING_USER_KEY); + } + + return UserUtils.getUserName(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/VarInt.java b/src/main/java/com/google/devtools/build/lib/util/VarInt.java new file mode 100644 index 0000000..fd5daab --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/VarInt.java
@@ -0,0 +1,286 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +/** + * Common methods to encode and decode varints and varlongs into ByteBuffers and + * arrays. + */ +public class VarInt { + + /** + * Maximum encoded size of 32-bit positive integers (in bytes) + */ + public static final int MAX_VARINT_SIZE = 5; + + /** + * maximum encoded size of 64-bit longs, and negative 32-bit ints (in bytes) + */ + public static final int MAX_VARLONG_SIZE = 10; + + private VarInt() { } + + /** Returns the encoding size in bytes of its input value. + * @param i the integer to be measured + * @return the encoding size in bytes of its input value + */ + public static int varIntSize(int i) { + int result = 0; + do { + result++; + i >>>= 7; + } while (i != 0); + return result; + } + + /** + * Reads a varint from src, places its values into the first element of + * dst and returns the offset in to src of the first byte after the varint. + * + * @param src source buffer to retrieve from + * @param offset offset within src + * @param dst the resulting int value + * @return the updated offset after reading the varint + */ + public static int getVarInt(byte[] src, int offset, int[] dst) { + int result = 0; + int shift = 0; + int b; + do { + if (shift >= 32) { + // Out of range + throw new IndexOutOfBoundsException("varint too long"); + } + // Get 7 bits from next byte + b = src[offset++]; + result |= (b & 0x7F) << shift; + shift += 7; + } while ((b & 0x80) != 0); + dst[0] = result; + return offset; + } + + /** + * Encodes an integer in a variable-length encoding, 7 bits per byte, into a + * destination byte[], following the protocol buffer convention. + * + * @param v the int value to write to sink + * @param sink the sink buffer to write to + * @param offset the offset within sink to begin writing + * @return the updated offset after writing the varint + */ + public static int putVarInt(int v, byte[] sink, int offset) { + do { + // Encode next 7 bits + terminator bit + int bits = v & 0x7F; + v >>>= 7; + byte b = (byte) (bits + ((v != 0) ? 0x80 : 0)); + sink[offset++] = b; + } while (v != 0); + return offset; + } + + /** + * Reads a varint from the current position of the given ByteBuffer and + * returns the decoded value as 32 bit integer. + * + * <p>The position of the buffer is advanced to the first byte after the + * decoded varint. + * + * @param src the ByteBuffer to get the var int from + * @return The integer value of the decoded varint + */ + public static int getVarInt(ByteBuffer src) { + int tmp; + if ((tmp = src.get()) >= 0) { + return tmp; + } + int result = tmp & 0x7f; + if ((tmp = src.get()) >= 0) { + result |= tmp << 7; + } else { + result |= (tmp & 0x7f) << 7; + if ((tmp = src.get()) >= 0) { + result |= tmp << 14; + } else { + result |= (tmp & 0x7f) << 14; + if ((tmp = src.get()) >= 0) { + result |= tmp << 21; + } else { + result |= (tmp & 0x7f) << 21; + result |= (tmp = src.get()) << 28; + while (tmp < 0) { + // We get into this loop only in the case of overflow. + // By doing this, we can call getVarInt() instead of + // getVarLong() when we only need an int. + tmp = src.get(); + } + } + } + } + return result; + } + + /** + * Encodes an integer in a variable-length encoding, 7 bits per byte, to a + * ByteBuffer sink. + * @param v the value to encode + * @param sink the ByteBuffer to add the encoded value + */ + public static void putVarInt(int v, ByteBuffer sink) { + while (true) { + int bits = v & 0x7f; + v >>>= 7; + if (v == 0) { + sink.put((byte) bits); + return; + } + sink.put((byte) (bits | 0x80)); + } + } + + /** + * Reads a varint from the given InputStream and returns the decoded value + * as an int. + * + * @param inputStream the InputStream to read from + */ + public static int getVarInt(InputStream inputStream) throws IOException { + int result = 0; + int shift = 0; + int b; + do { + if (shift >= 32) { + // Out of range + throw new IndexOutOfBoundsException("varint too long"); + } + // Get 7 bits from next byte + b = inputStream.read(); + result |= (b & 0x7F) << shift; + shift += 7; + } while ((b & 0x80) != 0); + return result; + } + + /** + * Encodes an integer in a variable-length encoding, 7 bits per byte, and + * writes it to the given OutputStream. + * + * @param v the value to encode + * @param outputStream the OutputStream to write to + */ + public static void putVarInt(int v, OutputStream outputStream) throws IOException { + byte[] bytes = new byte[varIntSize(v)]; + putVarInt(v, bytes, 0); + outputStream.write(bytes); + } + + /** + * Returns the encoding size in bytes of its input value. + * + * @param v the long to be measured + * @return the encoding size in bytes of a given long value. + */ + public static int varLongSize(long v) { + int result = 0; + do { + result++; + v >>>= 7; + } while (v != 0); + return result; + } + + /** + * Reads an up to 64 bit long varint from the current position of the + * given ByteBuffer and returns the decoded value as long. + * + * <p>The position of the buffer is advanced to the first byte after the + * decoded varint. + * + * @param src the ByteBuffer to get the var int from + * @return The integer value of the decoded long varint + */ + public static long getVarLong(ByteBuffer src) { + long tmp; + if ((tmp = src.get()) >= 0) { + return tmp; + } + long result = tmp & 0x7f; + if ((tmp = src.get()) >= 0) { + result |= tmp << 7; + } else { + result |= (tmp & 0x7f) << 7; + if ((tmp = src.get()) >= 0) { + result |= tmp << 14; + } else { + result |= (tmp & 0x7f) << 14; + if ((tmp = src.get()) >= 0) { + result |= tmp << 21; + } else { + result |= (tmp & 0x7f) << 21; + if ((tmp = src.get()) >= 0) { + result |= tmp << 28; + } else { + result |= (tmp & 0x7f) << 28; + if ((tmp = src.get()) >= 0) { + result |= tmp << 35; + } else { + result |= (tmp & 0x7f) << 35; + if ((tmp = src.get()) >= 0) { + result |= tmp << 42; + } else { + result |= (tmp & 0x7f) << 42; + if ((tmp = src.get()) >= 0) { + result |= tmp << 49; + } else { + result |= (tmp & 0x7f) << 49; + if ((tmp = src.get()) >= 0) { + result |= tmp << 56; + } else { + result |= (tmp & 0x7f) << 56; + result |= ((long) src.get()) << 63; + } + } + } + } + } + } + } + } + return result; + } + + /** + * Encodes a long integer in a variable-length encoding, 7 bits per byte, to a + * ByteBuffer sink. + * @param v the value to encode + * @param sink the ByteBuffer to add the encoded value + */ + public static void putVarLong(long v, ByteBuffer sink) { + while (true) { + int bits = ((int) v) & 0x7f; + v >>>= 7; + if (v == 0) { + sink.put((byte) bits); + return; + } + sink.put((byte) (bits | 0x80)); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/AnsiTerminal.java b/src/main/java/com/google/devtools/build/lib/util/io/AnsiTerminal.java new file mode 100644 index 0000000..93bc12a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/io/AnsiTerminal.java
@@ -0,0 +1,198 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util.io; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * A class which encapsulates the fancy curses-type stuff that you can do using + * standard ANSI terminal control sequences. + */ +public class AnsiTerminal { + private static final byte[] ESC = {27, (byte) '['}; + private static final byte BEL = 7; + private static final byte UP = (byte) 'A'; + private static final byte ERASE_LINE = (byte) 'K'; + private static final byte SET_GRAPHICS = (byte) 'm'; + private static final byte TEXT_BOLD = (byte) '1'; + private static final byte[] SET_TERM_TITLE = {27, (byte) ']', (byte) '0', (byte) ';'}; + + public static final String FG_BLACK = "30"; + public static final String FG_RED = "31"; + public static final String FG_GREEN = "32"; + public static final String FG_YELLOW = "33"; + public static final String FG_BLUE = "34"; + public static final String FG_MAGENTA = "35"; + public static final String FG_CYAN = "36"; + public static final String FG_GRAY = "37"; + public static final String BG_BLACK = "40"; + public static final String BG_RED = "41"; + public static final String BG_GREEN = "42"; + public static final String BG_YELLOW = "43"; + public static final String BG_BLUE = "44"; + public static final String BG_MAGENTA = "45"; + public static final String BG_CYAN = "46"; + public static final String BG_GRAY = "47"; + + public static byte[] CR = { 13 }; + + private final OutputStream out; + + /** + * Creates an AnsiTerminal object wrapping an output stream which is going to + * be displayed in an ANSI compatible terminal or shell window. + * + * @param out the output stream + */ + public AnsiTerminal(OutputStream out) { + this.out = out; + } + + /** + * Moves the cursor upwards by a specified number of lines. This will not + * cause any scrolling if it tries to move above the top of the terminal + * window. + */ + public void cursorUp(int numLines) throws IOException { + writeBytes(ESC, ("" + numLines).getBytes(), new byte[] { UP }); + } + + /** + * Clear the current terminal line from the cursor position to the end. + */ + public void clearLine() throws IOException { + writeEscapeSequence(ERASE_LINE); + } + + /** + * Makes any text output to the terminal appear in bold. + */ + public void textBold() throws IOException { + writeEscapeSequence(TEXT_BOLD, SET_GRAPHICS); + } + + /** + * Set the color of the foreground or background of the terminal. + * + * @param color one of the foreground or background color + * constants + */ + public void setTextColor(String color) throws IOException { + writeBytes(ESC, color.getBytes(), new byte[] { SET_GRAPHICS }); + } + + /** + * Resets the terminal colors and fonts to defaults. + */ + public void resetTerminal() throws IOException { + writeEscapeSequence((byte)'0', (byte)'m'); + } + + /** + * Makes text print on the terminal in red. + */ + public void textRed() throws IOException { + setTextColor(FG_RED); + } + + /** + * Makes text print on the terminal in blue. + */ + public void textBlue() throws IOException { + setTextColor(FG_BLUE); + } + + /** + * Makes text print on the terminal in red. + */ + public void textGreen() throws IOException { + setTextColor(FG_GREEN); + } + + /** + * Makes text print on the terminal in red. + */ + public void textMagenta() throws IOException { + setTextColor(FG_MAGENTA); + } + + /** + * Set the terminal title. + */ + public void setTitle(String title) throws IOException { + writeBytes(SET_TERM_TITLE, title.getBytes(), new byte[] { BEL }); + } + + /** + * Writes a string to the terminal using the current font, color and cursor + * position settings. + * + * @param text the text to write + */ + public void writeString(String text) throws IOException { + out.write(text.getBytes()); + } + + /** + * Writes a byte sequence to the terminal using the current font, color and + * cursor position settings. + * + * @param bytes the bytes to write + */ + public void writeBytes(byte[] bytes) throws IOException { + out.write(bytes); + } + + /** + * Utility method which makes it easier to generate the control sequences for + * the terminal. + * + * @param bytes bytes which should be prefixed with the terminal escape + * sequence to produce a valid control sequence + */ + private void writeEscapeSequence(byte... bytes) throws IOException { + writeBytes(ESC, bytes); + } + + /** + * Utility method for generating control sequences. Takes a collection of byte + * arrays, which contain the components of a control sequence, concatenates + * them, and prints them to the terminal. + * + * @param stuff the byte arrays that make up the sequence to be sent to the + * terminal + */ + private void writeBytes(byte[]... stuff) throws IOException { + for (byte[] bytes : stuff) { + out.write(bytes); + } + } + + /** + * Sends a carriage return to the terminal. + */ + public void cr() throws IOException { + writeBytes(CR); + } + + /** + * Flushes the underlying stream. + * This class does not do any buffering of its own, but the underlying + * OutputStream may. + */ + public void flush() throws IOException { + out.flush(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/AnsiTerminalPrinter.java b/src/main/java/com/google/devtools/build/lib/util/io/AnsiTerminalPrinter.java new file mode 100644 index 0000000..726c5dd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/io/AnsiTerminalPrinter.java
@@ -0,0 +1,156 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util.io; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.EnumSet; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +/** + * Allows to print "colored" strings by parsing predefined string keywords, + * which, depending on the useColor value are either replaced with ANSI terminal + * coloring sequences (as defined by the {@link AnsiTerminal} class) or stripped. + * + * Supported keywords are defined by the enum {@link AnsiTerminalPrinter.Mode}. + * Following keywords are supported: + * INFO - switches color to green. + * ERROR - switches color to bold red. + * WARNING - switches color to magenta. + * NORMAL - resets terminal to the default state. + * + * Each keyword is starts with prefix "{#" followed by the enum constant name + * and suffix "#}". Keywords should not be inserted manually - provided enum + * constants should be used instead. + */ +@ThreadCompatible +public class AnsiTerminalPrinter { + + private static final String MODE_PREFIX = "{#"; + private static final String MODE_SUFFIX = "#}"; + + // Mode pattern must match MODE_PREFIX and do lookahead for the rest of the + // mode string. + private static final String MODE_PATTERN = "\\{\\#(?=[A-Z]+\\#\\})"; + + /** + * List of supported coloring modes for the {@link AnsiTerminalPrinter}. + */ + public static enum Mode { + INFO, // green + ERROR, // bold red + WARNING, // magenta + DEFAULT; // default color + + @Override + public String toString() { + return MODE_PREFIX + name() + MODE_SUFFIX; + } + } + + private static final Logger LOG = Logger.getLogger(AnsiTerminalPrinter.class.getName()); + private static final EnumSet<Mode> MODES = EnumSet.allOf(Mode.class); + private static final Pattern PATTERN = Pattern.compile(MODE_PATTERN); + + private final OutputStream stream; + private final PrintWriter writer; + private final AnsiTerminal terminal; + private boolean useColor; + private Mode lastMode = Mode.DEFAULT; + + /** + * Creates new instance using provided OutputStream and sets coloring logic + * for that instance. + */ + public AnsiTerminalPrinter(OutputStream out, boolean useColor) { + this.useColor = useColor; + terminal = new AnsiTerminal(out); + writer = new PrintWriter(out, true); + stream = out; + } + + /** + * Writes the specified string to the output stream while injecting coloring + * sequences when appropriate mode keyword is found and flushes. + * + * List of supported mode keywords is defined by the enum {@link Mode}. + * + * See class documentation for details. + */ + public void print(String str) { + for (String part : PATTERN.split(str)) { + int index = part.indexOf(MODE_SUFFIX); + // Mode name will contain at least one character, so suffix index + // must be at least 1. If it isn't then there is no match. + if (index > 1) { + for (Mode mode : MODES) { + if (index == mode.name().length() && part.startsWith(mode.name())) { + setupTerminal(mode); + part = part.substring(index + MODE_SUFFIX.length()); + break; + } + } + } + writer.print(part); + writer.flush(); + } + } + + public void printLn(String str) { + print(str + "\n"); + } + + /** + * Returns the underlying OutputStream. + */ + public OutputStream getOutputStream() { + return stream; + } + + /** + * Injects coloring escape sequences if output should be colored and mode + * has been changed. + */ + private void setupTerminal(Mode mode) { + if (!useColor) { + return; + } + try { + if (lastMode != mode) { + terminal.resetTerminal(); + lastMode = mode; + if (mode == Mode.DEFAULT) { + return; // Terminal is already reset - nothing else to do. + } else if (mode == Mode.INFO) { + terminal.textGreen(); + } else if (mode == Mode.ERROR) { + terminal.textRed(); + terminal.textBold(); + } else if (mode == Mode.WARNING) { + terminal.textMagenta(); + } + } + } catch (IOException e) { + // AnsiTerminal state is now considered to be inconsistent - coloring + // should be disabled to prevent future use of AnsiTerminal instance. + LOG.warning("Disabling coloring due to " + e.getMessage()); + useColor = false; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/DelegatingOutErr.java b/src/main/java/com/google/devtools/build/lib/util/io/DelegatingOutErr.java new file mode 100644 index 0000000..83ccf2f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/io/DelegatingOutErr.java
@@ -0,0 +1,113 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util.io; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * An {@link OutErr} specialization that supports subscribing / removing + * sinks, using {@link #addSink(OutErr)} and {@link #removeSink(OutErr)}. + * A sink is a destination to which the {@link DelegatingOutErr} will write. + * + * Also, we can hook up {@link System#out} / {@link System#err} as sources. + */ +public final class DelegatingOutErr extends OutErr { + + /** + * Create a new instance to which no sinks have subscribed (basically just + * like a {@code /dev/null}. + */ + public DelegatingOutErr() { + super(new DelegatingOutputStream(), new DelegatingOutputStream()); + } + + + private final DelegatingOutputStream outSink() { + return (DelegatingOutputStream) getOutputStream(); + } + + private final DelegatingOutputStream errSink() { + return (DelegatingOutputStream) getErrorStream(); + } + + /** + * Add a sink, that is, after calling this method, {@code outErrSink} will + * receive all output / errors written to {@code this} object. + */ + public void addSink(OutErr outErrSink) { + outSink().addSink(outErrSink.getOutputStream()); + errSink().addSink(outErrSink.getErrorStream()); + } + + /** + * Remove the sink, that is, after calling this method, {@code outErrSink} + * will no longer receive output / errors written to {@code this} object. + */ + public void removeSink(OutErr outErrSink) { + outSink().removeSink(outErrSink.getOutputStream()); + errSink().removeSink(outErrSink.getErrorStream()); + } + + private static class DelegatingOutputStream extends OutputStream { + + private final List<OutputStream> sinks = new ArrayList<>(); + + public void addSink(OutputStream sink) { + sinks.add(sink); + } + + public void removeSink(OutputStream sink) { + sinks.remove(sink); + } + + @Override + public void write(int b) throws IOException { + for (OutputStream sink : sinks) { + sink.write(b); + } + } + + @Override + public void close() throws IOException { + for (OutputStream sink : sinks) { + sink.close(); + } + } + + @Override + public void flush() throws IOException { + for (OutputStream sink : sinks) { + sink.flush(); + } + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + for (OutputStream sink : sinks) { + sink.write(b, off, len); + } + } + + @Override + public void write(byte[] b) throws IOException { + for (OutputStream sink : sinks) { + sink.write(b); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/FileOutErr.java b/src/main/java/com/google/devtools/build/lib/util/io/FileOutErr.java new file mode 100644 index 0000000..4f9aecf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/io/FileOutErr.java
@@ -0,0 +1,404 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util.io; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; + +/** + * An implementation of {@link OutErr} that captures all out/err output into + * a file for stdout and a file for stderr. The files are only created if any + * output is made. + * The OutErr assumes that the directory that will contain the output file + * must exist. + * + * You should not use this object from multiple different threads. + */ +@ThreadSafety.ThreadCompatible +public class FileOutErr extends OutErr { + + /** + * Create a new FileOutErr that will write its input, + * if any, to the files specified by stdout/stderr. + * + * No other process may write to the files, + * + * @param stdout The file for the stdout of this outErr + * @param stderr The file for the stderr of this outErr + */ + public FileOutErr(Path stdout, Path stderr) { + super(new FileRecordingOutputStream(stdout), new FileRecordingOutputStream(stderr)); + } + + /** + * Creates a new FileOutErr that writes its input + * to the file specified by output. Both stdout/stderr will + * be copied into the single file. + * + * @param output The file for the both stdout and stderr of this outErr. + */ + public FileOutErr(Path output) { + // We don't need to create a synchronized funnel here, like in the OutErr -- The + // respective functions in the FileRecordingOutputStream take care of locking. + this(new FileRecordingOutputStream(output)); + } + + /** + * Creates a new FileOutErr that discards its input. Useful + * for testing purposes. + */ + @VisibleForTesting + public FileOutErr() { + this(new NullFileRecordingOutputStream()); + } + + private FileOutErr(OutputStream stream) { + // We need this function to duplicate the single new object into both arguments + // of the super-constructor. + super(stream, stream); + } + + /** + * Returns true if any output was recorded. + */ + public boolean hasRecordedOutput() { + return getFileOutputStream().hasRecordedOutput() || getFileErrorStream().hasRecordedOutput(); + } + + /** + * Returns true if output was recorded on stdout. + */ + public boolean hasRecordedStdout() { + return getFileOutputStream().hasRecordedOutput(); + } + + /** + * Returns true if output was recorded on stderr. + */ + public boolean hasRecordedStderr() { + return getFileErrorStream().hasRecordedOutput(); + } + + /** + * Returns the file this OutErr uses to buffer stdout + * + * The user must ensure that no other process is writing to the + * files at time of creation. + * + * @return the path object with the contents of stdout + */ + public Path getOutputFile() { + return getFileOutputStream().getFile(); + } + + /** + * Returns the file this OutErr uses to buffer stderr. + * + * @return the path object with the contents of stderr + */ + public Path getErrorFile() { + return getFileErrorStream().getFile(); + } + + /** + * Interprets the captured out content as an {@code ISO-8859-1} encoded + * string. + */ + public String outAsLatin1() { + return getFileOutputStream().getRecordedOutput(); + } + + /** + * Interprets the captured err content as an {@code ISO-8859-1} encoded + * string. + */ + public String errAsLatin1() { + return getFileErrorStream().getRecordedOutput(); + } + + /** + * Writes the captured out content to the given output stream, + * avoiding keeping the entire contents in memory. + */ + public void dumpOutAsLatin1(OutputStream out) { + getFileOutputStream().dumpOut(out); + } + + /** + * Writes the captured out content to the given output stream, + * avoiding keeping the entire contents in memory. + */ + public void dumpErrAsLatin1(OutputStream out) { + getFileErrorStream().dumpOut(out); + } + + private AbstractFileRecordingOutputStream getFileOutputStream() { + return (AbstractFileRecordingOutputStream) getOutputStream(); + } + + private AbstractFileRecordingOutputStream getFileErrorStream() { + return (AbstractFileRecordingOutputStream) getErrorStream(); + } + + /** + * An abstract supertype for the two other inner classes in this type + * to implement streams that can write to a file. + */ + private abstract static class AbstractFileRecordingOutputStream extends OutputStream { + + /** + * Returns true if this FileRecordingOutputStream has encountered an error. + * + * @return true there was an error, false otherwise. + */ + abstract boolean hadError(); + + /** + * Returns the file this FileRecordingOutputStream is writing to. + */ + abstract Path getFile(); + + /** + * Returns true if the FileOutErr has stored output. + */ + abstract boolean hasRecordedOutput(); + + /** + * Returns the output this AbstractFileOutErr has recorded. + */ + abstract String getRecordedOutput(); + + /** + * Writes the output to the given output stream, + * avoiding keeping the entire contents in memory. + */ + abstract void dumpOut(OutputStream out); + } + + /** + * An output stream that pretends to capture all its output into a file, + * but instead discards it. + */ + private static class NullFileRecordingOutputStream extends AbstractFileRecordingOutputStream { + + NullFileRecordingOutputStream() { + } + + @Override + boolean hadError() { + return false; + } + + @Override + Path getFile() { + return null; + } + + @Override + boolean hasRecordedOutput() { + return false; + } + + @Override + String getRecordedOutput() { + return ""; + } + + @Override + void dumpOut(OutputStream out) { + return; + } + + + @Override + public void write(byte[] b, int off, int len) { + } + + @Override + public void write(int b) { + } + + @Override + public void write(byte[] b) { + } + } + + + /** + * An output stream that captures all output into a file. + * The file is created only if output is received. + * + * The user must take care that nobody else is writing to the + * file that is backing the output stream. + * + * The write() methods of type are synchronized to ensure + * that writes from different threads are not mixed up. + * + * The outputStream is here only for the benefit of the pumping + * IO we're currently using for execution - Once that is gone, + * we can remove this output stream and fold its code into the + * FileOutErr. + */ + @ThreadSafety.ThreadCompatible + private static class FileRecordingOutputStream extends AbstractFileRecordingOutputStream { + + private final Path outputFile; + OutputStream outputStream; + String error; + + FileRecordingOutputStream(Path outputFile) { + this.outputFile = outputFile; + } + + @Override + boolean hadError() { + return error != null; + } + + @Override + Path getFile() { + return outputFile; + } + + private OutputStream getOutputStream() throws IOException { + // you should hold the lock before you invoke this method + if (outputStream == null) { + outputStream = outputFile.getOutputStream(); + } + return outputStream; + } + + private boolean hasOutputStream() { + return outputStream != null; + } + + /** + * Called whenever the FileRecordingOutputStream finds an error. + */ + private void recordError(IOException exception) { + String newErrorText = exception.getMessage(); + error = (error == null) ? newErrorText : error + "\n" + newErrorText; + } + + @Override + boolean hasRecordedOutput() { + if (hadError()) { + return true; + } + if (!outputFile.exists()) { + return false; + } + try { + return outputFile.getFileSize() > 0; + } catch (IOException ex) { + recordError(ex); + return true; + } + } + + @Override + String getRecordedOutput() { + StringBuilder result = new StringBuilder(); + try { + if (getFile().exists()) { + result.append(FileSystemUtils.readContentAsLatin1(getFile())); + } + } catch (IOException ex) { + recordError(ex); + } + + if (hadError()) { + result.append(error); + } + return result.toString(); + } + + @Override + void dumpOut(OutputStream out) { + InputStream in = null; + try { + if (getFile().exists()) { + in = new FileInputStream(getFile().getPathString()); + ByteStreams.copy(in, out); + } + } catch (IOException ex) { + recordError(ex); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + // Ignore. + } + } + } + + if (hadError()) { + PrintStream ps = new PrintStream(out); + ps.print(error); + ps.flush(); + } + } + + @Override + public synchronized void write(byte[] b, int off, int len) { + if (len > 0) { + try { + getOutputStream().write(b, off, len); + } catch (IOException ex) { + recordError(ex); + } + } + } + + @Override + public synchronized void write(int b) { + try { + getOutputStream().write(b); + } catch (IOException ex) { + recordError(ex); + } + } + + @Override + public synchronized void write(byte[] b) throws IOException { + if (b.length > 0) { + getOutputStream().write(b); + } + } + + @Override + public synchronized void flush() throws IOException { + if (hasOutputStream()) { + getOutputStream().flush(); + } + } + + @Override + public synchronized void close() throws IOException { + if (hasOutputStream()) { + getOutputStream().close(); + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/FileWatcher.java b/src/main/java/com/google/devtools/build/lib/util/io/FileWatcher.java new file mode 100644 index 0000000..9355cc3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/io/FileWatcher.java
@@ -0,0 +1,111 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util.io; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * The FileWatcher dumps the contents of a files into an OutErr. + * It then stays active and dumps any content to the OutErr that is + * added to the file, until it is told to stop and all output has + * been dumped. + * + * This is useful to emulate streaming test output. + */ +@ThreadSafe +public class FileWatcher extends Thread { + + // How often we check for updates in the file we watch. (in ms) + private static final int WATCH_INTERVAL = 100; + + private final Path outputFile; + private final OutErr output; + private volatile boolean finishPumping; + private long toSkip = 0; + + /** + * Creates a FileWatcher that will dump any output that is appended to + * outputFile onto output. If skipExisting is true, the watcher will not dump + * any output that is in outputFile when we construct the watcher. If + * skipExisting is false, already existing output will be dumped, too. + * + * @param outputFile the File to watch + * @param output the outErr to dump the files contents to + * @param skipExisting whether to dump already existing output or not. + */ + public FileWatcher(Path outputFile, OutErr output, boolean skipExisting) throws IOException { + super("Streaming Test Output Pump"); + this.outputFile = outputFile; + this.output = output; + finishPumping = false; + + if (outputFile.exists() && skipExisting) { + toSkip = outputFile.getFileSize(); + } + } + + /** + * Tells the FileWatcher to stop pumping output and finish. + * The FileWatcher will only finish until there is no data left to display. + * This means that it is rarely a good idea to unconditionally wait for the + * FileWatcher thread to terminate -- Instead, it is better to have a timeout. + */ + @ThreadSafe + public void stopPumping() { + finishPumping = true; + } + + @Override + public void run() { + + try { + + // Wait until the file exists, or we have to abort. + while (!outputFile.exists() && !finishPumping) { + Thread.sleep(WATCH_INTERVAL); + } + + // Check that we did not have abort before the file was created. + if (outputFile.exists()) { + try (InputStream inputStream = outputFile.getInputStream(); + OutputStream outputStream = output.getOutputStream();) { + byte[] buffer = new byte[1024]; + while (!finishPumping || (inputStream.available() != 0)) { + if (inputStream.available() != 0) { + if (toSkip > 0) { + toSkip -= inputStream.skip(toSkip); + } else { + int read = inputStream.read(buffer); + if (read > 0) { + outputStream.write(buffer, 0, read); + } + } + } else { + Thread.sleep(WATCH_INTERVAL); + } + } + } + } + } catch (IOException ex) { + output.printOutLn("Failure reading or writing: " + ex.getMessage()); + } catch (InterruptedException ex) { + // Don't do anything. + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/LineFlushingOutputStream.java b/src/main/java/com/google/devtools/build/lib/util/io/LineFlushingOutputStream.java new file mode 100644 index 0000000..a5a10cf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/io/LineFlushingOutputStream.java
@@ -0,0 +1,92 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util.io; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * This stream maintains a buffer, which it flushes upon encountering bytes + * that might be new line characters. This stream implements {@link #close()} + * as {@link #flush()}. + */ +abstract class LineFlushingOutputStream extends OutputStream { + + static final int BUFFER_LENGTH = 8192; + protected static byte NEWLINE = '\n'; + + /** + * The buffer containing the characters that have not been flushed yet. + */ + protected final byte[] buffer = new byte[BUFFER_LENGTH]; + + /** + * The length of the buffer that's actually used. + */ + protected int len = 0; + + @Override + public synchronized void write(byte[] b, int off, int inlen) + throws IOException { + if (len == BUFFER_LENGTH) { + flush(); + } + int charsInLine = 0; + while(inlen > charsInLine) { + boolean sawNewline = (b[off + charsInLine] == NEWLINE); + charsInLine++; + if (sawNewline || len + charsInLine == BUFFER_LENGTH) { + System.arraycopy(b, off, buffer, len, charsInLine); + len += charsInLine; + off += charsInLine; + inlen -= charsInLine; + flush(); + charsInLine = 0; + } + } + System.arraycopy(b, off, buffer, len, charsInLine); + len += charsInLine; + } + + @Override + public void write(int byteAsInt) throws IOException { + byte b = (byte) byteAsInt; // make sure we work with bytes in comparisons + write(new byte[] {b}, 0, 1); + } + + /** + * Close is implemented as {@link #flush()}. Client code must close the + * underlying output stream itself in case that's desired. + */ + @Override + public synchronized void close() throws IOException { + flush(); + } + + @Override + public final synchronized void flush() throws IOException { + flushingHook(); // The point of using a hook is to make it synchronized. + } + + /** + * The implementing class must define this method, which must at least flush + * the bytes in {@code buffer[0] - buffer[len - 1]}, and reset {@code len=0}. + * + * Don't forget to synchronized the implementation of this method on whatever + * underlying object it writes to! + */ + protected abstract void flushingHook() throws IOException; + +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/LinePrefixingOutputStream.java b/src/main/java/com/google/devtools/build/lib/util/io/LinePrefixingOutputStream.java new file mode 100644 index 0000000..23d6cd7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/io/LinePrefixingOutputStream.java
@@ -0,0 +1,73 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util.io; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * A stream that writes to another one, emittig a prefix before every line + * it emits. This stream will also add a newline for every flush; so it's not + * useful for anything other than simple text data (e.g. log files). Here's + * an example which demonstrates how an explicit flush or a flush caused by + * a full buffer causes a newline to be added to the output. + * + * <code> + * foo bar + * baz ba[flush]ng + * boo + * </code> + * + * This results in this output being emitted: + * + * <code> + * my prefix: foo bar + * my prefix: ba + * my prefix: ng + * my prefix: boo + * </code> + */ +public final class LinePrefixingOutputStream extends LineFlushingOutputStream { + + private byte[] linePrefix; + private final OutputStream sink; + + public LinePrefixingOutputStream(String linePrefix, OutputStream sink) { + this.linePrefix = linePrefix.getBytes(UTF_8); + this.sink = sink; + } + + @Override + protected void flushingHook() throws IOException { + synchronized (sink) { + if (len == 0) { + sink.flush(); + return; + } + byte lastByte = buffer[len - 1]; + boolean lineIsIncomplete = lastByte != NEWLINE; + sink.write(linePrefix); + sink.write(buffer, 0, len); + if (lineIsIncomplete) { + sink.write(NEWLINE); + } + sink.flush(); + len = 0; + } + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/OutErr.java b/src/main/java/com/google/devtools/build/lib/util/io/OutErr.java new file mode 100644 index 0000000..c4700ea --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/io/OutErr.java
@@ -0,0 +1,132 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util.io; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.PrintWriter; + +/** + * A pair of output streams to be used for redirecting the output and error + * streams of a subprocess. + */ +public class OutErr { + + private final OutputStream out; + private final OutputStream err; + + public static final OutErr SYSTEM_OUT_ERR = create(System.out, System.err); + + /** + * Creates a new OutErr instance from the specified output and error streams. + */ + public static OutErr create(OutputStream out, OutputStream err) { + return new OutErr(out, err); + } + + protected OutErr(OutputStream out, OutputStream err) { + this.out = out; + this.err = err; + } + + /** + * This method redirects {@link System#out} / {@link System#err} into + * {@code this} object. After calling this method, writing to + * {@link System#out} or {@link System#err} will result in + * {@code "System.out: " + message} or {@code "System.err: " + message} + * being written to the OutputStreams of {@code this} instance. + * + * Note: This method affects global variables. + */ + public void addSystemOutErrAsSource() { + System.setOut(new PrintStream(new LinePrefixingOutputStream("System.out: ", getOutputStream()), + /*autoflush=*/false)); + System.setErr(new PrintStream(new LinePrefixingOutputStream("System.err: ", getErrorStream()), + /*autoflush=*/false)); + } + + /** + * Creates a new OutErr instance from the specified stream. + * Writes to either the output or err of the new OutErr are written + * to outputStream, synchronized. + */ + public static OutErr createSynchronizedFunnel(final OutputStream outputStream) { + OutputStream syncOut = new OutputStream() { + + @Override + public synchronized void write(int b) throws IOException { + outputStream.write(b); + } + + @Override + public synchronized void write(byte b[]) throws IOException { + outputStream.write(b); + } + + @Override + public synchronized void write(byte b[], int off, int len) throws IOException { + outputStream.write(b, off, len); + } + + @Override + public synchronized void flush() throws IOException { + outputStream.flush(); + } + + @Override + public synchronized void close() throws IOException { + outputStream.close(); + } + }; + + return create(syncOut, syncOut); + } + + public OutputStream getOutputStream() { + return out; + } + + public OutputStream getErrorStream() { + return err; + } + + /** + * Writes the specified string to the output stream, and flushes. + */ + public void printOut(String s) { + PrintWriter writer = new PrintWriter(out, true); + writer.print(s); + writer.flush(); + } + + public void printOutLn(String s) { + printOut(s + "\n"); + } + + /** + * Writes the specified string to the error stream, and flushes. + */ + public void printErr(String s) { + PrintWriter writer = new PrintWriter(err, true); + writer.print(s); + writer.flush(); + } + + public void printErrLn(String s) { + printErr(s + "\n"); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/RecordingOutErr.java b/src/main/java/com/google/devtools/build/lib/util/io/RecordingOutErr.java new file mode 100644 index 0000000..d276afc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/io/RecordingOutErr.java
@@ -0,0 +1,91 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util.io; + +import java.io.ByteArrayOutputStream; +import java.io.UnsupportedEncodingException; + +/** + * An implementation of {@link OutErr} that captures all out / err output and + * makes it available as ISO-8859-1 strings. Useful for implementing test + * cases that assert particular output. + */ +public class RecordingOutErr extends OutErr { + + public RecordingOutErr() { + super(new ByteArrayOutputStream(), new ByteArrayOutputStream()); + } + + public RecordingOutErr(ByteArrayOutputStream out, ByteArrayOutputStream err) { + super(out, err); + } + + /** + * Reset the captured content; that is, reset the out / err buffers. + */ + public void reset() { + getOutputStream().reset(); + getErrorStream().reset(); + } + + /** + * Interprets the captured out content as an {@code ISO-8859-1} encoded + * string. + */ + public String outAsLatin1() { + try { + return getOutputStream().toString("ISO-8859-1"); + } catch (UnsupportedEncodingException e) { + throw new AssertionError(e); + } + } + + /** + * Interprets the captured err content as an {@code ISO-8859-1} encoded + * string. + */ + public String errAsLatin1() { + try { + return getErrorStream().toString("ISO-8859-1"); + } catch (UnsupportedEncodingException e) { + throw new AssertionError(e); + } + } + + /** + * Returns true if any output was recorded. + */ + public boolean hasRecordedOutput() { + return getOutputStream().size() > 0 || getErrorStream().size() > 0; + } + + @Override + public String toString() { + String out = outAsLatin1(); + String err = errAsLatin1(); + return "" + ((out.length() > 0) ? ("stdout: " + out + "\n") : "") + + ((err.length() > 0) ? ("stderr: " + err) : ""); + } + + @Override + public ByteArrayOutputStream getOutputStream() { + return (ByteArrayOutputStream) super.getOutputStream(); + } + + @Override + public ByteArrayOutputStream getErrorStream() { + return (ByteArrayOutputStream) super.getErrorStream(); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/StreamDemultiplexer.java b/src/main/java/com/google/devtools/build/lib/util/io/StreamDemultiplexer.java new file mode 100644 index 0000000..ffe0c19 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/io/StreamDemultiplexer.java
@@ -0,0 +1,186 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util.io; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * The dual of {@link StreamMultiplexer}: This is an output stream into which + * you can dump the multiplexed stream, and it delegates the de-multiplexed + * content back into separate channels (instances of {@link OutputStream}). + * + * The format of the tagged output stream is as follows: + * + * <pre> + * combined :: = [ control_line payload ... ]+ + * control_line :: = '@' marker '@'? '\n' + * payload :: = r'^[^\n]*\n' + * </pre> + * + * For more details, please see {@link StreamMultiplexer}. + */ +@ThreadCompatible +public final class StreamDemultiplexer extends OutputStream { + + @Override + public void close() throws IOException { + flush(); + } + + @Override + public void flush() throws IOException { + if (selectedStream != null) { + selectedStream.flush(); + } + } + + private static final byte AT = '@'; + private static final byte NEWLINE = '\n'; + + /** + * The output streams, conveniently in an array indexed by the marker byte. + * Some of these will be null, most likely. + */ + private final OutputStream[] outputStreams = + new OutputStream[Byte.MAX_VALUE + 1]; + + /** + * Each state in this FSM corresponds to a position in the grammar, which is + * simple enough that we can just move through it from beginning to end as we + * parse things. + */ + private enum State { + EXPECT_CONTROL_STARTING_AT, + EXPECT_MARKER_BYTE, + EXPECT_AT_OR_NEWLINE, + EXPECT_PAYLOAD_OR_NEWLINE + } + + private State state = State.EXPECT_CONTROL_STARTING_AT; + private boolean addNewlineToPayload; + private OutputStream selectedStream; + + /** + * Construct a new demultiplexer. The {@code smallestMarkerByte} indicates + * the marker byte we would expect for {@code outputStreams[0]} to be used. + * So, if this first stream is your stdout and you're using the + * {@link StreamMultiplexer}, then you will need to set this to + * {@code '1'}. Because {@link StreamDemultiplexer} extends + * {@link OutputStream}, this constructor effectively creates an + * {@link OutputStream} instance which demultiplexes the tagged data client + * code writes to it into {@code outputStreams}. + */ + public StreamDemultiplexer(byte smallestMarkerByte, + OutputStream... outputStreams) { + for (int i = 0; i < outputStreams.length; i++) { + this.outputStreams[smallestMarkerByte + i] = outputStreams[i]; + } + } + + @Override + public void write(int b) throws IOException { + // This dispatch traverses the finite state machine / grammar. + switch (state) { + case EXPECT_CONTROL_STARTING_AT: + parseControlStartingAt((byte) b); + resetFields(); + break; + case EXPECT_MARKER_BYTE: + parseMarkerByte((byte) b); + break; + case EXPECT_AT_OR_NEWLINE: + parseAtOrNewline((byte) b); + break; + case EXPECT_PAYLOAD_OR_NEWLINE: + parsePayloadOrNewline((byte) b); + break; + } + } + + /** + * Handles {@link State#EXPECT_PAYLOAD_OR_NEWLINE}, which is the payload + * we are actually transporting over the wire. At this point we can rely + * on a stream having been preselected into {@link #selectedStream}, and + * also we will add a newline if {@link #addNewlineToPayload} is set. + * Flushes at the end of every payload segment. + */ + private void parsePayloadOrNewline(byte b) throws IOException { + if (b == NEWLINE) { + if (addNewlineToPayload) { + selectedStream.write(NEWLINE); + } + selectedStream.flush(); + state = State.EXPECT_CONTROL_STARTING_AT; + } else { + selectedStream.write(b); + selectedStream.flush(); // slow? + } + } + + /** + * Handles {@link State#EXPECT_AT_OR_NEWLINE}, which is either the + * suppress newline indicator (at) at the end of a control line, or the end + * of a control line. + */ + private void parseAtOrNewline(byte b) throws IOException { + if (b == NEWLINE) { + state = State.EXPECT_PAYLOAD_OR_NEWLINE; + } else if (b == AT) { + addNewlineToPayload = false; + } else { + throw new IOException("Expected @ or \\n. (" + b + ")"); + } + } + + /** + * Reset the fields that are affected by our state. + */ + private void resetFields() { + selectedStream = null; + addNewlineToPayload = true; + } + + /** + * Handles {@link State#EXPECT_MARKER_BYTE}. The byte determines which stream + * we will be using, and will set {@link #selectedStream}. + */ + private void parseMarkerByte(byte markerByte) throws IOException { + if (markerByte < 0 || markerByte > Byte.MAX_VALUE) { + String msg = "Illegal marker byte (" + markerByte + ")"; + throw new IllegalArgumentException(msg); + } + if (markerByte > outputStreams.length + || outputStreams[markerByte] == null) { + throw new IOException("stream " + markerByte + " not registered."); + } + selectedStream = outputStreams[markerByte]; + state = State.EXPECT_AT_OR_NEWLINE; + } + + /** + * Handles {@link State#EXPECT_CONTROL_STARTING_AT}, the very first '@' with + * which each message starts. + */ + private void parseControlStartingAt(byte b) throws IOException { + if (b != AT) { + throw new IOException("Expected control starting @. (" + b + ", " + + (char) b + ")"); + } + state = State.EXPECT_MARKER_BYTE; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/StreamMultiplexer.java b/src/main/java/com/google/devtools/build/lib/util/io/StreamMultiplexer.java new file mode 100644 index 0000000..c214aa5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/io/StreamMultiplexer.java
@@ -0,0 +1,132 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util.io; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Instances of this class are multiplexers, which redirect multiple + * output streams into a single output stream with tagging so it can be + * de-multiplexed into multiple streams as needed. This allows us to + * use one connection for multiple streams, but more importantly it avoids + * multiple threads or select etc. on the receiving side: A client on the other + * end of a networking connection can simply read the tagged lines and then act + * on them within a sigle thread. + * + * The format of the tagged output stream is as follows: + * + * <pre> + * combined :: = [ control_line payload ... ]+ + * control_line :: = '@' marker '@'? '\n' + * payload :: = r'^[^\n]*\n' + * </pre> + * + * So basically: + * <ul> + * <li>Control lines alternate with payload lines</li> + * <li>Both types of lines end with a newline, and never have a newline in + * them.</li> + * <li>The marker indicates which stream we mean. + * For now, '1'=stdout, '2'=stderr.</li> + * <li>The optional second '@' indicates that the following line is + * incomplete.</li> + * </ul> + * + * This format is optimized for easy interpretation by a Python client, but it's + * also a compromise in that it's still easy to interpret by a human (let's say + * you have to read the traffic over a wire for some reason). + */ +@ThreadSafe +public final class StreamMultiplexer { + + public static final byte STDOUT_MARKER = '1'; + public static final byte STDERR_MARKER = '2'; + public static final byte CONTROL_MARKER = '3'; + + private static final byte AT = '@'; + + private final Object mutex = new Object(); + private final OutputStream multiplexed; + + public StreamMultiplexer(OutputStream multiplexed) { + this.multiplexed = multiplexed; + } + + private class MarkingStream extends LineFlushingOutputStream { + + private final byte markerByte; + + MarkingStream(byte markerByte) { + this.markerByte = markerByte; + } + + @Override + protected void flushingHook() throws IOException { + synchronized (mutex) { + if (len == 0) { + multiplexed.flush(); + return; + } + byte lastByte = buffer[len - 1]; + boolean lineIsIncomplete = lastByte != NEWLINE; + + multiplexed.write(AT); + multiplexed.write(markerByte); + if (lineIsIncomplete) { + multiplexed.write(AT); + } + multiplexed.write(NEWLINE); + multiplexed.write(buffer, 0, len); + if (lineIsIncomplete) { + multiplexed.write(NEWLINE); + } + multiplexed.flush(); + } + len = 0; + } + + } + + /** + * Create a stream that will tag its contributions into the multiplexed stream + * with the marker '1', which means 'stdout'. Each newline byte leads + * to a forced automatic flush. Also, this stream never closes the underlying + * stream it delegates to - calling its {@code close()} method is equivalent + * to calling {@code flush}. + */ + public OutputStream createStdout() { + return new MarkingStream(STDOUT_MARKER); + } + + /** + * Like {@link #createStdout()}, except it tags with the marker '2' to + * indicate 'stderr'. + */ + public OutputStream createStderr() { + return new MarkingStream(STDERR_MARKER); + } + + /** + * Like {@link #createStdout()}, except it tags with the marker '3' to + * indicate control flow.. + */ + public OutputStream createControl() { + return new MarkingStream(CONTROL_MARKER); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/TimestampGranularityMonitor.java b/src/main/java/com/google/devtools/build/lib/util/io/TimestampGranularityMonitor.java new file mode 100644 index 0000000..64575ae --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/io/TimestampGranularityMonitor.java
@@ -0,0 +1,194 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util.io; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.util.Clock; + +/** + * A utility class for dealing with filesystem timestamp granularity issues. + * + * <p> + * Consider a sequence of commands such as + * <pre> + * echo ... > foo/bar + * blaze query ... + * echo ... > foo/bar + * blaze query ... + * </pre> + * + * If these commands all run very fast, it is possible that the timestamp + * on foo/bar is not changed by the second command, even though some time has + * passed, because the times are the same when rounded to the file system + * timestamp granularity (often 1 second). + * For performance, we assume that files + * timestamps haven't changed can safely be cached without reexamining their contents. + * But this assumption would be violated in the above scenario. + * + * <p> + * To address this, we record the current time at the start of executing + * a Blaze command, and whenever we check the timestamp of a source file + * or BUILD file, we check if the timestamp of that source file matches + * the current time. If so, we set a flag. At the end of the command, + * if the flag was set, then we wait until the clock has advanced, so + * that any file modifications performed after the command exits will + * result in a different file timestamp. + * + * <p> + * This class implicitly assumes that each filesystem's clock + * is the same as either System.currentTimeMillis() or + * System.currentTimeMillis() rounded down to the nearest second. + * That is not strictly correct; there might be clock skew between + * the cpu clock and the file system clocks (e.g. for NFS file systems), + * and some file systems might have different granularity (e.g. the old + * DOS FAT filesystem has TWO-second granularity timestamps). + * Clock skew can be addressed using NTP. + * Other granularities could be addressed by small changes to this class, + * if it turns out to be needed. + * + * <p> + * Another alternative design that we considered was to write to a file and + * read its timestamp. But doing that is a little tricky because we don't have + * a FileSystem or Path handy. Also, if we were going to do this, the stamp + * file that is used should be in the same file system as the input files. + * But the input file system(s) might not be writeable, and even if it is, + * it's hard for Blaze to find a writable file on the same filesystem as the + * input files. + */ +@ThreadCompatible +public class TimestampGranularityMonitor { + + /** + * The time of the start of the current Blaze command, + * in milliseconds. + */ + private long commandStartTimeMillis; + + /** + * The time of the start of the current Blaze command, + * in milliseconds, rounded to one second granularity. + */ + private long commandStartTimeMillisRounded; + + /** + * True iff we detected a source file or BUILD file whose (unrounded) + * timestamp matched the time at the start of the current Blaze command + * rounded to the nearest second. + */ + private volatile boolean waitASecond; + + /** + * True iff we detected a source file or BUILD file whose timestamp + * exactly matched the time at the start of the current Blaze command + * (measuring both in integral numbers of milliseconds). + */ + private volatile boolean waitAMillisecond; + + private final Clock clock; + + public TimestampGranularityMonitor(Clock clock) { + this.clock = clock; + } + + /** + * Record the time at which the Blaze command started. + * This is needed for use by waitForTimestampGranularity(). + */ + public void setCommandStartTime() { + this.commandStartTimeMillis = clock.currentTimeMillis(); + this.commandStartTimeMillisRounded = roundDown(this.commandStartTimeMillis); + this.waitASecond = false; + this.waitAMillisecond = false; + } + + /** + * Record that the output of this Blaze command depended on the contents + * of a build file or source file with the specified time stamp. + */ + @ThreadSafe + public void notifyDependenceOnFileTime(long mtime) { + if (mtime == this.commandStartTimeMillis) { + this.waitAMillisecond = true; + } + if (mtime == this.commandStartTimeMillisRounded) { + this.waitASecond = true; + } + } + + /** + * If needed, wait until the next "tick" of the filesystem timestamp clock. + * This is done to ensure that files created after the current Blaze command + * finishes will have timestamps different than files created before the + * current Blaze command started. Otherwise a sequence of commands + * such as + * <pre> + * echo ... > foo/BUILD + * blaze query ... + * echo ... > foo/BUILD + * blaze query ... + * </pre> + * could return wrong results, due to the contents of package foo + * being cached even though foo/BUILD changed. + */ + public void waitForTimestampGranularity(OutErr outErr) { + if (this.waitASecond || this.waitAMillisecond) { + long startedWaiting = Profiler.nanoTimeMaybe(); + boolean interrupted = false; + + if (waitASecond) { + // 50ms slack after the whole-second boundary + while (clock.currentTimeMillis() < commandStartTimeMillisRounded + 1050) { + try { + Thread.sleep(50 /* milliseconds */); + } catch (InterruptedException e) { + if (!interrupted) { + outErr.printErrLn("INFO: Hang on a second..."); + interrupted = true; + } + } + } + } else { + while (clock.currentTimeMillis() == commandStartTimeMillis) { + try { + Thread.sleep(1 /* milliseconds */); + } catch (InterruptedException e) { + if (!interrupted) { + outErr.printErrLn("INFO: Hang on a millisecond..."); + interrupted = true; + } + } + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + + Profiler.instance().logSimpleTask(startedWaiting, ProfilerTask.WAIT, + "Timestamp granularity"); + } + } + + /** + * Rounds the specified time, in milliseconds, down to the nearest second, + * and returns the result in milliseconds. + */ + private static long roundDown(long millis) { + return millis / 1000 * 1000; + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/AbstractFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/AbstractFileSystem.java new file mode 100644 index 0000000..dd4375c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/AbstractFileSystem.java
@@ -0,0 +1,136 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.unix.FileAccessException; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * This class implements the FileSystem interface using direct calls to the + * UNIX filesystem. + */ +@ThreadSafe +abstract class AbstractFileSystem extends FileSystem { + + protected static final String ERR_PERMISSION_DENIED = " (Permission denied)"; + protected static final Profiler profiler = Profiler.instance(); + + @Override + protected InputStream getInputStream(Path path) throws FileNotFoundException { + // This loop is a workaround for an apparent bug in FileInputStrean.open, which delegates + // ultimately to JVM_Open in the Hotspot JVM. This call is not EINTR-safe, so we must do the + // retry here. + for (;;) { + try { + return createFileInputStream(path); + } catch (FileNotFoundException e) { + if (e.getMessage().endsWith("(Interrupted system call)")) { + continue; + } else { + throw e; + } + } + } + } + + /** + * Returns either normal or profiled FileInputStream. + */ + private InputStream createFileInputStream(Path path) throws FileNotFoundException { + final String name = path.toString(); + if (profiler.isActive() && (profiler.isProfiling(ProfilerTask.VFS_READ) || + profiler.isProfiling(ProfilerTask.VFS_OPEN))) { + long startTime = Profiler.nanoTimeMaybe(); + try { + // Replace default FileInputStream instance with the custom one that does profiling. + return new FileInputStream(name) { + @Override public int read(byte b[]) throws IOException { + return read(b, 0, b.length); + } + @Override public int read(byte b[], int off, int len) throws IOException { + long startTime = Profiler.nanoTimeMaybe(); + try { + return super.read(b, off, len); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_READ, name); + } + } + }; + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_OPEN, name); + } + } else { + // Use normal FileInputStream instance if profiler is not enabled. + return new FileInputStream(path.toString()); + } + } + + /** + * Returns either normal or profiled FileOutputStream. Should be used by subclasses + * to create default OutputStream instance. + */ + protected OutputStream createFileOutputStream(Path path, boolean append) + throws FileNotFoundException { + final String name = path.toString(); + if (profiler.isActive() && (profiler.isProfiling(ProfilerTask.VFS_WRITE) || + profiler.isProfiling(ProfilerTask.VFS_OPEN))) { + long startTime = Profiler.nanoTimeMaybe(); + try { + return new FileOutputStream(name, append) { + @Override public void write(byte b[]) throws IOException { + write(b, 0, b.length); + } + @Override public void write(byte b[], int off, int len) throws IOException { + long startTime = Profiler.nanoTimeMaybe(); + try { + super.write(b, off, len); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_WRITE, name); + } + } + }; + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_OPEN, name); + } + } else { + return new FileOutputStream(name, append); + } + } + + @Override + protected OutputStream getOutputStream(Path path, boolean append) throws IOException { + synchronized (path) { + try { + return createFileOutputStream(path, append); + } catch (FileNotFoundException e) { + // Why does it throw a *FileNotFoundException* if it can't write? + // That does not make any sense! And its in a completely different + // format than in other situations, no less! + if (e.getMessage().equals(path + ERR_PERMISSION_DENIED)) { + throw new FileAccessException(e.getMessage()); + } + throw e; + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/BatchStat.java b/src/main/java/com/google/devtools/build/lib/vfs/BatchStat.java new file mode 100644 index 0000000..5144f31 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/BatchStat.java
@@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import java.io.IOException; +import java.util.List; + +/** + * An interface for doing a batch of stat() calls. + */ +public interface BatchStat { + + /** + * + * @param includeDigest whether to include a file digest in the return values. + * @param includeLinks whether to include a symlink stat in the return values. + * @param paths The input paths to stat(), relative to the exec root. + * @return an array list of FileStatusWithDigest in the same order as the input. May + * contain null values. + * @throws IOException on unexpected failure. + * @throws InterruptedException on interrupt. + */ + public List<FileStatusWithDigest> batchStat(boolean includeDigest, + boolean includeLinks, + Iterable<PathFragment> paths) + throws IOException, InterruptedException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Canonicalizer.java b/src/main/java/com/google/devtools/build/lib/vfs/Canonicalizer.java new file mode 100644 index 0000000..294a066 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/Canonicalizer.java
@@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; + +/** + * Static singleton holder for certain interning pools. + */ +public final class Canonicalizer<E> { + + private static final Interner<PathFragment> FRAGMENT_INTERNER = + Interners.newWeakInterner(); + + /** + * Creates an instance of Canonicalizer tracking path fragments. + */ + public static Interner<PathFragment> fragments() { + return FRAGMENT_INTERNER; + } + + private Canonicalizer() { + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Dirent.java b/src/main/java/com/google/devtools/build/lib/vfs/Dirent.java new file mode 100644 index 0000000..a2ee203 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/Dirent.java
@@ -0,0 +1,72 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.common.base.Preconditions; + +import java.io.Serializable; +import java.util.Objects; + +/** + * Directory entry representation returned by {@link Path#readdir}. + */ +public class Dirent implements Serializable { + + /** Type of the directory entry */ + public enum Type { + FILE, + DIRECTORY, + SYMLINK, + UNKNOWN; + } + + private final String name; + private final Type type; + + /** Creates a new dirent with the given name and type, both of which must be non-null. */ + public Dirent(String name, Type type) { + this.name = Preconditions.checkNotNull(name); + this.type = Preconditions.checkNotNull(type); + } + + public String getName() { + return name; + } + + public Type getType() { + return type; + } + + @Override + public int hashCode() { + return Objects.hash(name, type); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Dirent)) { + return false; + } + if (this == other) { + return true; + } + Dirent otherDirent = (Dirent) other; + return name.equals(otherDirent.name) && type.equals(otherDirent.type); + } + + @Override + public String toString() { + return name + "[" + type.toString().toLowerCase() + "]"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileStatus.java b/src/main/java/com/google/devtools/build/lib/vfs/FileStatus.java new file mode 100644 index 0000000..c57b223 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/FileStatus.java
@@ -0,0 +1,82 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import java.io.IOException; + +/** + * File status: mode, mtime, size, etc. + * + * <p>The result of calling any {@code FileStatus} instance method is not + * guaranteed to result in I/O to the file system at the moment of the call. + * The I/O providing the result (and hence the throwing of an I/O exception, + * where applicable) may occur at any moment between the call to {@link + * FileSystem#stat} and the call of the {@code FileStatus} instance method. + * + * <p>Callers therefore cannot assume that all the values are populated + * atomically, or that the results of any two {@code FileStatus} methods + * correspond to state of the file system at a single moment in time. Nor may + * they assume that repeated successful calls to any method of the same + * instance will return the same value. + * + * <p>(This permits conforming implementations to use an atomic {@code stat(2)} + * call on file systems where it is available, and individual accessor methods + * on those where it is not. Caching is possible but not required.) + */ +public interface FileStatus { + + /** + * Returns true iff this file is a regular or special file (e.g. socket, + * fifo or device). + */ + boolean isFile(); + + /** + * Returns true iff this file is a directory. + */ + boolean isDirectory(); + + /** + * Returns true iff this file is a symbolic link. + */ + boolean isSymbolicLink(); + + /** + * Returns the total size, in bytes, of this file. + */ + long getSize() throws IOException; + + /** + * Returns the last modified time of this file's data (milliseconds since + * UNIX epoch). + */ + long getLastModifiedTime() throws IOException; + + /** + * Returns the last change time of this file, where change means any change + * to the file, including metadata changes (milliseconds since UNIX epoch). + * + * Note: UNIX uses seconds! + */ + long getLastChangeTime() throws IOException; + + /** + * Returns the unique file node id. Usually it is computed using both device + * and inode numbers. + * + * <p>Think of this value as a reference to the underlying inode. "mv"ing file a to file b + * ought to cause the node ID of b to change, but appending / modifying b should not. + */ + public long getNodeId() throws IOException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileStatusWithDigest.java b/src/main/java/com/google/devtools/build/lib/vfs/FileStatusWithDigest.java new file mode 100644 index 0000000..3dd62a1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/FileStatusWithDigest.java
@@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import java.io.IOException; + +import javax.annotation.Nullable; + +/** + * A FileStatus that also optionally returns a Digest. + */ +public interface FileStatusWithDigest extends FileStatus { + /** + * @return the digest of the file - optional. + */ + @Nullable + byte[] getDigest() throws IOException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileStatusWithDigestAdapter.java b/src/main/java/com/google/devtools/build/lib/vfs/FileStatusWithDigestAdapter.java new file mode 100644 index 0000000..3f608ce --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/FileStatusWithDigestAdapter.java
@@ -0,0 +1,76 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.common.base.Preconditions; + +import java.io.IOException; + +import javax.annotation.Nullable; + +/** + * An adapter from FileStatus to FileStatusWithDigest. + */ +public class FileStatusWithDigestAdapter implements FileStatusWithDigest { + private final FileStatus stat; + + public static FileStatusWithDigest adapt(FileStatus stat) { + return stat == null ? null : new FileStatusWithDigestAdapter(stat); + } + + private FileStatusWithDigestAdapter(FileStatus stat) { + this.stat = Preconditions.checkNotNull(stat); + } + + @Nullable + @Override + public byte[] getDigest() { + return null; + } + + @Override + public boolean isFile() { + return stat.isFile(); + } + + @Override + public boolean isDirectory() { + return stat.isDirectory(); + } + + @Override + public boolean isSymbolicLink() { + return stat.isSymbolicLink(); + } + + @Override + public long getSize() throws IOException { + return stat.getSize(); + } + + @Override + public long getLastModifiedTime() throws IOException { + return stat.getLastModifiedTime(); + } + + @Override + public long getLastChangeTime() throws IOException { + return stat.getLastChangeTime(); + } + + @Override + public long getNodeId() throws IOException { + return stat.getNodeId(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java new file mode 100644 index 0000000..9d416098 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java
@@ -0,0 +1,632 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.vfs; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.collect.Lists; +import com.google.common.hash.Hashing; +import com.google.common.io.ByteSource; +import com.google.common.io.CharStreams; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.vfs.Dirent.Type; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.util.Collection; +import java.util.List; + +/** + * This interface models a file system using UNIX the naming scheme. + */ +@ThreadSafe +public abstract class FileSystem { + + /** + * An exception thrown when attempting to resolve an ordinary file as a symlink. + */ + protected static final class NotASymlinkException extends IOException { + public NotASymlinkException(Path path) { + super(path.toString()); + } + } + + protected final Path rootPath; + + protected FileSystem() { + this.rootPath = createRootPath(); + } + + /** + * Creates the root of all paths used by this filesystem. This is a hook + * allowing subclasses to define their own root path class. All other paths + * are created via the root path's {@link Path#createChildPath(String)} method. + * <p> + * Beware: this is called during the FileSystem constructor which may occur + * before subclasses are completely initialized. + */ + protected Path createRootPath() { + return new Path(this); + } + + /** + * Returns an absolute path instance, given an absolute path name, without + * double slashes, .., or . segments. While this method will normalize the + * path representation by creating a structured/parsed representation, it will + * not cause any IO. (e.g., it will not resolve symbolic links if it's a Unix + * file system. + */ + public Path getPath(String pathName) { + return getPath(new PathFragment(pathName)); + } + + /** + * Returns an absolute path instance, given an absolute path name, without + * double slashes, .., or . segments. While this method will normalize the + * path representation by creating a structured/parsed representation, it will + * not cause any IO. (e.g., it will not resolve symbolic links if it's a Unix + * file system. + */ + public Path getPath(PathFragment pathName) { + if (!pathName.isAbsolute()) { + throw new IllegalArgumentException(pathName.getPathString() + " (not an absolute path)"); + } + return rootPath.getRelative(pathName); + } + + /** + * Returns a path representing the root directory of the current file system. + */ + public final Path getRootDirectory() { + return rootPath; + } + + /** + * Returns whether or not the FileSystem supports modifications of files and + * file entries. + * + * <p>Returns true if FileSystem supports the following: + * <ul> + * <li>{@link #setWritable(Path, boolean)}</li> + * <li>{@link #setExecutable(Path, boolean)}</li> + * </ul> + * + * The above calls will result in an {@link UnsupportedOperationException} on + * a FileSystem where this method returns {@code false}. + */ + public abstract boolean supportsModifications(); + + /** + * Returns whether or not the FileSystem supports symbolic links. + * + * <p>Returns true if FileSystem supports the following: + * <ul> + * <li>{@link #createSymbolicLink(Path, PathFragment)}</li> + * <li>{@link #getFileSize(Path, boolean)} where {@code followSymlinks=false}</li> + * <li>{@link #getLastModifiedTime(Path, boolean)} where {@code followSymlinks=false}</li> + * <li>{@link #readSymbolicLink(Path)} where the link points to a non-existent file</li> + * </ul> + * + * The above calls will result in an {@link UnsupportedOperationException} on + * a FileSystem where this method returns {@code false}. + */ + public abstract boolean supportsSymbolicLinks(); + + /** + * Returns the type of the file system path belongs to. + * + * <p>The string returned is obtained directly from the operating system, so + * it's a best guess in absence of a guaranteed api. + * + * <p>This implementation uses <code>/proc/mounts</code> to determine the + * file system type. + */ + public String getFileSystemType(Path path) { + String fileSystem = "unknown"; + int bestMountPointSegmentCount = -1; + try { + Path canonicalPath = path.resolveSymbolicLinks(); + Path mountTable = path.getRelative("/proc/mounts"); + for (String line : CharStreams.readLines(new InputStreamReader(mountTable.getInputStream(), + ISO_8859_1))) { + String[] words = line.split("\\s+"); + if (words.length >= 3) { + if (!words[1].startsWith("/")) { + continue; + } + Path mountPoint = path.getFileSystem().getPath(words[1]); + int segmentCount = mountPoint.asFragment().segmentCount(); + if (canonicalPath.startsWith(mountPoint) && segmentCount > bestMountPointSegmentCount) { + bestMountPointSegmentCount = segmentCount; + fileSystem = words[2]; + } + } + } + } catch (IOException e) { + // pass + } + return fileSystem; + } + + + /** + * Creates a directory with the name of the current path. See + * {@link Path#createDirectory} for specification. + */ + protected abstract boolean createDirectory(Path path) throws IOException; + + /** + * Returns the size in bytes of the file denoted by {@code path}. See + * {@link Path#getFileSize(Symlinks)} for specification. + * + * <p>Note: for <@link FileSystem>s where {@link #supportsSymbolicLinks()} + * returns false, this method will throw an + * {@link UnsupportedOperationException} if {@code followSymLinks=false}. + */ + protected abstract long getFileSize(Path path, boolean followSymlinks) throws IOException; + + /** + * Deletes the file denoted by {@code path}. See {@link Path#delete} for + * specification. + */ + protected abstract boolean delete(Path path) throws IOException; + + /** + * Returns the last modification time of the file denoted by {@code path}. + * See {@link Path#getLastModifiedTime(Symlinks)} for specification. + * + * Note: for {@link FileSystem}s where {@link #supportsSymbolicLinks()} returns + * false, this method will throw an {@link UnsupportedOperationException} if + * {@code followSymLinks=false}. + */ + protected abstract long getLastModifiedTime(Path path, + boolean followSymlinks) + throws IOException; + + /** + * Sets the last modification time of the file denoted by {@code path}. See + * {@link Path#setLastModifiedTime} for specification. + */ + protected abstract void setLastModifiedTime(Path path, long newTime) throws IOException; + + /** + * Returns value of the given extended attribute name or null if attribute + * does not exist or file system does not support extended attributes. + * <p>Default implementation assumes that file system does not support + * extended attributes and always returns null. Specific file system + * implementations should override this method if they do provide support + * for extended attributes. + * + * @param path the file whose extended attribute is to be returned. + * @param name the name of the extended attribute key. + * @return the value of the extended attribute associated with 'path', if + * any, or null if no such attribute is defined (ENODATA) or file + * system does not support extended attributes at all. + * @throws IOException if the call failed for any other reason. + */ + protected byte[] getxattr(Path path, String name, boolean followSymlinks) throws IOException { + return null; + } + + /** + * Returns the type of digest that may be returned by {@link #getFastDigest}, or {@code null} + * if the filesystem doesn't support them. + */ + protected String getFastDigestFunctionType(Path path) { + return null; + } + + /** + * Gets a fast digest for the given path, or {@code null} if there isn't one available or the + * filesystem doesn't support them. This digest should be suitable for detecting changes to the + * file. + */ + protected byte[] getFastDigest(Path path) throws IOException { + return null; + } + + /** + * Returns the MD5 digest of the file denoted by {@code path}. See + * {@link Path#getMD5Digest} for specification. + */ + protected byte[] getMD5Digest(final Path path) throws IOException { + // Naive I/O implementation. Subclasses may (and do) optimize. + // This code is only used by the InMemory or Zip or other weird FSs. + return new ByteSource() { + @Override + public InputStream openStream() throws IOException { + return getInputStream(path); + } + }.hash(Hashing.md5()).asBytes(); + } + + /** + * Returns true if "path" denotes an existing symbolic link. See + * {@link Path#isSymbolicLink} for specification. + */ + protected abstract boolean isSymbolicLink(Path path); + + /** + * Appends a single regular path segment 'child' to 'dir', recursively + * resolving symbolic links in 'child'. 'dir' must be canonical. 'maxLinks' is + * the maximum number of symbolic links that may be traversed before it gives + * up (the Linux kernel uses 32). + * + * <p>(This method does not need to be synchronized; but the result may be + * stale in the case of concurrent modification.) + * + * @throws IOException if 'dir' is not an existing directory; or if + * stat(child) fails for any reason, or if 'child' is a symlink and + * readlink(child) fails for any reason (e.g. ENOENT, EACCES), or if + * the chain of symbolic links exceeds 'maxLinks'. + */ + private Path appendSegment(Path dir, String child, int maxLinks) throws IOException { + Path naive = dir.getChild(child); + + PathFragment linkTarget = resolveOneLink(naive); + if (linkTarget == null) { + return naive; // regular file or directory + } + + if (maxLinks-- == 0) { + throw new IOException(naive + " (Too many levels of symbolic links)"); + } + if (linkTarget.isAbsolute()) { dir = rootPath; } + for (String name : linkTarget.segments()) { + if (name.equals(".") || name.equals("")) { + // no-op + } else if (name.equals("..")) { + Path parent = dir.getParentDirectory(); + // root's parent is root, when canonicalizing, so this is a no-op. + if (parent != null) { dir = parent; } + } else { + dir = appendSegment(dir, name, maxLinks); + } + } + return dir; + } + + /** + * Helper method of {@link #resolveSymbolicLinks(Path)}. This method + * encapsulates the I/O component of a full canonicalization operation. + * Subclasses can (and do) provide more efficient implementations. + * + * <p>(This method does not need to be synchronized; but the result may be + * stale in the case of concurrent modification.) + * + * @param path a path, of which all but the last segment is guaranteed to be + * canonical + * @return {@link #readSymbolicLink} iff path is a symlink or null iff + * path exists but is not a symlink + * @throws IOException if the file did not exist, or a parent directory could + * not be searched + */ + protected PathFragment resolveOneLink(Path path) throws IOException { + try { + return readSymbolicLink(path); + } catch (NotASymlinkException e) { + // Not a symbolic link. Check it exists. + + // (A simple call to lstat would replace all of this.) + if (!exists(path, false)) { + throw new FileNotFoundException(path + " (No such file or directory)"); + } + + // TODO(bazel-team): (2009) ideally, throw ENOTDIR if dir is not a dir, but that + // would require twice as many stats, or a much more convoluted + // implementation (like glibc's canonicalize.c). + + return null; // exists. + } + } + + /** + * Returns the canonical path for the given path. See + * {@link Path#resolveSymbolicLinks} for specification. + */ + protected final Path resolveSymbolicLinks(Path path) + throws IOException { + Path parentNode = path.getParentDirectory(); + return parentNode == null + ? path // (root) + : appendSegment(resolveSymbolicLinks(parentNode), path.getBaseName(), 32); + } + + /** + * Returns the status of a file. See {@link Path#stat(Symlinks)} for + * specification. + * + * <p>The default implementation of this method is a "lazy" one, based on + * other accessor methods such as {@link #isFile}, etc. Subclasses may provide + * more efficient specializations. However, we still try to follow Unix-like + * semantics of failing fast in case of non-existent files (or in case of + * permission issues). + */ + protected FileStatus stat(final Path path, final boolean followSymlinks) throws IOException { + FileStatus status = new FileStatus() { + volatile Boolean isFile; + volatile Boolean isDirectory; + volatile Boolean isSymbolicLink; + volatile long size = -1; + volatile long mtime = -1; + + @Override + public boolean isFile() { + if (isFile == null) { isFile = FileSystem.this.isFile(path, followSymlinks); } + return isFile; + } + + @Override + public boolean isDirectory() { + if (isDirectory == null) { + isDirectory = FileSystem.this.isDirectory(path, followSymlinks); + } + return isDirectory; + } + + @Override + public boolean isSymbolicLink() { + if (isSymbolicLink == null) { isSymbolicLink = FileSystem.this.isSymbolicLink(path); } + return isSymbolicLink; + } + + @Override + public long getSize() throws IOException { + if (size == -1) { size = getFileSize(path, followSymlinks); } + return size; + } + + @Override + public long getLastModifiedTime() throws IOException { + if (mtime == -1) { mtime = FileSystem.this.getLastModifiedTime(path, followSymlinks); } + return mtime; + } + + @Override + public long getLastChangeTime() { + throw new UnsupportedOperationException(); + } + + @Override + public long getNodeId() { + throw new UnsupportedOperationException(); + } + }; + + // Fail fast in case if some operations will actually fail, since stat() call sometimes used + // to verify file existence as well. We will use getLastModifiedTime() method for that purpose. + status.getLastModifiedTime(); + + return status; + } + + /** + * Like stat(), but returns null on failures instead of throwing. + */ + protected FileStatus statNullable(Path path, boolean followSymlinks) { + try { + return stat(path, followSymlinks); + } catch (IOException e) { + return null; + } + } + + /** + * Like {@link #stat}, but returns null if the file is not found (corresponding to + * {@code ENOENT} or {@code ENOTDIR} in Unix's stat(2) function) instead of throwing. Note that + * this implementation does <i>not</i> successfully catch {@code ENOTDIR} exceptions. If the + * instantiated filesystem can catch such errors, it should override this method to do so. + */ + protected FileStatus statIfFound(Path path, boolean followSymlinks) throws IOException { + try { + return stat(path, followSymlinks); + } catch (FileNotFoundException e) { + return null; + } + } + + /** + * Returns true iff {@code path} denotes an existing directory. See + * {@link Path#isDirectory(Symlinks)} for specification. + */ + protected abstract boolean isDirectory(Path path, boolean followSymlinks); + + /** + * Returns true iff {@code path} denotes an existing regular or special file. + * See {@link Path#isFile(Symlinks)} for specification. + */ + protected abstract boolean isFile(Path path, boolean followSymlinks); + + /** + * Creates a symbolic link. See {@link Path#createSymbolicLink(Path)} for + * specification. + * + * <p>Note: for {@link FileSystem}s where {@link #supportsSymbolicLinks()} + * returns false, this method will throw an + * {@link UnsupportedOperationException} + */ + protected abstract void createSymbolicLink(Path linkPath, PathFragment targetFragment) + throws IOException; + + /** + * Returns the target of a symbolic link. See {@link Path#readSymbolicLink} + * for specification. + * + * <p>Note: for {@link FileSystem}s where {@link #supportsSymbolicLinks()} + * returns false, this method will throw an + * {@link UnsupportedOperationException} if the link points to a non-existent + * file. + * + * @throws NotASymlinkException if the current path is not a symbolic link + * @throws IOException if the contents of the link could not be read for any reason. + */ + protected abstract PathFragment readSymbolicLink(Path path) throws IOException; + + /** + * Returns true iff {@code path} denotes an existing file of any kind. See + * {@link Path#exists(Symlinks)} for specification. + */ + protected abstract boolean exists(Path path, boolean followSymlinks); + + /** + * Returns a collection containing the names of all entities within the + * directory denoted by the {@code path}. + * + * @throws IOException if there was an error reading the directory entries + */ + protected abstract Collection<Path> getDirectoryEntries(Path path) throws IOException; + + /** + * Returns a Dirents structure, listing the names of all entries within the + * directory {@code path}, plus their types (file, directory, other). + * + * @param followSymlinks whether to follow symlinks when determining the file types of + * individual directory entries. No matter the value of this parameter, symlinks are + * followed when resolving the directory whose entries are to be read. + * @throws IOException if there was an error reading the directory entries + */ + protected Collection<Dirent> readdir(Path path, boolean followSymlinks) throws IOException { + Collection<Path> children = getDirectoryEntries(path); + List<Dirent> dirents = Lists.newArrayListWithCapacity(children.size()); + for (Path child : children) { + FileStatus stat = statNullable(child, followSymlinks); + Dirent.Type type; + if (stat == null) { + type = Type.UNKNOWN; + } else if (stat.isFile()) { + type = Type.FILE; + } else if (stat.isDirectory()) { + type = Type.DIRECTORY; + } else if (stat.isSymbolicLink()) { + type = Type.SYMLINK; + } else { + type = Type.UNKNOWN; + } + dirents.add(new Dirent(child.getBaseName(), type)); + } + return dirents; + } + + /** + * Returns true iff the file represented by {@code path} is readable. + * + * @throws IOException if there was an error reading the file's metadata + */ + protected abstract boolean isReadable(Path path) throws IOException; + + /** + * Sets the file to readable (if the argument is true) or non-readable (if the + * argument is false) + * + * <p>Note: for {@link FileSystem}s where {@link #supportsModifications()} + * returns false or which do not support unreadable files, this method will + * throw an {@link UnsupportedOperationException}. + * + * @throws IOException if there was an error reading or writing the file's metadata + */ + protected abstract void setReadable(Path path, boolean readable) + throws IOException; + + /** + * Returns true iff the file represented by {@code path} is writable. + * + * @throws IOException if there was an error reading the file's metadata + */ + protected abstract boolean isWritable(Path path) throws IOException; + + /** + * Sets the file to writable (if the argument is true) or non-writable (if the + * argument is false) + * + * <p>Note: for {@link FileSystem}s where {@link #supportsModifications()} + * returns false, this method will throw an + * {@link UnsupportedOperationException}. + * + * @throws IOException if there was an error reading or writing the file's metadata + */ + protected abstract void setWritable(Path path, boolean writable) + throws IOException; + + /** + * Returns true iff the file represented by the path is executable. + * + * @throws IOException if there was an error reading the file's metadata + */ + protected abstract boolean isExecutable(Path path) throws IOException; + + /** + * Sets the file to executable, if the argument is true. It is currently not + * supported to unset the executable status of a file, so {code + * executable=false} yields an {@link UnsupportedOperationException}. + * + * <p>Note: for {@link FileSystem}s where {@link #supportsModifications()} + * returns false, this method will throw an + * {@link UnsupportedOperationException}. + * + * @throws IOException if there was an error reading or writing the file's metadata + */ + protected abstract void setExecutable(Path path, boolean executable) throws IOException; + + /** + * Sets the file permissions. If permission changes on this {@link FileSystem} + * are slow (e.g. one syscall per change), this method should aim to be faster + * than setting each permission individually. If this {@link FileSystem} does + * not support group or others permissions, those bits will be ignored. + * + * <p>Note: for {@link FileSystem}s where {@link #supportsModifications()} + * returns false, this method will throw an + * {@link UnsupportedOperationException}. + * + * @throws IOException if there was an error reading or writing the file's metadata + */ + protected void chmod(Path path, int mode) throws IOException { + setReadable(path, (mode & 0400) != 0); + setWritable(path, (mode & 0200) != 0); + setExecutable(path, (mode & 0100) != 0); + } + + /** + * Creates an InputStream accessing the file denoted by the path. + * + * @throws IOException if there was an error opening the file for reading + */ + protected abstract InputStream getInputStream(Path path) throws IOException; + + /** + * Creates an OutputStream accessing the file denoted by path. + * + * @throws IOException if there was an error opening the file for writing + */ + protected final OutputStream getOutputStream(Path path) throws IOException { + return getOutputStream(path, false); + } + + /** + * Creates an OutputStream accessing the file denoted by path. + * + * @param append whether to open the output stream in append mode + * @throws IOException if there was an error opening the file for writing + */ + protected abstract OutputStream getOutputStream(Path path, boolean append) throws IOException; + + /** + * Renames the file denoted by "sourceNode" to the location "targetNode". + * See {@link Path#renameTo} for specification. + */ + protected abstract void renameTo(Path sourcePath, Path targetPath) throws IOException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileSystemUtils.java b/src/main/java/com/google/devtools/build/lib/vfs/FileSystemUtils.java new file mode 100644 index 0000000..bc55032 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/FileSystemUtils.java
@@ -0,0 +1,988 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.io.ByteSink; +import com.google.common.io.ByteSource; +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Helper functions that implement often-used complex operations on file + * systems. + */ +@ConditionallyThreadSafe // ThreadSafe except for deleteTree. +public class FileSystemUtils { + + static final Logger LOG = Logger.getLogger(FileSystemUtils.class.getName()); + static final boolean LOG_FINER = LOG.isLoggable(Level.FINER); + + private FileSystemUtils() {} + + /**************************************************************************** + * Path and PathFragment functions. + */ + + /** + * Throws exceptions if {@code baseName} is not a valid base name. A valid + * base name: + * <ul> + * <li>Is not null + * <li>Is not an empty string + * <li>Is not "." or ".." + * <li>Does not contain a slash + * </ul> + */ + @ThreadSafe + public static void checkBaseName(String baseName) { + if (baseName.length() == 0) { + throw new IllegalArgumentException("Child must not be empty string ('')"); + } + if (baseName.equals(".") || baseName.equals("..")) { + throw new IllegalArgumentException("baseName must not be '" + baseName + "'"); + } + if (baseName.indexOf('/') != -1) { + throw new IllegalArgumentException("baseName must not contain a slash: '" + baseName + "'"); + } + } + + /** + * Returns the common ancestor between two paths, or null if none (including + * if they are on different filesystems). + */ + public static Path commonAncestor(Path a, Path b) { + while (a != null && !b.startsWith(a)) { + a = a.getParentDirectory(); // returns null at root + } + return a; + } + + /** + * Returns the longest common ancestor of the two path fragments, or either "/" or "" (depending + * on whether {@code a} is absolute or relative) if there is none. + */ + public static PathFragment commonAncestor(PathFragment a, PathFragment b) { + while (a != null && !b.startsWith(a)) { + a = a.getParentDirectory(); + } + + return a; + } + /** + * Returns a path fragment from a given from-dir to a given to-path. May be + * either a short relative path "foo/bar", an up'n'over relative path + * "../../foo/bar" or an absolute path. + */ + public static PathFragment relativePath(Path fromDir, Path to) { + if (to.getFileSystem() != fromDir.getFileSystem()) { + throw new IllegalArgumentException("fromDir and to must be on the same FileSystem"); + } + + return relativePath(fromDir.asFragment(), to.asFragment()); + } + + /** + * Returns a path fragment from a given from-dir to a given to-path. + */ + public static PathFragment relativePath(PathFragment fromDir, PathFragment to) { + if (to.equals(fromDir)) { + return new PathFragment("."); // same dir, just return '.' + } + if (to.startsWith(fromDir)) { + return to.relativeTo(fromDir); // easy case--it's a descendant + } + PathFragment ancestor = commonAncestor(fromDir, to); + if (ancestor == null) { + return to; // no common ancestor, use 'to' + } + int levels = fromDir.relativeTo(ancestor).segmentCount(); + StringBuilder dotdots = new StringBuilder(); + for (int i = 0; i < levels; i++) { + dotdots.append("../"); + } + return new PathFragment(dotdots.toString()).getRelative(to.relativeTo(ancestor)); + } + + /** + * Returns the longest prefix from a given set of 'prefixes' that are + * contained in 'path'. I.e the closest ancestor directory containing path. + * Returns null if none found. + */ + public static PathFragment longestPathPrefix(PathFragment path, Set<PathFragment> prefixes) { + for (int i = path.segmentCount(); i >= 1; i--) { + PathFragment prefix = path.subFragment(0, i); + if (prefixes.contains(prefix)) { + return prefix; + } + } + return null; + } + + /** + * Removes the shortest suffix beginning with '.' from the basename of the + * filename string. If the basename contains no '.', the filename is returned + * unchanged. + * + * e.g. "foo/bar.x" -> "foo/bar" + */ + @ThreadSafe + public static String removeExtension(String filename) { + int lastDotIndex = filename.lastIndexOf('.'); + if (lastDotIndex == -1) { return filename; } + int lastSlashIndex = filename.lastIndexOf('/'); + if (lastSlashIndex > lastDotIndex) { + return filename; + } + return filename.substring(0, lastDotIndex); + } + + /** + * Removes the shortest suffix beginning with '.' from the basename of the + * PathFragment. If the basename contains no '.', the filename is returned + * unchanged. + * + * <p>e.g. "foo/bar.x" -> "foo/bar" + */ + @ThreadSafe + public static PathFragment removeExtension(PathFragment path) { + return path.replaceName(removeExtension(path.getBaseName())); + } + + /** + * Removes the shortest suffix beginning with '.' from the basename of the + * Path. If the basename contains no '.', the filename is returned + * unchanged. + * + * <p>e.g. "foo/bar.x" -> "foo/bar" + */ + @ThreadSafe + public static Path removeExtension(Path path) { + return path.getFileSystem().getPath(removeExtension(path.asFragment())); + } + + /** + * Returns a new {@code PathFragment} formed by replacing the extension of the + * last path segment of {@code path} with {@code newExtension}. Null is + * returned iff {@code path} has zero segments. + */ + public static PathFragment replaceExtension(PathFragment path, String newExtension) { + return path.replaceName(removeExtension(path.getBaseName()) + newExtension); + } + + /** + * Returns a new {@code PathFragment} formed by replacing the extension of the + * last path segment of {@code path} with {@code newExtension}. Null is + * returned iff {@code path} has zero segments or it doesn't end with {@code oldExtension}. + */ + public static PathFragment replaceExtension(PathFragment path, String newExtension, + String oldExtension) { + String base = path.getBaseName(); + if (!base.endsWith(oldExtension)) { + return null; + } + String newBase = base.substring(0, base.length() - oldExtension.length()) + newExtension; + return path.replaceName(newBase); + } + + /** + * Returns a new {@code Path} formed by replacing the extension of the + * last path segment of {@code path} with {@code newExtension}. Null is + * returned iff {@code path} has zero segments. + */ + public static Path replaceExtension(Path path, String newExtension) { + PathFragment fragment = replaceExtension(path.asFragment(), newExtension); + return fragment == null ? null : path.getFileSystem().getPath(fragment); + } + + /** + * Returns a new {@code PathFragment} formed by adding the extension to the last path segment of + * {@code path}. Null is returned if {@code path} has zero segments. + */ + public static PathFragment appendExtension(PathFragment path, String newExtension) { + return path.replaceName(path.getBaseName() + newExtension); + } + + /** + * Returns a new {@code PathFragment} formed by replacing the first, or all if + * {@code replaceAll} is true, {@code oldSegment} of {@code path} with {@code + * newSegment}. + */ + public static PathFragment replaceSegments(PathFragment path, + String oldSegment, String newSegment, boolean replaceAll) { + int count = path.segmentCount(); + for (int i = 0; i < count; i++) { + if (path.getSegment(i).equals(oldSegment)) { + path = new PathFragment(path.subFragment(0, i), + new PathFragment(newSegment), + path.subFragment(i+1, count)); + if (!replaceAll) { + return path; + } + } + } + return path; + } + + /** + * Returns a new {@code PathFragment} formed by appending the given string to the last path + * segment of {@code path} without removing the extension. Returns null if {@code path} + * has zero segments. + */ + public static PathFragment appendWithoutExtension(PathFragment path, String toAppend) { + return path.replaceName(appendWithoutExtension(path.getBaseName(), toAppend)); + } + + /** + * Given a string that represents a file with an extension separated by a '.' and a string + * to append, return a string in which {@code toAppend} has been appended to {@code name} + * before the last '.' character. If {@code name} does not include a '.', appends {@code + * toAppend} at the end. + * + * <p>For example, + * ("libfoo.jar", "-src") ==> "libfoo-src.jar" + * ("libfoo", "-src") ==> "libfoo-src" + */ + private static String appendWithoutExtension(String name, String toAppend) { + int dotIndex = name.lastIndexOf("."); + if (dotIndex > 0) { + String baseName = name.substring(0, dotIndex); + String extension = name.substring(dotIndex); + return baseName + toAppend + extension; + } else { + return name + toAppend; + } + } + + /**************************************************************************** + * FileSystem property functions. + */ + + /** + * Return the current working directory as expressed by the System property + * 'user.dir'. + */ + public static Path getWorkingDirectory(FileSystem fs) { + return fs.getPath(getWorkingDirectory()); + } + + /** + * Returns the current working directory as expressed by the System property + * 'user.dir'. This version does not require a {@link FileSystem}. + */ + public static PathFragment getWorkingDirectory() { + return new PathFragment(System.getProperty("user.dir", "/")); + } + + /**************************************************************************** + * Path FileSystem mutating operations. + */ + + /** + * "Touches" the file or directory specified by the path, following symbolic + * links. If it does not exist, it is created as an empty file; otherwise, the + * time of last access is updated to the current time. + * + * @throws IOException if there was an error while touching the file + */ + @ThreadSafe + public static void touchFile(Path path) throws IOException { + if (path.exists()) { + // -1L means "use the current time", and is ultimately implemented by + // utime(path, null), thereby using the kernel's clock, not the JVM's. + // (A previous implementation based on the JVM clock was found to be + // skewy.) + path.setLastModifiedTime(-1L); + } else { + createEmptyFile(path); + } + } + + /** + * Creates an empty regular file with the name of the current path, following + * symbolic links. + * + * @throws IOException if the file could not be created for any reason + * (including that there was already a file at that location) + */ + public static void createEmptyFile(Path path) throws IOException { + path.getOutputStream().close(); + } + + /** + * Creates or updates a symbolic link from 'link' to 'target'. Replaces + * existing symbolic links with target, and skips the link creation if it is + * already present. Will also create any missing ancestor directories of the + * link. This method is non-atomic + * + * <p>Note: this method will throw an IOException if there is an unequal + * non-symlink at link. + * + * @throws IOException if the creation of the symbolic link was unsuccessful + * for any reason. + */ + @ThreadSafe // but not atomic + public static void ensureSymbolicLink(Path link, Path target) throws IOException { + ensureSymbolicLink(link, target.asFragment()); + } + + /** + * Creates or updates a symbolic link from 'link' to 'target'. Replaces + * existing symbolic links with target, and skips the link creation if it is + * already present. Will also create any missing ancestor directories of the + * link. This method is non-atomic + * + * <p>Note: this method will throw an IOException if there is an unequal + * non-symlink at link. + * + * @throws IOException if the creation of the symbolic link was unsuccessful + * for any reason. + */ + @ThreadSafe // but not atomic + public static void ensureSymbolicLink(Path link, String target) throws IOException { + ensureSymbolicLink(link, new PathFragment(target)); + } + + /** + * Creates or updates a symbolic link from 'link' to 'target'. Replaces + * existing symbolic links with target, and skips the link creation if it is + * already present. Will also create any missing ancestor directories of the + * link. This method is non-atomic + * + * <p>Note: this method will throw an IOException if there is an unequal + * non-symlink at link. + * + * @throws IOException if the creation of the symbolic link was unsuccessful + * for any reason. + */ + @ThreadSafe // but not atomic + public static void ensureSymbolicLink(Path link, PathFragment target) throws IOException { + // TODO(bazel-team): (2009) consider adding the logic for recovering from the case when + // we have already created a parent directory symlink earlier. + try { + if (link.readSymbolicLink().equals(target)) { + return; // Do nothing if the link is already there. + } + } catch (IOException e) { // link missing or broken + /* fallthru and do the work below */ + } + if (link.isSymbolicLink()) { + link.delete(); // Remove the symlink since it is pointing somewhere else. + } else { + createDirectoryAndParents(link.getParentDirectory()); + } + try { + link.createSymbolicLink(target); + } catch (IOException e) { + // Only pass on exceptions caused by a true link creation failure. + if (!link.isSymbolicLink() || + !link.resolveSymbolicLinks().equals(link.getRelative(target))) { + throw e; + } + } + } + + private static ByteSource asByteSource(final Path path) { + return new ByteSource() { + @Override public InputStream openStream() throws IOException { + return path.getInputStream(); + } + }; + } + + private static ByteSink asByteSink(final Path path, final boolean append) { + return new ByteSink() { + @Override public OutputStream openStream() throws IOException { + return path.getOutputStream(append); + } + }; + } + + private static ByteSink asByteSink(final Path path) { + return asByteSink(path, false); + } + + /** + * Copies the file from location "from" to location "to", while overwriting a + * potentially existing "to". File's last modified time, executable and + * writable bits are also preserved. + * + * <p>If no error occurs, the method returns normally. If a parent directory does + * not exist, a FileNotFoundException is thrown. An IOException is thrown when + * other erroneous situations occur. (e.g. read errors) + */ + @ThreadSafe // but not atomic + public static void copyFile(Path from, Path to) throws IOException { + try { + to.delete(); + } catch (IOException e) { + throw new IOException("error copying file: " + + "couldn't delete destination: " + e.getMessage()); + } + asByteSource(from).copyTo(asByteSink(to)); + to.setLastModifiedTime(from.getLastModifiedTime()); // Preserve mtime. + if (!from.isWritable()) { + to.setWritable(false); // Make file read-only if original was read-only. + } + to.setExecutable(from.isExecutable()); // Copy executable bit. + } + + /** + * Copies a tool binary from one path to another, returning the target path. + * The directory of the target path must already exist. The target copy's time + * is set to match, as well as its read-only and executable flags. The + * operation is skipped if the target file has the same time and size as the + * source. + */ + public static Path copyTool(Path source, Path target) throws IOException { + FileStatus sourceStat = null; + FileStatus targetStat = target.statNullable(); + if (targetStat != null) { + // stat the source file only if we'll need the stat. + sourceStat = source.stat(Symlinks.FOLLOW); + } + if (targetStat == null || + targetStat.getLastModifiedTime() != sourceStat.getLastModifiedTime() || + targetStat.getSize() != sourceStat.getSize()) { + copyFile(source, target); + target.setWritable(source.isWritable()); + target.setExecutable(source.isExecutable()); + target.setLastModifiedTime(source.getLastModifiedTime()); + } + return target; + } + + /**************************************************************************** + * Directory tree operations. + */ + + /** + * Returns a new collection containing all of the paths below a given root + * path, for which the given predicate is true. Symbolic links are not + * followed, and may appear in the result. + * + * @throws IOException If the root does not denote a directory + */ + @ThreadSafe + public static Collection<Path> traverseTree(Path root, Predicate<? super Path> predicate) + throws IOException { + List<Path> paths = new ArrayList<>(); + traverseTree(paths, root, predicate); + return paths; + } + + /** + * Populates an existing Path List, adding all of the paths below a given root + * path for which the given predicate is true. Symbolic links are not + * followed, and may appear in the result. + * + * @throws IOException If the root does not denote a directory + */ + @ThreadSafe + public static void traverseTree(Collection<Path> paths, Path root, + Predicate<? super Path> predicate) throws IOException { + for (Path p : root.getDirectoryEntries()) { + if (predicate.apply(p)) { + paths.add(p); + } + if (p.isDirectory(Symlinks.NOFOLLOW)) { + traverseTree(paths, p, predicate); + } + } + } + + /** + * Deletes 'p', and everything recursively beneath it if it's a directory. + * Does not follow any symbolic links. + * + * @throws IOException if any file could not be removed. + */ + @ThreadSafe + public static void deleteTree(Path p) throws IOException { + deleteTreesBelow(p); + p.delete(); + } + + /** + * Deletes all dir trees recursively beneath 'dir' if it's a directory, + * nothing otherwise. Does not follow any symbolic links. + * + * @throws IOException if any file could not be removed. + */ + @ThreadSafe + public static void deleteTreesBelow(Path dir) throws IOException { + if (dir.isDirectory(Symlinks.NOFOLLOW)) { // real directories (not symlinks) + dir.setReadable(true); + dir.setWritable(true); + dir.setExecutable(true); + for (Path child : dir.getDirectoryEntries()) { + deleteTree(child); + } + } + } + + /** + * Delete all dir trees under a given 'dir' that don't start with one of a set + * of given 'prefixes'. Does not follow any symbolic links. + */ + @ThreadSafe + public static void deleteTreesBelowNotPrefixed(Path dir, String[] prefixes) throws IOException { + dirloop: + for (Path p : dir.getDirectoryEntries()) { + String name = p.getBaseName(); + for (int i = 0; i < prefixes.length; i++) { + if (name.startsWith(prefixes[i])) { + continue dirloop; + } + } + deleteTree(p); + } + } + + /** + * Copies all dir trees under a given 'from' dir to location 'to', while overwriting + * all files in the potentially existing 'to'. Does not follow any symbolic links, + * but copies them instead. + * + * <p>The source and the destination must be non-overlapping, otherwise an + * IllegalArgumentException will be thrown. This method cannot be used to copy + * a dir tree to a sub tree of itself. + * + * <p>If no error occurs, the method returns normally. If the given 'from' does + * not exist, a FileNotFoundException is thrown. An IOException is thrown when + * other erroneous situations occur. (e.g. read errors) + */ + @ThreadSafe + public static void copyTreesBelow(Path from , Path to) throws IOException { + if (to.startsWith(from)) { + throw new IllegalArgumentException(to + " is a subdirectory of " + from); + } + + Collection<Path> entries = from.getDirectoryEntries(); + for (Path entry : entries) { + if (entry.isDirectory(Symlinks.NOFOLLOW)) { + Path subDir = to.getChild(entry.getBaseName()); + subDir.createDirectory(); + copyTreesBelow(entry, subDir); + } else if (entry.isSymbolicLink()) { + Path newLink = to.getChild(entry.getBaseName()); + newLink.createSymbolicLink(entry.readSymbolicLink()); + } else { + Path newEntry = to.getChild(entry.getBaseName()); + copyFile(entry, newEntry); + } + } + } + + /** + * Attempts to create a directory with the name of the given path, creating + * ancestors as necessary. + * + * <p>Postcondition: completes normally iff {@code dir} denotes an existing + * directory (not necessarily canonical); completes abruptly otherwise. + * + * @return true if the directory was successfully created anew, false if it + * already existed (including the case where {@code dir} denotes a symlink + * to an existing directory) + * @throws IOException if the directory could not be created + */ + @ThreadSafe + public static boolean createDirectoryAndParents(Path dir) throws IOException { + // Optimised for minimal number of I/O calls. + + // Don't attempt to create the root directory. + if (dir.getParentDirectory() == null) { return false; } + + FileSystem filesystem = dir.getFileSystem(); + if (filesystem instanceof UnionFileSystem) { + // If using UnionFS, make sure that we do not traverse filesystem boundaries when creating + // parent directories by rehoming the path on the most specific filesystem. + FileSystem delegate = ((UnionFileSystem) filesystem).getDelegate(dir); + dir = delegate.getPath(dir.asFragment()); + } + + try { + return dir.createDirectory(); + } catch (IOException e) { + if (e.getMessage().endsWith(" (No such file or directory)")) { // ENOENT + createDirectoryAndParents(dir.getParentDirectory()); + return dir.createDirectory(); + } else if (e.getMessage().endsWith(" (File exists)") && dir.isDirectory()) { // EEXIST + return false; + } else { + throw e; // some other error (e.g. ENOTDIR, EACCES, etc.) + } + } + } + + /** + * Attempts to remove a relative chain of directories under a given base. + * Returns {@code true} if the removal was successful, and returns {@code + * false} if the removal fails because a directory was not empty. An + * {@link IOException} is thrown for any other errors. + */ + @ThreadSafe + public static boolean removeDirectoryAndParents(Path base, PathFragment toRemove) { + if (toRemove.isAbsolute()) { + return false; + } + try { + for (; toRemove.segmentCount() > 0; toRemove = toRemove.getParentDirectory()) { + Path p = base.getRelative(toRemove); + if (p.exists()) { + p.delete(); + } + } + } catch (IOException e) { + return false; + } + return true; + } + + /** + * Takes a map of directory fragments to root paths, and creates a symlink + * forest under an existing linkRoot to the corresponding source dirs or + * files. Symlink are made at the highest dir possible, linking files directly + * only when needed with nested packages. + */ + public static void plantLinkForest(ImmutableMap<PathFragment, Path> packageRootMap, Path linkRoot) + throws IOException { + // Create a sorted map of all dirs (packages and their ancestors) to sets of their roots. + // Packages come from exactly one root, but their shared ancestors may come from more. + // The map is maintained sorted lexicographically, so parents are before their children. + Map<PathFragment, Set<Path>> dirRootsMap = Maps.newTreeMap(); + for (Map.Entry<PathFragment, Path> entry : packageRootMap.entrySet()) { + PathFragment pkgDir = entry.getKey(); + Path pkgRoot = entry.getValue(); + for (int i = 1; i <= pkgDir.segmentCount(); i++) { + PathFragment dir = pkgDir.subFragment(0, i); + Set<Path> roots = dirRootsMap.get(dir); + if (roots == null) { + roots = Sets.newHashSet(); + dirRootsMap.put(dir, roots); + } + roots.add(pkgRoot); + } + } + // Now add in roots for all non-pkg dirs that are in between two packages, and missed above. + for (Map.Entry<PathFragment, Set<Path>> entry : dirRootsMap.entrySet()) { + PathFragment dir = entry.getKey(); + if (!packageRootMap.containsKey(dir)) { + PathFragment pkgDir = longestPathPrefix(dir, packageRootMap.keySet()); + if (pkgDir != null) { + entry.getValue().add(packageRootMap.get(pkgDir)); + } + } + } + // Create output dirs for all dirs that have more than one root and need to be split. + for (Map.Entry<PathFragment, Set<Path>> entry : dirRootsMap.entrySet()) { + PathFragment dir = entry.getKey(); + if (entry.getValue().size() > 1) { + if (LOG_FINER) { + LOG.finer("mkdir " + linkRoot.getRelative(dir)); + } + createDirectoryAndParents(linkRoot.getRelative(dir)); + } + } + // Make dir links for single rooted dirs. + for (Map.Entry<PathFragment, Set<Path>> entry : dirRootsMap.entrySet()) { + PathFragment dir = entry.getKey(); + Set<Path> roots = entry.getValue(); + // Simple case of one root for this dir. + if (roots.size() == 1) { + if (dir.segmentCount() > 1 && dirRootsMap.get(dir.getParentDirectory()).size() == 1) { + continue; // skip--an ancestor will link this one in from above + } + // This is the top-most dir that can be linked to a single root. Make it so. + Path root = roots.iterator().next(); // lone root in set + if (LOG_FINER) { + LOG.finer("ln -s " + root.getRelative(dir) + " " + linkRoot.getRelative(dir)); + } + linkRoot.getRelative(dir).createSymbolicLink(root.getRelative(dir)); + } + } + // Make links for dirs within packages, skip parent-only dirs. + for (Map.Entry<PathFragment, Set<Path>> entry : dirRootsMap.entrySet()) { + PathFragment dir = entry.getKey(); + if (entry.getValue().size() > 1) { + // If this dir is at or below a package dir, link in its contents. + PathFragment pkgDir = longestPathPrefix(dir, packageRootMap.keySet()); + if (pkgDir != null) { + Path root = packageRootMap.get(pkgDir); + try { + Path absdir = root.getRelative(dir); + if (absdir.isDirectory()) { + if (LOG_FINER) { + LOG.finer("ln -s " + absdir + "/* " + linkRoot.getRelative(dir) + "/"); + } + for (Path target : absdir.getDirectoryEntries()) { + PathFragment p = target.relativeTo(root); + if (!dirRootsMap.containsKey(p)) { + //LOG.finest("ln -s " + target + " " + linkRoot.getRelative(p)); + linkRoot.getRelative(p).createSymbolicLink(target); + } + } + } else { + LOG.fine("Symlink planting skipping dir '" + absdir + "'"); + } + } catch (IOException e) { + e.printStackTrace(); + } + // Otherwise its just an otherwise empty common parent dir. + } + } + } + } + + /**************************************************************************** + * Whole-file I/O utilities for characters and bytes. These convenience + * methods are not efficient and should not be used for large amounts of data! + */ + + private static char[] convertFromLatin1(byte[] content) { + char[] latin1 = new char[content.length]; + for (int i = 0; i < latin1.length; i++) { // yeah, latin1 is this easy! :-) + latin1[i] = (char) (0xff & content[i]); + } + return latin1; + } + + /** + * Writes lines to file using ISO-8859-1 encoding (isolatin1). + */ + @ThreadSafe // but not atomic + public static void writeIsoLatin1(Path file, String... lines) throws IOException { + writeLinesAs(file, ISO_8859_1, lines); + } + + /** + * Append lines to file using ISO-8859-1 encoding (isolatin1). + */ + @ThreadSafe // but not atomic + public static void appendIsoLatin1(Path file, String... lines) throws IOException { + appendLinesAs(file, ISO_8859_1, lines); + } + + /** + * Writes the specified String as ISO-8859-1 (latin1) encoded bytes to the + * file. Follows symbolic links. + * + * @throws IOException if there was an error + */ + public static void writeContentAsLatin1(Path outputFile, String content) throws IOException { + writeContent(outputFile, ISO_8859_1, content); + } + + /** + * Writes the specified String using the specified encoding to the file. + * Follows symbolic links. + * + * @throws IOException if there was an error + */ + public static void writeContent(Path outputFile, Charset charset, String content) + throws IOException { + asByteSink(outputFile).asCharSink(charset).write(content); + } + + /** + * Writes lines to file using the given encoding, ending every line with a + * line break '\n' character. + */ + @ThreadSafe // but not atomic + public static void writeLinesAs(Path file, Charset charset, String... lines) + throws IOException { + writeLinesAs(file, charset, Arrays.asList(lines)); + } + + /** + * Appends lines to file using the given encoding, ending every line with a + * line break '\n' character. + */ + @ThreadSafe // but not atomic + public static void appendLinesAs(Path file, Charset charset, String... lines) + throws IOException { + appendLinesAs(file, charset, Arrays.asList(lines)); + } + + /** + * Writes lines to file using the given encoding, ending every line with a + * line break '\n' character. + */ + @ThreadSafe // but not atomic + public static void writeLinesAs(Path file, Charset charset, Iterable<String> lines) + throws IOException { + createDirectoryAndParents(file.getParentDirectory()); + asByteSink(file).asCharSink(charset).writeLines(lines); + } + + /** + * Appends lines to file using the given encoding, ending every line with a + * line break '\n' character. + */ + @ThreadSafe // but not atomic + public static void appendLinesAs(Path file, Charset charset, Iterable<String> lines) + throws IOException { + createDirectoryAndParents(file.getParentDirectory()); + asByteSink(file, true).asCharSink(charset).writeLines(lines); + } + + /** + * Writes the specified byte array to the output file. Follows symbolic links. + * + * @throws IOException if there was an error + */ + public static void writeContent(Path outputFile, byte[] content) throws IOException { + asByteSink(outputFile).write(content); + } + + /** + * Returns the entirety of the specified input stream and returns it as a char + * array, decoding characters using ISO-8859-1 (Latin1). + * + * @throws IOException if there was an error + */ + public static char[] readContentAsLatin1(InputStream in) throws IOException { + return convertFromLatin1(ByteStreams.toByteArray(in)); + } + + /** + * Returns the entirety of the specified file and returns it as a char array, + * decoding characters using ISO-8859-1 (Latin1). + * + * @throws IOException if there was an error + */ + public static char[] readContentAsLatin1(Path inputFile) throws IOException { + return convertFromLatin1(readContent(inputFile)); + } + + /** + * Returns an iterable that allows iterating over ISO-8859-1 (Latin1) text + * file contents line by line. If the file ends in a line break, the iterator + * will return an empty string as the last element. + * + * @throws IOException if there was an error + */ + public static Iterable<String> iterateLinesAsLatin1(Path inputFile) throws IOException { + return asByteSource(inputFile).asCharSource(ISO_8859_1).readLines(); + } + + /** + * Returns the entirety of the specified file and returns it as a byte array. + * + * @throws IOException if there was an error + */ + public static byte[] readContent(Path inputFile) throws IOException { + return asByteSource(inputFile).read(); + } + + /** + * Reads at most {@code limit} bytes from {@code inputFile} and returns it as a byte array. + * + * @throws IOException if there was an error. + */ + public static byte[] readContentWithLimit(Path inputFile, int limit) throws IOException { + Preconditions.checkArgument(limit >= 0, "limit needs to be >=0, but it is %s", limit); + ByteSource byteSource = asByteSource(inputFile); + byte[] buffer = new byte[limit]; + try (InputStream inputStream = byteSource.openBufferedStream()) { + int read = ByteStreams.read(inputStream, buffer, 0, limit); + return Arrays.copyOf(buffer, read); + } + } + + /** + * Dumps diagnostic information about the specified filesystem to {@code out}. + * This is the implementation of the filesystem part of the 'blaze dump' + * command. It lives here, rather than in DumpCommand, because it requires + * privileged access to members of this package. + * + * <p>Its results are unspecified and MUST NOT be interpreted programmatically. + */ + public static void dump(FileSystem fs, final PrintStream out) { + if (!(fs instanceof UnixFileSystem)) { + out.println(" Not a UnixFileSystem."); + return; + } + + // Unfortunately there's no "letrec" for anonymous functions so we have to + // (a) name the function, (b) put it in a box and (c) use List not array + // because of the generic type. *sigh*. + final List<Predicate<Path>> dumpFunction = new ArrayList<>(); + dumpFunction.add(new Predicate<Path>() { + @Override + public boolean apply(Path child) { + Path path = child; + out.println(" " + path + " (" + path.toDebugString() + ")"); + path.applyToChildren(dumpFunction.get(0)); + return false; + } + }); + + fs.getRootDirectory().applyToChildren(dumpFunction.get(0)); + } + + /** + * Returns the type of the file system path belongs to. + */ + public static String getFileSystem(Path path) { + return path.getFileSystem().getFileSystemType(path); + } + + /** + * Returns whether the given path starts with any of the paths in the given + * list of prefixes. + */ + public static boolean startsWithAny(Path path, Iterable<Path> prefixes) { + for (Path prefix : prefixes) { + if (path.startsWith(prefix)) { + return true; + } + } + return false; + } + + /** + * Returns whether the given path starts with any of the paths in the given + * list of prefixes. + */ + public static boolean startsWithAny(PathFragment path, Iterable<PathFragment> prefixes) { + for (PathFragment prefix : prefixes) { + if (path.startsWith(prefix)) { + return true; + } + } + return false; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileSystems.java b/src/main/java/com/google/devtools/build/lib/vfs/FileSystems.java new file mode 100644 index 0000000..1e7aaae --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/FileSystems.java
@@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.vfs; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +/** + * This static file system singleton manages access to a single default + * {@link FileSystem} instance created within the methods of this class. + */ +@ThreadSafe +@Deprecated // Instantiate and inject FileSystem instances directly, or use + // com.google.devtools.build.lib.vfs.util.FileSystems in tests. +public final class FileSystems { + + private FileSystems() {} + + private static FileSystem defaultFileSystem; + + /** + * Initializes the default {@link FileSystem} instance as a platform native + * (Unix) file system, creating one iff needed, and returns the instance. + * + * <p>This method is idempotent as long as the initialization is of the same + * type (Native/JavaIo/Union). + */ + public static synchronized FileSystem initDefaultAsNative() { + if (!(defaultFileSystem instanceof UnixFileSystem)) { + defaultFileSystem = new UnixFileSystem(); + } + return defaultFileSystem; + } + + /** + * Initializes the default {@link FileSystem} instance as a java.io.File + * file system, creating one iff needed, and returns the instance. + * + * <p>This method is idempotent as long as the initialization is of the same + * type (Native/JavaIo/Union). + */ + public static synchronized FileSystem initDefaultAsJavaIo() { + if (!(defaultFileSystem instanceof JavaIoFileSystem)) { + defaultFileSystem = new JavaIoFileSystem(); + } + return defaultFileSystem; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/IORuntimeException.java b/src/main/java/com/google/devtools/build/lib/vfs/IORuntimeException.java new file mode 100644 index 0000000..2769fe9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/IORuntimeException.java
@@ -0,0 +1,78 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// All Rights Reserved. + +package com.google.devtools.build.lib.vfs; + +import java.io.IOException; + +/** + * Signals that an I/O exception of some sort has occurred. Contrary to + * <code>java.io.IOException</code>, this class is a subclass of + * <code>RuntimeException</code>, which allows you to signal an I/O problem + * without polluting the callers. For details on why checked exceptions is bad, + * try searching for "java checked exception mistake" on Google. + */ +public class IORuntimeException extends RuntimeException { + /** + * Constructs a new IORuntimeException with null as its detail message. + */ + public IORuntimeException() { + super(); + } + + /** + * Constructs a new IORuntimeException with the specified detail message. + */ + public IORuntimeException(String message) { + super(message); + } + + /** + * Constructs a new IORuntimeException with the specified detail message and + * cause. + * + * @param message the detail message, which is saved for later retrieval by + * the <code>Throwable.getMessage()</code> method. + * @param cause the cause (which is saved for later retrieval by the + * <code>Throwable.getCause()</code> method). (A null value is + * permitted, and indicates that the cause is nonexistent or unknown.) + */ + public IORuntimeException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new IORuntimeException as a wrapper on a root cause + */ + public IORuntimeException(Throwable cause) { + super(cause); + } + + /** + * @return the actual IOException that caused this exception, or null if it + * was not caused by an IOException. Call <code>getCause()</code> + * instead if it was caused by other types of exceptions. + */ + public IOException getCauseIOException() { + Throwable cause = getCause(); + if (cause instanceof IOException) { + return (IOException) cause; + } else { + return null; + } + } + + private static final long serialVersionUID = 1L; +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/JavaIoFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/JavaIoFileSystem.java new file mode 100644 index 0000000..08e67f7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/JavaIoFileSystem.java
@@ -0,0 +1,486 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.unix.FileAccessException; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Collection; + +/** + * A FileSystem that does not use any JNI and hence, does not require a shared library be present at + * execution. + * + * <p>Note: Blaze profiler tasks are defined on the system call level - thus we do not distinguish + * (from profiling perspective) between different methods on this class that end up doing stat() + * system call - they all are associated with the VFS_STAT task. + */ +@ThreadSafe +public class JavaIoFileSystem extends AbstractFileSystem { + private static final LinkOption[] NO_LINK_OPTION = new LinkOption[0]; + // This isn't generally safe; we rely on the file system APIs not modifying the array. + private static final LinkOption[] NOFOLLOW_LINKS_OPTION = + new LinkOption[] { LinkOption.NOFOLLOW_LINKS }; + + protected static final String ERR_IS_DIRECTORY = " (Is a directory)"; + protected static final String ERR_DIRECTORY_NOT_EMPTY = " (Directory not empty)"; + protected static final String ERR_FILE_EXISTS = " (File exists)"; + protected static final String ERR_NO_SUCH_FILE_OR_DIR = " (No such file or directory)"; + protected static final String ERR_NOT_A_DIRECTORY = " (Not a directory)"; + + protected File getIoFile(Path path) { + return new File(path.toString()); + } + + private LinkOption[] linkOpts(boolean followSymlinks) { + return followSymlinks ? NO_LINK_OPTION : NOFOLLOW_LINKS_OPTION; + } + + @Override + protected Collection<Path> getDirectoryEntries(Path path) throws IOException { + File file = getIoFile(path); + String[] entries = null; + long startTime = Profiler.nanoTimeMaybe(); + try { + entries = file.list(); + if (entries == null) { + if (file.exists()) { + throw new IOException(path + ERR_NOT_A_DIRECTORY); + } else { + throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR); + } + } + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_DIR, file.getPath()); + } + Collection<Path> result = new ArrayList<>(entries.length); + for (String entry : entries) { + if (!entry.equals(".") && !entry.equals("..")) { + result.add(path.getChild(entry)); + } + } + return result; + } + + @Override + protected boolean exists(Path path, boolean followSymlinks) { + File file = getIoFile(path); + long startTime = Profiler.nanoTimeMaybe(); + try { + return Files.exists(file.toPath(), linkOpts(followSymlinks)); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, path.toString()); + } + } + + @Override + protected boolean isDirectory(Path path, boolean followSymlinks) { + File file = getIoFile(path); + long startTime = Profiler.nanoTimeMaybe(); + try { + if (!followSymlinks && fileIsSymbolicLink(file)) { + return false; + } + return file.isDirectory(); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, path.toString()); + } + } + + @Override + protected boolean isFile(Path path, boolean followSymlinks) { + File file = getIoFile(path); + long startTime = Profiler.nanoTimeMaybe(); + try { + if (!followSymlinks && fileIsSymbolicLink(file)) { + return false; + } + return file.isFile(); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, path.toString()); + } + } + + @Override + protected boolean isReadable(Path path) throws IOException { + File file = getIoFile(path); + long startTime = Profiler.nanoTimeMaybe(); + try { + if (!file.exists()) { + throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR); + } + return file.canRead(); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, file.getPath()); + } + } + + @Override + protected boolean isWritable(Path path) throws IOException { + File file = getIoFile(path); + long startTime = Profiler.nanoTimeMaybe(); + try { + if (!file.exists()) { + if (linkExists(file)) { + throw new IOException(path + ERR_PERMISSION_DENIED); + } else { + throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR); + } + } + return file.canWrite(); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, file.getPath()); + } + } + + @Override + protected boolean isExecutable(Path path) throws IOException { + File file = getIoFile(path); + long startTime = Profiler.nanoTimeMaybe(); + try { + if (!file.exists()) { + throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR); + } + return file.canExecute(); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, file.getPath()); + } + } + + @Override + protected void setReadable(Path path, boolean readable) throws IOException { + File file = getIoFile(path); + if (!file.exists()) { + throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR); + } + file.setReadable(readable); + } + + @Override + protected void setWritable(Path path, boolean writable) throws IOException { + File file = getIoFile(path); + if (!file.exists()) { + throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR); + } + file.setWritable(writable); + } + + @Override + protected void setExecutable(Path path, boolean executable) throws IOException { + File file = getIoFile(path); + if (!file.exists()) { + throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR); + } + file.setExecutable(executable); + } + + @Override + public boolean supportsModifications() { + return true; + } + + @Override + public boolean supportsSymbolicLinks() { + return true; + } + + @Override + protected boolean createDirectory(Path path) throws IOException { + + // We always synchronize on the current path before doing it on the parent path and file system + // path structure ensures that this locking order will never be reversed. + // When refactoring, check that subclasses still work as expected and there can be no + // deadlocks. + synchronized (path) { + File file = getIoFile(path); + if (file.mkdir()) { + return true; + } + + // We will be checking the state of the parent path as well. Synchronize on it before + // attempting anything. + Path parentDirectory = path.getParentDirectory(); + synchronized (parentDirectory) { + if (fileIsSymbolicLink(file)) { + throw new IOException(path + ERR_FILE_EXISTS); + } + if (file.isDirectory()) { + return false; // directory already existed + } else if (file.exists()) { + throw new IOException(path + ERR_FILE_EXISTS); + } else if (!file.getParentFile().exists()) { + throw new FileNotFoundException(path.getParentDirectory() + ERR_NO_SUCH_FILE_OR_DIR); + } + // Parent directory apparently exists - try to create our directory again - protecting + // against the case where parent directory would be created right before us obtaining + // synchronization lock. + if (file.mkdir()) { + return true; // Everything is fine finally. + } else if (!file.getParentFile().canWrite()) { + throw new FileAccessException(path + ERR_PERMISSION_DENIED); + } else { + // Parent exists, is writable, yet we can't create our directory. + throw new FileNotFoundException(path.getParentDirectory() + ERR_NOT_A_DIRECTORY); + } + } + } + } + + private boolean linkExists(File file) { + String shortName = file.getName(); + File parentFile = file.getParentFile(); + if (parentFile == null) { + return false; + } + String[] filenames = parentFile.list(); + if (filenames == null) { + return false; + } + for (String name : filenames) { + if (name.equals(shortName)) { + return true; + } + } + return false; + } + + @Override + protected void createSymbolicLink(Path linkPath, PathFragment targetFragment) + throws IOException { + File file = getIoFile(linkPath); + try { + Files.createSymbolicLink(file.toPath(), new File(targetFragment.getPathString()).toPath()); + } catch (java.nio.file.FileAlreadyExistsException e) { + throw new IOException(linkPath + ERR_FILE_EXISTS); + } catch (java.nio.file.AccessDeniedException e) { + throw new IOException(linkPath + ERR_PERMISSION_DENIED); + } catch (java.nio.file.NoSuchFileException e) { + throw new FileNotFoundException(linkPath + ERR_NO_SUCH_FILE_OR_DIR); + } + } + + @Override + protected PathFragment readSymbolicLink(Path path) throws IOException { + File file = getIoFile(path); + long startTime = Profiler.nanoTimeMaybe(); + try { + String link = Files.readSymbolicLink(file.toPath()).toString(); + return new PathFragment(link); + } catch (java.nio.file.NotLinkException e) { + throw new NotASymlinkException(path); + } catch (java.nio.file.NoSuchFileException e) { + throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_LINK, file.getPath()); + } + } + + @Override + protected void renameTo(Path sourcePath, Path targetPath) throws IOException { + synchronized (sourcePath) { + File sourceFile = getIoFile(sourcePath); + File targetFile = getIoFile(targetPath); + if (!sourceFile.renameTo(targetFile)) { + if (!sourceFile.exists()) { + throw new FileNotFoundException(sourcePath + ERR_NO_SUCH_FILE_OR_DIR); + } + if (targetFile.exists()) { + if (targetFile.isDirectory() && targetFile.list().length > 0) { + throw new IOException(targetPath + ERR_DIRECTORY_NOT_EMPTY); + } else if (sourceFile.isDirectory() && targetFile.isFile()) { + throw new IOException(sourcePath + " -> " + targetPath + ERR_NOT_A_DIRECTORY); + } else if (sourceFile.isFile() && targetFile.isDirectory()) { + throw new IOException(sourcePath + " -> " + targetPath + ERR_IS_DIRECTORY); + } else { + throw new IOException(sourcePath + " -> " + targetPath + ERR_PERMISSION_DENIED); + } + } else { + throw new FileAccessException(sourcePath + " -> " + targetPath + ERR_PERMISSION_DENIED); + } + } + } + } + + @Override + protected long getFileSize(Path path, boolean followSymlinks) throws IOException { + long startTime = Profiler.nanoTimeMaybe(); + try { + return stat(path, followSymlinks).getSize(); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, path); + } + } + + @Override + protected boolean delete(Path path) throws IOException { + File file = getIoFile(path); + long startTime = Profiler.nanoTimeMaybe(); + synchronized (path) { + try { + if (file.delete()) { + return true; + } + if (file.exists()) { + if (file.isDirectory() && file.list().length > 0) { + throw new IOException(path + ERR_DIRECTORY_NOT_EMPTY); + } else { + throw new IOException(path + ERR_PERMISSION_DENIED); + } + } + return false; + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_DELETE, file.getPath()); + } + } + } + + @Override + protected long getLastModifiedTime(Path path, boolean followSymlinks) throws IOException { + File file = getIoFile(path); + long startTime = Profiler.nanoTimeMaybe(); + try { + return stat(path, followSymlinks).getLastModifiedTime(); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, file.getPath()); + } + } + + @Override + protected boolean isSymbolicLink(Path path) { + File file = getIoFile(path); + long startTime = Profiler.nanoTimeMaybe(); + try { + return fileIsSymbolicLink(file); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, file.getPath()); + } + } + + private boolean fileIsSymbolicLink(File file) { + return Files.isSymbolicLink(file.toPath()); + } + + @Override + protected void setLastModifiedTime(Path path, long newTime) throws IOException { + File file = getIoFile(path); + if (!file.setLastModified(newTime)) { + if (!file.exists()) { + throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR); + } else if (!file.getParentFile().canWrite()) { + throw new FileAccessException(path.getParentDirectory() + ERR_PERMISSION_DENIED); + } else { + throw new FileAccessException(path + ERR_PERMISSION_DENIED); + } + } + } + + @Override + protected byte[] getMD5Digest(Path path) throws IOException { + String name = path.toString(); + long startTime = Profiler.nanoTimeMaybe(); + try { + return super.getMD5Digest(path); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_MD5, name); + } + } + + /** + * Returns the status of a file. See {@link Path#stat(Symlinks)} for + * specification. + * + * <p>The default implementation of this method is a "lazy" one, based on + * other accessor methods such as {@link #isFile}, etc. Subclasses may provide + * more efficient specializations. However, we still try to follow Unix-like + * semantics of failing fast in case of non-existent files (or in case of + * permission issues). + */ + @Override + protected FileStatus stat(final Path path, final boolean followSymlinks) throws IOException { + File file = getIoFile(path); + final BasicFileAttributes attributes; + try { + attributes = Files.readAttributes( + file.toPath(), BasicFileAttributes.class, linkOpts(followSymlinks)); + } catch (java.nio.file.FileSystemException e) { + throw new FileNotFoundException(path + ERR_NO_SUCH_FILE_OR_DIR); + } + FileStatus status = new FileStatus() { + @Override + public boolean isFile() { + return attributes.isRegularFile(); + } + + @Override + public boolean isDirectory() { + return attributes.isDirectory(); + } + + @Override + public boolean isSymbolicLink() { + return attributes.isSymbolicLink(); + } + + @Override + public long getSize() throws IOException { + return attributes.size(); + } + + @Override + public long getLastModifiedTime() throws IOException { + return attributes.lastModifiedTime().toMillis(); + } + + @Override + public long getLastChangeTime() { + // This is the best we can do with Java NIO... + return attributes.lastModifiedTime().toMillis(); + } + + @Override + public long getNodeId() { + // TODO(bazel-team): Consider making use of attributes.fileKey(). + return -1; + } + }; + + return status; + } + + @Override + protected FileStatus statIfFound(Path path, boolean followSymlinks) { + try { + return stat(path, followSymlinks); + } catch (FileNotFoundException e) { + // JavaIoFileSystem#stat (incorrectly) only throws FileNotFoundException (because it calls + // #getLastModifiedTime, which can only throw a FileNotFoundException), so we always hit this + // codepath. Thus, this method will incorrectly not throw an exception for some filesystem + // errors. + return null; + } catch (IOException e) { + // If this codepath is ever hit, then this method should be rewritten to properly distinguish + // between not-found exceptions and others. + throw new IllegalStateException(e); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/ModifiedFileSet.java b/src/main/java/com/google/devtools/build/lib/vfs/ModifiedFileSet.java new file mode 100644 index 0000000..3d6a638 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/ModifiedFileSet.java
@@ -0,0 +1,126 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.common.collect.ImmutableSet; + +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * An immutable set of modified source files. The scope of these files is context-dependent; in some + * uses this may mean information about all files in the client, while in other uses this may mean + * information about some specific subset of files. {@link #EVERYTHING_MODIFIED} can be used to + * indicate that all files of interest have been modified. + */ +public final class ModifiedFileSet { + + public static final ModifiedFileSet EVERYTHING_MODIFIED = new ModifiedFileSet(null); + public static final ModifiedFileSet NOTHING_MODIFIED = new ModifiedFileSet( + ImmutableSet.<PathFragment>of()); + + @Nullable private final ImmutableSet<PathFragment> modified; + + /** + * Whether all files of interest should be treated as potentially modified. + */ + public boolean treatEverythingAsModified() { + return modified == null; + } + + /** + * The set of files of interest that were modified. + * + * @throws IllegalStateException if {@link #treatEverythingAsModified} returns true. + */ + public ImmutableSet<PathFragment> modifiedSourceFiles() { + if (treatEverythingAsModified()) { + throw new IllegalStateException(); + } + return modified; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ModifiedFileSet)) { + return false; + } + ModifiedFileSet other = (ModifiedFileSet) o; + return Objects.equals(modified, other.modified); + } + + @Override + public int hashCode() { + return Objects.hash(modified); + } + + @Override + public String toString() { + if (this == EVERYTHING_MODIFIED) { + return "EVERYTHING_MODIFIED"; + } else if (this == NOTHING_MODIFIED) { + return "NOTHING_MODIFIED"; + } else { + return modified.toString(); + } + } + + private ModifiedFileSet(ImmutableSet<PathFragment> modified) { + this.modified = modified; + } + + /** + * The builder for {@link ModifiedFileSet}. + */ + public static class Builder { + private final ImmutableSet.Builder<PathFragment> setBuilder = + ImmutableSet.<PathFragment>builder(); + + public ModifiedFileSet build() { + ImmutableSet<PathFragment> modified = setBuilder.build(); + return modified.isEmpty() ? NOTHING_MODIFIED : new ModifiedFileSet(modified); + } + + public Builder modify(PathFragment pathFragment) { + setBuilder.add(pathFragment); + return this; + } + + public Builder modifyAll(Iterable<PathFragment> pathFragments) { + setBuilder.addAll(pathFragments); + return this; + } + } + + public static Builder builder() { + return new Builder(); + } + + public static ModifiedFileSet union(ModifiedFileSet mfs1, ModifiedFileSet mfs2) { + if (mfs1.treatEverythingAsModified() || mfs2.treatEverythingAsModified()) { + return ModifiedFileSet.EVERYTHING_MODIFIED; + } + if (mfs1.equals(ModifiedFileSet.NOTHING_MODIFIED)) { + return mfs2; + } + if (mfs2.equals(ModifiedFileSet.NOTHING_MODIFIED)) { + return mfs1; + } + return ModifiedFileSet.builder() + .modifyAll(mfs1.modifiedSourceFiles()) + .modifyAll(mfs2.modifiedSourceFiles()) + .build(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Path.java b/src/main/java/com/google/devtools/build/lib/vfs/Path.java new file mode 100644 index 0000000..de222fe --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/Path.java
@@ -0,0 +1,1099 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.StringCanonicalizer; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.io.Serializable; +import java.lang.ref.Reference; +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; +import java.util.Arrays; +import java.util.Collection; +import java.util.IdentityHashMap; +import java.util.Objects; + +/** + * <p>Instances of this class represent pathnames, forming a tree + * structure to implement sharing of common prefixes (parent directory names). + * A node in these trees is something like foo, bar, .., ., or /. If the + * instance is not a root path, it will have a parent path. A path can also + * have children, which are indexed by name in a map. + * + * <p>There is some limited support for Windows-style paths. Most importantly, drive identifiers + * in front of a path (c:/abc) are supported. However, Windows-style backslash separators + * (C:\\foo\\bar) and drive-relative paths ("C:foo") are explicitly not supported, same with + * advanced features like \\\\network\\paths and \\\\?\\unc\\paths. + * + * <p>{@link FileSystem} implementations maintain pointers into this graph. + */ +@ThreadSafe +public class Path implements Comparable<Path>, Serializable { + + private static FileSystem fileSystemForSerialization; + + /** + * We need to specify used FileSystem. In this case we can save memory during the serialization. + */ + public static void setFileSystemForSerialization(FileSystem fileSystem) { + fileSystemForSerialization = fileSystem; + } + + /** + * Returns FileSystem that we are using. + */ + public static FileSystem getFileSystemForSerialization() { + return fileSystemForSerialization; + } + + // These are basically final, but can't be marked as such in order to support serialization. + private FileSystem fileSystem; + private String name; + private Path parent; + private int depth; + private int hashCode; + + private static final ReferenceQueue<Path> REFERENCE_QUEUE = new ReferenceQueue<>(); + + private static class PathWeakReferenceForCleanup extends WeakReference<Path> { + final Path parent; + final String baseName; + + PathWeakReferenceForCleanup(Path referent, ReferenceQueue<Path> referenceQueue) { + super(referent, referenceQueue); + parent = referent.getParentDirectory(); + baseName = referent.getBaseName(); + } + } + + private static final Thread PATH_CHILD_CACHE_CLEANUP_THREAD = new Thread("Path cache cleanup") { + @Override + public void run() { + while (true) { + try { + PathWeakReferenceForCleanup ref = (PathWeakReferenceForCleanup) REFERENCE_QUEUE.remove(); + Path parent = ref.parent; + synchronized (parent) { + // It's possible that since this reference was enqueued for deletion, the Path was + // recreated with a new entry in the map. We definitely shouldn't delete that entry, so + // check to make sure they're the same. + Reference<Path> currentRef = ref.parent.children.get(ref.baseName); + if (currentRef == ref) { + ref.parent.children.remove(ref.baseName); + } + } + } catch (InterruptedException e) { + // Ignored. + } + } + } + }; + + static { + PATH_CHILD_CACHE_CLEANUP_THREAD.setDaemon(true); + PATH_CHILD_CACHE_CLEANUP_THREAD.start(); + } + + /** + * A mapping from a child file name to the {@link Path} representing it. + * + * <p>File names must be a single path segment. The strings must be + * canonical. We use IdentityHashMap instead of HashMap for reasons of space + * efficiency: instances are smaller by a single word. Also, since all path + * segments are interned, the universe of Paths holds a minimal number of + * references to strings. (It's doubtful that there's any time gain from use + * of an IdentityHashMap, since the time saved by avoiding string equality + * tests during hash lookups is probably equal to the time spent eagerly + * interning strings, unless the collision rate is high.) + * + * <p>The Paths are stored as weak references to ensure that a live + * Path for a directory does not hold a strong reference to all of its + * descendants, which would prevent collection of paths we never intend to + * use again. Stale references in the map must be treated as absent. + * + * <p>A Path may be recycled once there is no Path that refers to it or + * to one of its descendants. This means that any data stored in the + * Path instance by one of its subclasses must be recoverable: it's ok to + * store data in Paths as an optimization, but there must be another + * source for that data in case the Path is recycled. + * + * <p>We intentionally avoid using the existing library classes for reasons of + * space efficiency: while ConcurrentHashMap would reduce our locking + * overhead, and ReferenceMap would simplify the code a little, both of those + * classes have much higher per-instance overheads than IdentityHashMap. + * + * <p>The Path object must be synchronized while children is being + * accessed. + */ + private IdentityHashMap<String, Reference<Path>> children; + + /** + * Create a path instance. Should only be called by {@link #createChildPath}. + * + * @param name the name of this path; it must be canonicalized with {@link + * StringCanonicalizer#intern} + * @param parent this path's parent + */ + protected Path(FileSystem fileSystem, String name, Path parent) { + this.fileSystem = fileSystem; + this.name = name; + this.parent = parent; + this.depth = parent == null ? 0 : parent.depth + 1; + this.hashCode = Objects.hash(parent, name); + } + + /** + * Create the root path. Should only be called by + * {@link FileSystem#createRootPath()}. + */ + protected Path(FileSystem fileSystem) { + this(fileSystem, StringCanonicalizer.intern("/"), null); + } + + private void writeObject(ObjectOutputStream out) throws IOException { + Preconditions.checkState(fileSystem == fileSystemForSerialization, fileSystem); + out.writeUTF(getPathString()); + } + + private void readObject(ObjectInputStream in) throws IOException { + fileSystem = fileSystemForSerialization; + String p = in.readUTF(); + PathFragment pf = new PathFragment(p); + PathFragment parentDir = pf.getParentDirectory(); + if (parentDir == null) { + this.name = "/"; + this.parent = null; + this.depth = 0; + } else { + this.name = pf.getBaseName(); + this.parent = fileSystem.getPath(parentDir); + this.depth = this.parent.depth + 1; + } + this.hashCode = Objects.hash(parent, name); + } + + /** + * Returns the filesystem instance to which this path belongs. + */ + public FileSystem getFileSystem() { + return fileSystem; + } + + public boolean isRootDirectory() { + return parent == null; + } + + protected Path createChildPath(String childName) { + return new Path(fileSystem, childName, this); + } + + /** + * Returns the child path named name, or creates such a path (and caches it) + * if it doesn't already exist. + */ + private Path getCachedChildPath(String childName) { + // Don't hold the lock for the interning operation. It increases lock contention. + childName = StringCanonicalizer.intern(childName); + synchronized(this) { + if (children == null) { + // 66% of Paths have size == 1, 80% <= 2 + children = new IdentityHashMap<String, Reference<Path>>(1); + } + Reference<Path> childRef = children.get(childName); + Path child; + if (childRef == null || (child = childRef.get()) == null) { + child = createChildPath(childName); + children.put(childName, new PathWeakReferenceForCleanup(child, REFERENCE_QUEUE)); + } + return child; + } + } + + /** + * Applies the specified function to each {@link Path} that is an existing direct + * descendant of this one. The Predicate is evaluated only for its + * side-effects. + * + * <p>This function exists to hide the "children" field, whose complex + * synchronization and identity requirements are too unsafe to be exposed to + * subclasses. For example, the "children" field must be synchronized for + * the duration of any iteration over it; it may be null; and references + * within it may be stale, and must be ignored. + */ + protected synchronized void applyToChildren(Predicate<Path> function) { + if (children != null) { + for (Reference<Path> childRef : children.values()) { + Path child = childRef.get(); + if (child != null) { + function.apply(child); + } + } + } + } + + /** + * Returns whether this path is recursively "under" {@code prefix} - that is, + * whether {@code path} is a prefix of this path. + * + * <p>This method returns {@code true} when called with this path itself. This + * method acts independently of the existence of files or folders. + * + * @param prefix a path which may or may not be a prefix of this path + */ + public boolean startsWith(Path prefix) { + Path n = this; + for (int i = 0, len = depth - prefix.depth; i < len; i++) { + n = n.getParentDirectory(); + } + return prefix.equals(n); + } + + /** + * Computes a string representation of this path, and writes it to the + * given string builder. Only called locally with a new instance. + */ + private void buildPathString(StringBuilder result) { + if (isRootDirectory()) { + result.append('/'); + } else { + if (parent.isWindowsVolumeName()) { + result.append(parent.name); + } else { + parent.buildPathString(result); + } + if (!parent.isRootDirectory()) { + result.append('/'); + } + result.append(name); + } + } + + /** + * Returns true if the current path represents a Windows volume name (such as "c:" or "d:"). + * + * <p>Paths such as '\\\\vol\\foo' are not supported. + */ + private boolean isWindowsVolumeName() { + return OS.getCurrent() == OS.WINDOWS + && parent != null && parent.isRootDirectory() && name.length() == 2 + && PathFragment.getWindowsDriveLetter(name) != '\0'; + } + + /** + * Returns the path as a string. + */ + public String getPathString() { + // Profile driven optimization: + // Preallocate a size determined by the depth, so that + // we do not have to expand the capacity of the StringBuilder + StringBuilder builder = new StringBuilder(depth * 20); + buildPathString(builder); + return builder.toString(); + } + + /** + * Returns the path as a string. + */ + @Override + public String toString() { + return getPathString(); + } + + @Override + public int hashCode() { + return hashCode; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof Path)) { + return false; + } + Path otherPath = (Path) other; + return fileSystem.equals(otherPath.fileSystem) && name.equals(otherPath.name) + && Objects.equals(parent, otherPath.parent); + } + + /** + * Returns a string of debugging information associated with this path. + * The results are unspecified and MUST NOT be interpreted programmatically. + */ + protected String toDebugString() { + return ""; + } + + /** + * Returns a path representing the parent directory of this path, + * or null iff this Path represents the root of the filesystem. + * + * <p>Note: This method normalises ".." and "." path segments by string + * processing, not by directory lookups. + */ + public Path getParentDirectory() { + return parent; + } + + /** + * Returns true iff this path denotes an existing file of any kind. Follows + * symbolic links. + */ + public boolean exists() { + return fileSystem.exists(this, true); + } + + /** + * Returns true iff this path denotes an existing file of any kind. + * + * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a + * symbolic link, the link is dereferenced until a file other than a + * symbolic link is found + */ + public boolean exists(Symlinks followSymlinks) { + return fileSystem.exists(this, followSymlinks.toBoolean()); + } + + /** + * Returns a new, immutable collection containing the names of all entities + * within the directory denoted by the current path. Follows symbolic links. + * + * @throws FileNotFoundException If the directory is not found + * @throws IOException If the path does not denote a directory + */ + public Collection<Path> getDirectoryEntries() throws IOException, FileNotFoundException { + return fileSystem.getDirectoryEntries(this); + } + + /** + * Returns a collection of the names and types of all entries within the directory + * denoted by the current path. Follows symbolic links if {@code followSymlinks} is true. + * Note that the order of the returned entries is not guaranteed. + * + * @param followSymlinks whether to follow symlinks or not + * + * @throws FileNotFoundException If the directory is not found + * @throws IOException If the path does not denote a directory + */ + public Collection<Dirent> readdir(Symlinks followSymlinks) throws IOException { + return fileSystem.readdir(this, followSymlinks.toBoolean()); + } + + /** + * Returns a new, immutable collection containing the names of all entities + * within the directory denoted by the current path, for which the given + * predicate is true. + * + * @throws FileNotFoundException If the directory is not found + * @throws IOException If the path does not denote a directory + */ + public Collection<Path> getDirectoryEntries(Predicate<? super Path> predicate) + throws IOException, FileNotFoundException { + return ImmutableList.<Path>copyOf(Iterables.filter(getDirectoryEntries(), predicate)); + } + + /** + * Returns the status of a file, following symbolic links. + * + * @throws IOException if there was an error obtaining the file status. Note, + * some implementations may defer the I/O, and hence the throwing of + * the exception, until the accessor methods of {@code FileStatus} are + * called. + */ + public FileStatus stat() throws IOException { + return fileSystem.stat(this, true); + } + + /** + * Like stat(), but returns null on file-nonexistence instead of throwing. + */ + public FileStatus statNullable() { + return statNullable(Symlinks.FOLLOW); + } + + /** + * Like stat(), but returns null on file-nonexistence instead of throwing. + */ + public FileStatus statNullable(Symlinks symlinks) { + return fileSystem.statNullable(this, symlinks.toBoolean()); + } + + /** + * Returns the status of a file, optionally following symbolic links. + * + * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a + * symbolic link, the link is dereferenced until a file other than a + * symbolic link is found + * @throws IOException if there was an error obtaining the file status. Note, + * some implementations may defer the I/O, and hence the throwing of + * the exception, until the accessor methods of {@code FileStatus} are + * called + */ + public FileStatus stat(Symlinks followSymlinks) throws IOException { + return fileSystem.stat(this, followSymlinks.toBoolean()); + } + + /** + * Like {@link #stat}, but may return null if the file is not found (corresponding to + * {@code ENOENT} and {@code ENOTDIR} in Unix's stat(2) function) instead of throwing. Follows + * symbolic links. + */ + public FileStatus statIfFound() throws IOException { + return fileSystem.statIfFound(this, true); + } + + /** + * Like {@link #stat}, but may return null if the file is not found (corresponding to + * {@code ENOENT} and {@code ENOTDIR} in Unix's stat(2) function) instead of throwing. + * + * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a + * symbolic link, the link is dereferenced until a file other than a + * symbolic link is found + */ + public FileStatus statIfFound(Symlinks followSymlinks) throws IOException { + return fileSystem.statIfFound(this, followSymlinks.toBoolean()); + } + + + /** + * Returns true iff this path denotes an existing directory. Follows symbolic + * links. + */ + public boolean isDirectory() { + return fileSystem.isDirectory(this, true); + } + + /** + * Returns true iff this path denotes an existing directory. + * + * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a + * symbolic link, the link is dereferenced until a file other than a + * symbolic link is found + */ + public boolean isDirectory(Symlinks followSymlinks) { + return fileSystem.isDirectory(this, followSymlinks.toBoolean()); + } + + /** + * Returns true iff this path denotes an existing regular or special file. + * Follows symbolic links. + * + * <p>For our purposes, "file" includes special files (socket, fifo, block or + * char devices) too; it excludes symbolic links and directories. + */ + public boolean isFile() { + return fileSystem.isFile(this, true); + } + + /** + * Returns true iff this path denotes an existing regular or special file. + * + * <p>For our purposes, a "file" includes special files (socket, fifo, block + * or char devices) too; it excludes symbolic links and directories. + * + * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a + * symbolic link, the link is dereferenced until a file other than a + * symbolic link is found. + */ + public boolean isFile(Symlinks followSymlinks) { + return fileSystem.isFile(this, followSymlinks.toBoolean()); + } + + /** + * Returns true iff this path denotes an existing symbolic link. Does not + * follow symbolic links. + */ + public boolean isSymbolicLink() { + return fileSystem.isSymbolicLink(this); + } + + /** + * Returns the last segment of this path, or "/" for the root directory. + */ + public String getBaseName() { + return name; + } + + /** + * Interprets the name of a path segment relative to the current path and + * returns the result. + * + * <p>This is a purely syntactic operation, i.e. it does no I/O, it does not + * validate the existence of any path, nor resolve symbolic links. If 'prefix' + * is not canonical, then a 'name' of '..' will be interpreted incorrectly. + * + * @precondition segment contains no slashes. + */ + private Path getCanonicalPath(String segment) { + if (segment.equals(".") || segment.equals("")) { + return this; // that's a noop + } else if (segment.equals("..")) { + // root's parent is root, when canonicalising: + return parent == null || isWindowsVolumeName() ? this : parent; + } else { + return getCachedChildPath(segment); + } + } + + /** + * Returns the path formed by appending the single non-special segment + * "baseName" to this path. + * + * <p>You should almost always use {@link #getRelative} instead, which has + * the same performance characteristics if the given name is a valid base + * name, and which also works for '.', '..', and strings containing '/'. + * + * @throws IllegalArgumentException if {@code baseName} is not a valid base + * name according to {@link FileSystemUtils#checkBaseName} + */ + public Path getChild(String baseName) { + FileSystemUtils.checkBaseName(baseName); + return getCachedChildPath(baseName); + } + + /** + * Returns the path formed by appending the relative or absolute path fragment + * {@code suffix} to this path. + * + * <p>If suffix is absolute, the current path will be ignored; otherwise, they + * will be combined. Up-level references ("..") cause the preceding path + * segment to be elided; this interpretation is only correct if the base path + * is canonical. + */ + public Path getRelative(PathFragment suffix) { + Path result = suffix.isAbsolute() ? fileSystem.getRootDirectory() : this; + if (!suffix.windowsVolume().isEmpty()) { + result = result.getCanonicalPath(suffix.windowsVolume()); + } + for (String segment : suffix.segments()) { + result = result.getCanonicalPath(segment); + } + return result; + } + + /** + * Returns the path formed by appending the relative or absolute string + * {@code path} to this path. + * + * <p>If the given path string is absolute, the current path will be ignored; + * otherwise, they will be combined. Up-level references ("..") cause the + * preceding path segment to be elided. + * + * <p>This is a purely syntactic operation, i.e. it does no I/O, it does not + * validate the existence of any path, nor resolve symbolic links. + */ + public Path getRelative(String path) { + // Fast path for valid base names. + if ((path.length() == 0) || (path.equals("."))) { + return this; + } else if (path.equals("..")) { + return parent == null ? this : parent; + } else if ((path.indexOf('/') != -1)) { + return getRelative(new PathFragment(path)); + } else { + return getCachedChildPath(path); + } + } + + /** + * Returns an absolute PathFragment representing this path. + */ + public PathFragment asFragment() { + String[] resultSegments = new String[depth]; + Path currentPath = this; + for (int pos = depth - 1; pos >= 0; pos--) { + resultSegments[pos] = currentPath.getBaseName(); + currentPath = currentPath.getParentDirectory(); + } + + char driveLetter = '\0'; + if (resultSegments.length > 0) { + driveLetter = PathFragment.getWindowsDriveLetter(resultSegments[0]); + if (driveLetter != '\0') { + // Strip off the first segment that contains the volume name. + resultSegments = Arrays.copyOfRange(resultSegments, 1, resultSegments.length); + } + } + + return new PathFragment(driveLetter, true, resultSegments); + } + + + /** + * Returns a relative path fragment to this path, relative to {@code + * ancestorDirectory}. {@code ancestorDirectory} must be on the same + * filesystem as this path. (Currently, both this path and "ancestorDirectory" + * must be absolute, though this restriction could be loosened.) + * <p> + * <code>x.relativeTo(z) == y</code> implies + * <code>z.getRelative(y.getPathString()) == x</code>. + * <p> + * For example, <code>"/foo/bar/wiz".relativeTo("/foo")</code> returns + * <code>"bar/wiz"</code>. + * + * @throws IllegalArgumentException if this path is not beneath {@code + * ancestorDirectory} or if they are not part of the same filesystem + */ + public PathFragment relativeTo(Path ancestorPath) { + checkSameFilesystem(ancestorPath); + + // Fast path: when otherPath is the ancestor of this path + int resultSegmentCount = depth - ancestorPath.depth; + if (resultSegmentCount >= 0) { + String[] resultSegments = new String[resultSegmentCount]; + Path currentPath = this; + for (int pos = resultSegmentCount - 1; pos >= 0; pos--) { + resultSegments[pos] = currentPath.getBaseName(); + currentPath = currentPath.getParentDirectory(); + } + if (ancestorPath.equals(currentPath)) { + return new PathFragment('\0', false, resultSegments); + } + } + + throw new IllegalArgumentException("Path " + this + " is not beneath " + ancestorPath); + } + + /** + * Checks that "this" and "that" are paths on the same filesystem. + */ + protected void checkSameFilesystem(Path that) { + if (this.fileSystem != that.fileSystem) { + throw new IllegalArgumentException("Files are on different filesystems: " + + this + ", " + that); + } + } + + /** + * Returns an output stream to the file denoted by the current path, creating + * it and truncating it if necessary. The stream is opened for writing. + * + * @throws FileNotFoundException If the file cannot be found or created. + * @throws IOException If a different error occurs. + */ + public OutputStream getOutputStream() throws IOException, FileNotFoundException { + return getOutputStream(false); + } + + /** + * Returns an output stream to the file denoted by the current path, creating + * it and truncating it if necessary. The stream is opened for writing. + * + * @param append whether to open the file in append mode. + * @throws FileNotFoundException If the file cannot be found or created. + * @throws IOException If a different error occurs. + */ + public OutputStream getOutputStream(boolean append) throws IOException, FileNotFoundException { + return fileSystem.getOutputStream(this, append); + } + + /** + * Creates a directory with the name of the current path, not following + * symbolic links. Returns normally iff the directory exists after the call: + * true if the directory was created by this call, false if the directory was + * already in existence. Throws an exception if the directory could not be + * created for any reason. + * + * @throws IOException if the directory creation failed for any reason + */ + public boolean createDirectory() throws IOException { + return fileSystem.createDirectory(this); + } + + /** + * Creates a symbolic link with the name of the current path, following + * symbolic links. The referent of the created symlink is is the absolute path + * "target"; it is not possible to create relative symbolic links via this + * method. + * + * @throws IOException if the creation of the symbolic link was unsuccessful + * for any reason + */ + public void createSymbolicLink(Path target) throws IOException { + checkSameFilesystem(target); + fileSystem.createSymbolicLink(this, target.asFragment()); + } + + /** + * Creates a symbolic link with the name of the current path, following + * symbolic links. The referent of the created symlink is is the path fragment + * "target", which may be absolute or relative. + * + * @throws IOException if the creation of the symbolic link was unsuccessful + * for any reason + */ + public void createSymbolicLink(PathFragment target) throws IOException { + fileSystem.createSymbolicLink(this, target); + } + + /** + * Returns the target of the current path, which must be a symbolic link. The + * link contents are returned exactly, and may contain an absolute or relative + * path. Analogous to readlink(2). + * + * @return the content (i.e. target) of the symbolic link + * @throws IOException if the current path is not a symbolic link, or the + * contents of the link could not be read for any reason + */ + public PathFragment readSymbolicLink() throws IOException { + return fileSystem.readSymbolicLink(this); + } + + /** + * Returns the canonical path for this path, by repeatedly replacing symbolic + * links with their referents. Analogous to realpath(3). + * + * @return the canonical path for this path + * @throws IOException if any symbolic link could not be resolved, or other + * error occurred (for example, the path does not exist) + */ + public Path resolveSymbolicLinks() throws IOException { + return fileSystem.resolveSymbolicLinks(this); + } + + /** + * Renames the file denoted by the current path to the location "target", not + * following symbolic links. + * + * <p>Files cannot be atomically renamed across devices; copying is required. + * Use {@link FileSystemUtils#copyFile} followed by {@link Path#delete}. + * + * @throws IOException if the rename failed for any reason + */ + public void renameTo(Path target) throws IOException { + checkSameFilesystem(target); + fileSystem.renameTo(this, target); + } + + /** + * Returns the size in bytes of the file denoted by the current path, + * following symbolic links. + * + * <p>The size of directory or special file is undefined. + * + * @throws FileNotFoundException if the file denoted by the current path does + * not exist + * @throws IOException if the file's metadata could not be read, or some other + * error occurred + */ + public long getFileSize() throws IOException, FileNotFoundException { + return fileSystem.getFileSize(this, true); + } + + /** + * Returns the size in bytes of the file denoted by the current path. + * + * <p>The size of directory or special file is undefined. The size of a symbolic + * link is the length of the name of its referent. + * + * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a + * symbolic link, the link is deferenced until a file other than a + * symbol link is found + * @throws FileNotFoundException if the file denoted by the current path does + * not exist + * @throws IOException if the file's metadata could not be read, or some other + * error occurred + */ + public long getFileSize(Symlinks followSymlinks) throws IOException, FileNotFoundException { + return fileSystem.getFileSize(this, followSymlinks.toBoolean()); + } + + /** + * Deletes the file denoted by this path, not following symbolic links. + * Returns normally iff the file doesn't exist after the call: true if this + * call deleted the file, false if the file already didn't exist. Throws an + * exception if the file could not be deleted for any reason. + * + * @return true iff the file was actually deleted by this call + * @throws IOException if the deletion failed but the file was present prior + * to the call + */ + public boolean delete() throws IOException { + return fileSystem.delete(this); + } + + /** + * Returns the last modification time of the file, in milliseconds since the + * UNIX epoch, of the file denoted by the current path, following symbolic + * links. + * + * <p>Caveat: many filesystems store file times in seconds, so do not rely on + * the millisecond precision. + * + * @throws IOException if the operation failed for any reason + */ + public long getLastModifiedTime() throws IOException { + return fileSystem.getLastModifiedTime(this, true); + } + + /** + * Returns the last modification time of the file, in milliseconds since the + * UNIX epoch, of the file denoted by the current path. + * + * <p>Caveat: many filesystems store file times in seconds, so do not rely on + * the millisecond precision. + * + * @param followSymlinks if {@link Symlinks#FOLLOW}, and this path denotes a + * symbolic link, the link is dereferenced until a file other than a + * symbolic link is found + * @throws IOException if the modification time for the file could not be + * obtained for any reason + */ + public long getLastModifiedTime(Symlinks followSymlinks) throws IOException { + return fileSystem.getLastModifiedTime(this, followSymlinks.toBoolean()); + } + + /** + * Sets the modification time of the file denoted by the current path. Follows + * symbolic links. If newTime is -1, the current time according to the kernel + * is used; this may differ from the JVM's clock. + * + * <p>Caveat: many filesystems store file times in seconds, so do not rely on + * the millisecond precision. + * + * @param newTime time, in milliseconds since the UNIX epoch, or -1L, meaning + * use the kernel's current time + * @throws IOException if the modification time for the file could not be set + * for any reason + */ + public void setLastModifiedTime(long newTime) throws IOException { + fileSystem.setLastModifiedTime(this, newTime); + } + + /** + * Returns value of the given extended attribute name or null if attribute does not exist or + * file system does not support extended attributes. Follows symlinks. + */ + public byte[] getxattr(String name) throws IOException { + return fileSystem.getxattr(this, name, true); + } + + /** + * Returns the type of digest that may be returned by {@link #getFastDigest}, or {@code null} + * if the filesystem doesn't support them. + */ + public String getFastDigestFunctionType() { + return fileSystem.getFastDigestFunctionType(this); + } + + /** + * Gets a fast digest for the given path, or {@code null} if there isn't one available. The + * digest should be suitable for detecting changes to the file. + */ + public byte[] getFastDigest() throws IOException { + return fileSystem.getFastDigest(this); + } + + /** + * Returns the MD5 digest of the file denoted by the current path, following + * symbolic links. + * + * <p>This method runs in O(n) time where n is the length of the file, but + * certain implementations may be much faster than the worst case. + * + * @return a new 16-byte array containing the file's MD5 digest + * @throws IOException if the MD5 digest could not be computed for any reason + */ + public byte[] getMD5Digest() throws IOException { + return fileSystem.getMD5Digest(this); + } + + /** + * Opens the file denoted by this path, following symbolic links, for reading, + * and returns an input stream to it. + * + * @throws IOException if the file was not found or could not be opened for + * reading + */ + public InputStream getInputStream() throws IOException { + return fileSystem.getInputStream(this); + } + + /** + * Returns a java.io.File representation of this path. + * + * <p>Caveat: the result may be useless if this path's getFileSystem() is not + * the UNIX filesystem. + */ + public File getPathFile() { + return new File(getPathString()); + } + + /** + * Returns true if the file denoted by the current path, following symbolic + * links, is writable for the current user. + * + * @throws FileNotFoundException if the file does not exist, a dangling + * symbolic link was encountered, or the file's metadata could not be + * read + */ + public boolean isWritable() throws IOException, FileNotFoundException { + return fileSystem.isWritable(this); + } + + /** + * Sets the read permissions of the file denoted by the current path, + * following symbolic links. Permissions apply to the current user. + * + * @param readable if true, the file is set to readable; otherwise the file is + * made non-readable + * @throws FileNotFoundException if the file does not exist + * @throws IOException If the action cannot be taken (ie. permissions) + */ + public void setReadable(boolean readable) throws IOException, FileNotFoundException { + fileSystem.setReadable(this, readable); + } + + /** + * Sets the write permissions of the file denoted by the current path, + * following symbolic links. Permissions apply to the current user. + * + * <p>TODO(bazel-team): (2009) what about owner/group/others? + * + * @param writable if true, the file is set to writable; otherwise the file is + * made non-writable + * @throws FileNotFoundException if the file does not exist + * @throws IOException If the action cannot be taken (ie. permissions) + */ + public void setWritable(boolean writable) throws IOException, FileNotFoundException { + fileSystem.setWritable(this, writable); + } + + /** + * Returns true iff the file specified by the current path, following symbolic + * links, is executable by the current user. + * + * @throws FileNotFoundException if the file does not exist or a dangling + * symbolic link was encountered + * @throws IOException if some other I/O error occurred + */ + public boolean isExecutable() throws IOException, FileNotFoundException { + return fileSystem.isExecutable(this); + } + + /** + * Returns true iff the file specified by the current path, following symbolic + * links, is readable by the current user. + * + * @throws FileNotFoundException if the file does not exist or a dangling + * symbolic link was encountered + * @throws IOException if some other I/O error occurred + */ + public boolean isReadable() throws IOException, FileNotFoundException { + return fileSystem.isReadable(this); + } + + /** + * Sets the execute permission on the file specified by the current path, + * following symbolic links. Permissions apply to the current user. + * + * @throws FileNotFoundException if the file does not exist or a dangling + * symbolic link was encountered + * @throws IOException if the metadata change failed, for example because of + * permissions + */ + public void setExecutable(boolean executable) throws IOException, FileNotFoundException { + fileSystem.setExecutable(this, executable); + } + + /** + * Sets the permissions on the file specified by the current path, following + * symbolic links. If permission changes on this path's {@link FileSystem} are + * slow (e.g. one syscall per change), this method should aim to be faster + * than setting each permission individually. If this path's + * {@link FileSystem} does not support group and others permissions, those + * bits will be ignored. + * + * @throws FileNotFoundException if the file does not exist or a dangling + * symbolic link was encountered + * @throws IOException if the metadata change failed, for example because of + * permissions + */ + public void chmod(int mode) throws IOException { + fileSystem.chmod(this, mode); + } + + /** + * Compare Paths of the same file system using their PathFragments. + * + * <p>Paths from different filesystems will be compared using the identity + * hash code of their respective filesystems. + */ + @Override + public int compareTo(Path o) { + // Fast-path. + if (equals(o)) { + return 0; + } + + // If they are on different file systems, the file system decides the ordering. + FileSystem otherFs = o.getFileSystem(); + if (!fileSystem.equals(otherFs)) { + int thisFileSystemHash = System.identityHashCode(fileSystem); + int otherFileSystemHash = System.identityHashCode(otherFs); + if (thisFileSystemHash < otherFileSystemHash) { + return -1; + } else if (thisFileSystemHash > otherFileSystemHash) { + return 1; + } else { + // TODO(bazel-team): Add a name to every file system to be used here. + return 0; + } + } + + // Equal file system, but different paths, because of the canonicalization. + // We expect to often compare Paths that are very similar, for example for files in the same + // directory. This can be done efficiently by going up segment by segment until we get the + // identical path (canonicalization again), and then just compare the immediate child segments. + // Overall this is much faster than creating PathFragment instances, and comparing those, which + // requires us to always go up to the top-level directory and copy all segments into a new + // string array. + // This was previously showing up as a hotspot in a profile of globbing a large directory. + Path a = this, b = o; + int maxDepth = Math.min(a.depth, b.depth); + while (a.depth > maxDepth) { + a = a.getParentDirectory(); + } + while (b.depth > maxDepth) { + b = b.getParentDirectory(); + } + // One is the child of the other. + if (a.equals(b)) { + // If a is the same as this, this.depth must be less than o.depth. + return equals(a) ? -1 : 1; + } + Path previousa, previousb; + do { + previousa = a; + previousb = b; + a = a.getParentDirectory(); + b = b.getParentDirectory(); + } while (a != b); // This has to happen eventually. + return previousa.name.compareTo(previousb.name); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/PathFragment.java b/src/main/java/com/google/devtools/build/lib/vfs/PathFragment.java new file mode 100644 index 0000000..1ee65a8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/PathFragment.java
@@ -0,0 +1,655 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.StringCanonicalizer; + +import java.io.File; +import java.io.InvalidObjectException; +import java.io.ObjectInputStream; +import java.io.Serializable; +import java.util.Arrays; +import java.util.Set; + +/** + * This class represents an immutable UNIX filesystem path, which may be absolute or relative. The + * path is maintained as a simple ordered list of path segment strings. + * + * <p>This class is independent from other VFS classes, especially anything requiring native code. + * It is safe to use in places that need simple segmented string path functionality. + * + * <p>There is some limited support for Windows-style paths. Most importantly, drive identifiers + * in front of a path (c:/abc) are supported and such paths are correctly recognized as absolute. + * However, Windows-style backslash separators (C:\\foo\\bar) are explicitly not supported, same + * with advanced features like \\\\network\\paths and \\\\?\\unc\\paths. + */ +@Immutable @ThreadSafe +public final class PathFragment implements Comparable<PathFragment>, Serializable { + + public static final int INVALID_SEGMENT = -1; + + public static final char SEPARATOR_CHAR = '/'; + + public static final char EXTRA_SEPARATOR_CHAR = + (OS.getCurrent() == OS.WINDOWS) ? '\\' : '/'; + + public static final String ROOT_DIR = "/"; + + /** An empty path fragment. */ + public static final PathFragment EMPTY_FRAGMENT = new PathFragment(""); + + public static final Function<String, PathFragment> TO_PATH_FRAGMENT = + new Function<String, PathFragment>() { + @Override + public PathFragment apply(String str) { + return new PathFragment(str); + } + }; + + public static final Predicate<PathFragment> IS_ABSOLUTE = + new Predicate<PathFragment>() { + @Override + public boolean apply(PathFragment input) { + return input.isAbsolute(); + } + }; + + private static final Function<PathFragment, String> TO_SAFE_PATH_STRING = + new Function<PathFragment, String>() { + @Override + public String apply(PathFragment path) { + return path.getSafePathString(); + } + }; + + // We have 3 word-sized fields (segments, hashCode and path), and 2 + // byte-sized ones, which fits in 16 bytes. Object sizes are rounded + // to 16 bytes. Medium sized builds can easily hold millions of + // live PathFragments, so do not add further fields on a whim. + + // The individual path components. + private final String[] segments; + + // True both for UNIX-style absolute paths ("/foo") and Windows-style ("C:/foo"). + private final boolean isAbsolute; + + // Upper case windows drive letter, or '\0' if none. While a volumeName string is more + // general, we create a lot of these objects, so space is at a premium. + private final char driveLetter; + + // hashCode and path are lazily initialized but semantically immutable. + private int hashCode; + private String path; + + /** + * Construct a PathFragment from a string, which is an absolute or relative UNIX or Windows path. + */ + public PathFragment(String path) { + this.driveLetter = getWindowsDriveLetter(path); + if (driveLetter != '\0') { + path = path.substring(2); + // TODO(bazel-team): Decide what to do about non-absolute paths with a volume name, e.g. C:x. + } + this.isAbsolute = path.length() > 0 && isSeparator(path.charAt(0)); + this.segments = segment(path, isAbsolute ? 1 : 0); + } + + private static boolean isSeparator(char c) { + return c == SEPARATOR_CHAR || c == EXTRA_SEPARATOR_CHAR; + } + + /** + * Construct a PathFragment from a java.io.File, which is an absolute or + * relative UNIX path. Does not support Windows-style Files. + */ + public PathFragment(File path) { + this(path.getPath()); + } + + /** + * Constructs a PathFragment, taking ownership of segments. Package-private, + * because it does not perform a defensive clone of the segments array. Used + * here in PathFragment, and by Path.asFragment() and Path.relativeTo(). + */ + PathFragment(char driveLetter, boolean isAbsolute, String[] segments) { + this.driveLetter = driveLetter; + this.isAbsolute = isAbsolute; + this.segments = segments; + } + + /** + * Construct a PathFragment from a sequence of other PathFragments. The new + * fragment will be absolute iff the first fragment was absolute. + */ + public PathFragment(PathFragment first, PathFragment second, PathFragment... more) { + // TODO(bazel-team): The handling of absolute path fragments in this constructor is unexpected. + this.segments = new String[sumLengths(first, second, more)]; + int offset = 0; + offset += addSegments(offset, first); + offset += addSegments(offset, second); + for (PathFragment fragment : more) { + offset += addSegments(offset, fragment); + } + this.isAbsolute = first.isAbsolute; + this.driveLetter = first.driveLetter; + } + + private int addSegments(int offset, PathFragment fragment) { + int count = fragment.segmentCount(); + System.arraycopy(fragment.segments, 0, this.segments, offset, count); + return count; + } + + private static int sumLengths(PathFragment first, PathFragment second, PathFragment[] more) { + int total = first.segmentCount() + second.segmentCount(); + for (PathFragment fragment : more) { + total += fragment.segmentCount(); + } + return total; + } + + /** + * Segments the string passed in as argument and returns an array of strings. + * The split is performed along occurrences of (sequences of) the slash + * character. + * + * @param toSegment the string to segment + * @param offset how many characters from the start of the string to ignore. + */ + private static String[] segment(String toSegment, int offset) { + char[] chars = toSegment.toCharArray(); + int length = chars.length; + + // Handle "/" and "" quickly. + if (length == offset) { + return new String[0]; + } + + // We make two passes through the array of characters: count & alloc, + // because simply using ArrayList was a bottleneck showing up during profiling. + int seg = 0; + int start = offset; + for (int i = offset; i < length; i++) { + if (isSeparator(chars[i])) { + if (i > start) { // to skip repeated separators + seg++; + } + start = i + 1; + } + } + if (start < length) { + seg++; + } + String[] result = new String[seg]; + seg = 0; + start = offset; + for (int i = offset; i < length; i++) { + if (isSeparator(chars[i])) { + if (i > start) { // to skip repeated separators + // Make a copy of the String here to allow the interning to save memory. String.substring + // does not make a copy, but refers to the original char array, preventing garbage + // collection of the parts that are unnecessary. + result[seg] = StringCanonicalizer.intern(new String(chars, start, i - start)); + seg++; + } + start = i + 1; + } + } + if (start < length) { + result[seg] = StringCanonicalizer.intern(new String(chars, start, length - start)); + seg++; + } + return result; + } + + private Object writeReplace() { + return new PathFragmentSerializationProxy(toString()); + } + + private void readObject(ObjectInputStream stream) throws InvalidObjectException { + throw new InvalidObjectException("Serialization is allowed only by proxy"); + } + + /** + * Returns the path string using '/' as the name-separator character. Returns "" if the path + * is both relative and empty. + */ + public String getPathString() { + // Double-checked locking works, even without volatile, because path is a String, according to: + // http://www.cs.umd.edu/~pugh/java/memoryModel/DoubleCheckedLocking.html + if (path == null) { + synchronized (this) { + if (path == null) { + path = StringCanonicalizer.intern(joinSegments(SEPARATOR_CHAR)); + } + } + } + return path; + } + + /** + * Returns "." if the path fragment is both relative and empty, or {@link + * #getPathString} otherwise. + */ + // TODO(bazel-team): Change getPathString to do this - this behavior makes more sense. + public String getSafePathString() { + return (!isAbsolute && (segmentCount() == 0)) ? "." : getPathString(); + } + + /** + * Returns a sequence consisting of the {@link #getSafePathString()} return of each item in + * {@code fragments}. + */ + public static Iterable<String> safePathStrings(Iterable<PathFragment> fragments) { + return Iterables.transform(fragments, TO_SAFE_PATH_STRING); + } + + private String joinSegments(char separatorChar) { + if (segments.length == 0 && isAbsolute) { + return windowsVolume() + ROOT_DIR; + } + + // Profile driven optimization: + // Preallocate a size determined by the number of segments, so that + // we do not have to expand the capacity of the StringBuilder. + // Heuristically, this estimate is right for about 99% of the time. + int estimateSize = + ((driveLetter != '\0') ? 2 : 0) + + ((segments.length == 0) ? 0 : (segments.length + 1) * 20); + StringBuilder result = new StringBuilder(estimateSize); + result.append(windowsVolume()); + boolean initialSegment = true; + for (String segment : segments) { + if (!initialSegment || isAbsolute) { + result.append(separatorChar); + } + initialSegment = false; + result.append(segment); + } + return result.toString(); + } + + /** + * Return true iff none of the segments are either "." or "..". + */ + public boolean isNormalized() { + for (String segment : segments) { + if (segment.equals(".") || segment.equals("..")) { + return false; + } + } + return true; + } + + /** + * Normalizes the path fragment: removes "." and ".." segments if possible + * (if there are too many ".." segments, the resulting PathFragment will still + * start with ".."). + */ + public PathFragment normalize() { + String[] scratchSegments = new String[segments.length]; + int segmentCount = 0; + + for (String segment : segments) { + if (segment.equals(".")) { + // Just discard it + } else if (segment.equals("..")) { + if (segmentCount > 0 && !scratchSegments[segmentCount - 1].equals("..")) { + // Remove the last segment, if there is one and it is not "..". This + // means that the resulting PathFragment can still contain ".." + // segments at the beginning. + segmentCount--; + } else { + scratchSegments[segmentCount++] = segment; + } + } else { + scratchSegments[segmentCount++] = segment; + } + } + + if (segmentCount == segments.length) { + // Optimization, no new PathFragment needs to be created. + return this; + } + + return new PathFragment(driveLetter, isAbsolute, + subarray(scratchSegments, 0, segmentCount)); + } + + /** + * Returns the path formed by appending the relative or absolute path fragment + * {@code suffix} to this path. + * + * <p>If suffix is absolute, the current path will be ignored; otherwise, they + * will be concatenated. This is a purely syntactic operation, with no path + * normalization or I/O performed. + */ + public PathFragment getRelative(PathFragment otherFragment) { + return otherFragment.isAbsolute() + ? otherFragment + : new PathFragment(this, otherFragment); + } + + /** + * Returns the path formed by appending the relative or absolute string + * {@code path} to this path. + * + * <p>If the given path string is absolute, the current path will be ignored; + * otherwise, they will be concatenated. This is a purely syntactic operation, + * with no path normalization or I/O performed. + */ + public PathFragment getRelative(String path) { + return getRelative(new PathFragment(path)); + } + + /** + * Returns the path formed by appending the single non-special segment + * "baseName" to this path. + * + * <p>You should almost always use {@link #getRelative} instead, which has + * the same performance characteristics if the given name is a valid base + * name, and which also works for '.', '..', and strings containing '/'. + * + * @throws IllegalArgumentException if {@code baseName} is not a valid base + * name according to {@link FileSystemUtils#checkBaseName} + */ + public PathFragment getChild(String baseName) { + FileSystemUtils.checkBaseName(baseName); + baseName = StringCanonicalizer.intern(baseName); + String[] newSegments = new String[segments.length + 1]; + System.arraycopy(segments, 0, newSegments, 0, segments.length); + newSegments[newSegments.length - 1] = baseName; + return new PathFragment(driveLetter, isAbsolute, newSegments); + } + + /** + * Returns the last segment of this path, or "" for the empty fragment. + */ + public String getBaseName() { + return (segments.length == 0) ? "" : segments[segments.length - 1]; + } + + /** + * Returns a relative path fragment to this path, relative to + * {@code ancestorDirectory}. + * <p> + * <code>x.relativeTo(z) == y</code> implies + * <code>z.getRelative(y) == x</code>. + * <p> + * For example, <code>"foo/bar/wiz".relativeTo("foo")</code> + * returns <code>"bar/wiz"</code>. + */ + public PathFragment relativeTo(PathFragment ancestorDirectory) { + String[] ancestorSegments = ancestorDirectory.segments(); + int ancestorLength = ancestorSegments.length; + + if (isAbsolute != ancestorDirectory.isAbsolute() + || segments.length < ancestorLength) { + throw new IllegalArgumentException("PathFragment " + this + + " is not beneath " + ancestorDirectory); + } + + for (int index = 0; index < ancestorLength; index++) { + if (!segments[index].equals(ancestorSegments[index])) { + throw new IllegalArgumentException("PathFragment " + this + + " is not beneath " + ancestorDirectory); + } + } + + int length = segments.length - ancestorLength; + String[] resultSegments = subarray(segments, ancestorLength, length); + return new PathFragment('\0', false, resultSegments); + } + + /** + * Returns a relative path fragment to this path, relative to {@code path}. + */ + public PathFragment relativeTo(String path) { + return relativeTo(new PathFragment(path)); + } + + /** + * Returns a new PathFragment formed by appending {@code newName} to the + * parent directory. Null is returned iff this method is called on a + * PathFragment with zero segments. If {@code newName} designates an absolute path, + * the value of {@code this} will be ignored and a PathFragment corresponding to + * {@code newName} will be returned. This behavior is consistent with the behavior of + * {@link #getRelative(String)}. + */ + public PathFragment replaceName(String newName) { + return segments.length == 0 ? null : getParentDirectory().getRelative(newName); + } + + /** + * Returns a path representing the parent directory of this path, + * or null iff this Path represents the root of the filesystem. + * + * <p>Note: This method DOES NOT normalize ".." and "." path segments. + */ + public PathFragment getParentDirectory() { + return segments.length == 0 ? null : subFragment(0, segments.length - 1); + } + + /** + * Returns true iff {@code prefix}, considered as a list of path segments, is + * a prefix of {@code this}, and that they are both relative or both + * absolute. + * + * This is a reflexive, transitive, anti-symmetric relation (i.e. a partial + * order) + */ + public boolean startsWith(PathFragment prefix) { + if (this.isAbsolute != prefix.isAbsolute || + this.segments.length < prefix.segments.length || + this.driveLetter != prefix.driveLetter) { + return false; + } + for (int i = 0, len = prefix.segments.length; i < len; i++) { + if (!this.segments[i].equals(prefix.segments[i])) { + return false; + } + } + return true; + } + + /** + * Returns true iff {@code suffix}, considered as a list of path segments, is + * relative and a suffix of {@code this}, or both are absolute and equal. + * + * This is a reflexive, transitive, anti-symmetric relation (i.e. a partial + * order) + */ + public boolean endsWith(PathFragment suffix) { + if ((suffix.isAbsolute && !suffix.equals(this)) || + this.segments.length < suffix.segments.length) { + return false; + } + int offset = this.segments.length - suffix.segments.length; + for (int i = 0; i < suffix.segments.length; i++) { + if (!this.segments[offset + i].equals(suffix.segments[i])) { + return false; + } + } + return true; + } + + private static String[] subarray(String[] array, int start, int length) { + String[] subarray = new String[length]; + System.arraycopy(array, start, subarray, 0, length); + return subarray; + } + + /** + * Returns a new path fragment that is a sub fragment of this one. + * The sub fragment begins at the specified <code>beginIndex</code> segment + * and ends at the segment at index <code>endIndex - 1</code>. Thus the number + * of segments in the new PathFragment is <code>endIndex - beginIndex</code>. + * + * @param beginIndex the beginning index, inclusive. + * @param endIndex the ending index, exclusive. + * @return the specified sub fragment, never null. + * @exception IndexOutOfBoundsException if the + * <code>beginIndex</code> is negative, or + * <code>endIndex</code> is larger than the length of + * this <code>String</code> object, or + * <code>beginIndex</code> is larger than + * <code>endIndex</code>. + */ + public PathFragment subFragment(int beginIndex, int endIndex) { + int count = segments.length; + if ((beginIndex < 0) || (beginIndex > endIndex) || (endIndex > count)) { + throw new IndexOutOfBoundsException(String.format("path: %s, beginIndex: %d endIndex: %d", + toString(), beginIndex, endIndex)); + } + boolean isAbsolute = (beginIndex == 0) && this.isAbsolute; + return ((beginIndex == 0) && (endIndex == count)) ? this : + new PathFragment(driveLetter, isAbsolute, + subarray(segments, beginIndex, endIndex - beginIndex)); + } + + /** + * Returns true iff the path represented by this object is absolute. + */ + public boolean isAbsolute() { + return isAbsolute; + } + + /** + * Returns the segments of this path fragment. This array should not be + * modified. + */ + String[] segments() { + return segments; + } + + public String windowsVolume() { + if (OS.getCurrent() != OS.WINDOWS) { + return ""; + } + return (driveLetter != '\0') ? driveLetter + ":" : ""; + } + + /** + * Returns the number of segments in this path. + */ + public int segmentCount() { + return segments.length; + } + + /** + * Returns the specified segment of this path; index must be positive and + * less than numSegments(). + */ + public String getSegment(int index) { + return segments[index]; + } + + /** + * Returns the index of the first segment which equals one of the input values + * or {@link PathFragment#INVALID_SEGMENT} if none of the segments match. + */ + public int getFirstSegment(Set<String> values) { + for (int i = 0; i < segments.length; i++) { + if (values.contains(segments[i])) { + return i; + } + } + return INVALID_SEGMENT; + } + + /** + * Returns true iff this path contains uplevel references "..". + */ + public boolean containsUplevelReferences() { + for (String segment : segments) { + if (segment.equals("..")) { + return true; + } + } + return false; + } + + /** + * Given a path, returns the Windows drive letter ('X'), or an null character if no volume + * name was specified. + */ + static char getWindowsDriveLetter(String path) { + if (OS.getCurrent() == OS.WINDOWS + && path.length() >= 2 && path.charAt(1) == ':' && Character.isLetter(path.charAt(0))) { + return Character.toUpperCase(path.charAt(0)); + } + return '\0'; + } + + @Override + public int hashCode() { + int h = hashCode; + if (h == 0) { + h = isAbsolute ? 1 : 0; + for (String segment : segments) { + h = h * 31 + segment.hashCode(); + } + hashCode = h; + } + return h; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof PathFragment)) { + return false; + } + PathFragment otherPath = (PathFragment) other; + return isAbsolute == otherPath.isAbsolute && + Arrays.equals(otherPath.segments, segments); + } + + /** + * Compares two PathFragments using the lexicographical order. + */ + @Override + public int compareTo(PathFragment p2) { + if (isAbsolute != p2.isAbsolute) { + return isAbsolute ? -1 : 1; + } + PathFragment p1 = this; + String[] segments1 = p1.segments; + String[] segments2 = p2.segments; + int len1 = segments1.length; + int len2 = segments2.length; + int n = Math.min(len1, len2); + for (int i = 0; i < n; i++) { + String segment1 = segments1[i]; + String segment2 = segments2[i]; + if (!segment1.equals(segment2)) { + return segment1.compareTo(segment2); + } + } + return len1 - len2; + } + + @Override + public String toString() { + return getPathString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/PathFragmentSerializationProxy.java b/src/main/java/com/google/devtools/build/lib/vfs/PathFragmentSerializationProxy.java new file mode 100644 index 0000000..6e1b04d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/PathFragmentSerializationProxy.java
@@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectOutput; + + +/** + * A helper proxy for serializing immutable {@link PathFragment} objects. + */ +public final class PathFragmentSerializationProxy implements Externalizable { + private String pathFragmentString; + + public PathFragmentSerializationProxy(String pathFragmentString) { + this.pathFragmentString = pathFragmentString; + } + + // For deserialization machinery. + public PathFragmentSerializationProxy() { + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + // Manual serialization gives us about a 30% reduction in size. + out.writeUTF(pathFragmentString); + } + + @Override + public void readExternal(java.io.ObjectInput in) throws IOException { + this.pathFragmentString = in.readUTF(); + } + + private Object readResolve() { + return new PathFragment(pathFragmentString); + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/ReadonlyFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/ReadonlyFileSystem.java new file mode 100644 index 0000000..fc668db --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/ReadonlyFileSystem.java
@@ -0,0 +1,103 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * An abstract partial implementation of FileSystem for read-only + * implementations. + * + * <p>Any ReadonlyFileSystem does not support the following: + * <ul> + * <li>{@link #createDirectory(Path)}</li> + * <li>{@link #createSymbolicLink(Path, PathFragment)}</li> + * <li>{@link #delete(Path)}</li> + * <li>{@link #getOutputStream(Path)}</li> + * <li>{@link #renameTo(Path, Path)}</li> + * <li>{@link #setExecutable(Path, boolean)}</li> + * <li>{@link #setLastModifiedTime(Path, long)}</li> + * <li>{@link #setWritable(Path, boolean)}</li> + * </ul> + * The above calls will always result in an {@link IOException}. + */ +public abstract class ReadonlyFileSystem extends FileSystem { + + protected ReadonlyFileSystem() { + } + + protected IOException modificationException() { + String longname = this.getClass().getName(); + String shortname = longname.substring(longname.lastIndexOf(".") + 1); + return new IOException( + shortname + " does not support mutating operations"); + } + + @Override + protected OutputStream getOutputStream(Path path, boolean append) throws IOException { + throw modificationException(); + } + + @Override + protected void setReadable(Path path, boolean readable) throws IOException { + throw modificationException(); + } + + @Override + protected void setWritable(Path path, boolean writable) throws IOException { + throw modificationException(); + } + + @Override + protected void setExecutable(Path path, boolean executable) { + throw new UnsupportedOperationException("setExecutable"); + } + + @Override + public boolean supportsModifications() { + return false; + } + + @Override + public boolean supportsSymbolicLinks() { + return false; + } + + @Override + protected boolean createDirectory(Path path) throws IOException { + throw modificationException(); + } + + @Override + protected void createSymbolicLink(Path linkPath, PathFragment targetFragment) throws IOException { + throw modificationException(); + } + + @Override + protected void renameTo(Path sourcePath, Path targetPath) throws IOException { + throw modificationException(); + } + + @Override + protected boolean delete(Path path) throws IOException { + throw modificationException(); + } + + @Override + protected void setLastModifiedTime(Path path, long newTime) throws IOException { + throw modificationException(); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/RootedPath.java b/src/main/java/com/google/devtools/build/lib/vfs/RootedPath.java new file mode 100644 index 0000000..c753aa6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/RootedPath.java
@@ -0,0 +1,116 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.common.base.Preconditions; + +import java.io.Serializable; +import java.util.Objects; + +/** + * A {@link PathFragment} relative to a root, which is an absolute {@link Path}. Typically the root + * will be a package path entry. + * + * Two {@link RootedPath}s are considered equal iff they have equal roots and equal relative paths. + * + * TODO(bazel-team): refactor Artifact to use this instead of Root. + * TODO(bazel-team): use an opaque root representation so as to not expose the absolute path to + * clients via #asPath or #getRoot. + */ +public class RootedPath implements Serializable { + + private final Path root; + private final PathFragment relativePath; + private final Path path; + + /** + * Constructs a {@link RootedPath} from an absolute root path and a non-absolute relative path. + */ + private RootedPath(Path root, PathFragment relativePath) { + Preconditions.checkState(!relativePath.isAbsolute(), "relativePath: %s root: %s", relativePath, + root); + this.root = root; + this.relativePath = relativePath.normalize(); + this.path = root.getRelative(this.relativePath); + } + + /** + * Returns a rooted path representing {@code relativePath} relative to {@code root}. + */ + public static RootedPath toRootedPath(Path root, PathFragment relativePath) { + return new RootedPath(root, relativePath); + } + + /** + * Returns a rooted path representing {@code path} under the root {@code root}. + */ + public static RootedPath toRootedPath(Path root, Path path) { + Preconditions.checkState(path.startsWith(root), "path: %s root: %s", path, root); + return new RootedPath(root, path.relativeTo(root)); + } + + /** + * Returns a rooted path representing {@code path} under one of the package roots, or under the + * filesystem root if it's not under any package root. + */ + public static RootedPath toRootedPathMaybeUnderRoot(Path path, Iterable<Path> packagePathRoots) { + for (Path root : packagePathRoots) { + if (path.startsWith(root)) { + return toRootedPath(root, path); + } + } + return toRootedPath(path.getFileSystem().getRootDirectory(), path); + } + + public Path asPath() { + // Ideally, this helper method would not be needed. But Skyframe's FileFunction and + // DirectoryListingFunction need to do filesystem operations on the absolute path and + // Path#getRelative(relPath) is O(relPath.segmentCount()). Therefore we precompute the absolute + // path represented by this relative path. + return path; + } + + public Path getRoot() { + return root; + } + + /** + * Returns the (normalized) path relative to {@code #getRoot}. + */ + public PathFragment getRelativePath() { + return relativePath; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof RootedPath)) { + return false; + } + RootedPath other = (RootedPath) obj; + return Objects.equals(root, other.root) && Objects.equals(relativePath, other.relativePath); + } + + @Override + public int hashCode() { + return Objects.hash(root, relativePath); + } + + @Override + public String toString() { + return "[" + root.toString() + "]/[" + relativePath.toString() + "]"; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/ScopeEscapableFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/ScopeEscapableFileSystem.java new file mode 100644 index 0000000..429de18 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/ScopeEscapableFileSystem.java
@@ -0,0 +1,143 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile; + +import java.io.IOException; + +/** + * A file system that's capable of identifying paths residing outside its scope + * and using a delegator (such as {@link UnionFileSystem}) to re-route them + * to appropriate alternative file systems. + * + * <p>This is most useful for symlinks, which may ostensibly fall beneath some + * file system but resolve to paths outside that file system. + * + * <p>Note that we don't protect against cross-filesystem circular references. + * Therefore, care should be taken not to mix two scopable file systems that + * can reference each other. This theoretical safety cost is balanced by + * decreased code complexity requirements in implementations. + */ +public abstract class ScopeEscapableFileSystem extends FileSystem { + + private FileSystem delegator; + protected final PathFragment scopeRoot; + private boolean enableScopeChecking = true; // Used for testing. + + /** + * Instantiates a new ScopeEscapableFileSystem. + * + * @param scopeRoot the root path for the file system's scope. Any path + * that isn't beneath this one is considered out of scope according + * to {@link #inScope}. If null, scope checking is disabled. Note + * this is not the same thing as {@link FileSystem#rootPath}, which + * generally resolves to "/". + */ + protected ScopeEscapableFileSystem(PathFragment scopeRoot) { + this.scopeRoot = scopeRoot; + } + + @VisibleForTesting + void enableScopeChecking(boolean enable) { + this.enableScopeChecking = enable; + } + + /** + * Sets the delegator used to resolve paths that fall outside this file + * system's scope. + * + * <p>This method is not thread safe. It's intended to be called during + * instance initialization, not during active usage. The only reason this + * isn't set as immutable state within the constructor is that the delegator + * may need a reference to this instance for its own constructor. + */ + @ThreadHostile + public void setDelegator(FileSystem delegator) { + this.delegator = delegator; + } + + /** + * Uses the delegator to convert a path fragment to a path that's bound + * to the file system that manages that path. + */ + protected Path getDelegatedPath(PathFragment path) { + Preconditions.checkState(delegator != null); + return delegator.getPath(path); + } + + /** + * Proxy for {@link FileSystem#resolveOneLink} that sends the input path + * through the delegator. + */ + protected PathFragment resolveOneLinkWithDelegator(final PathFragment path) throws IOException { + Preconditions.checkState(delegator != null); + return delegator.resolveOneLink(getDelegatedPath(path)); + } + + /** + * Proxy for {@link FileSystem#stat} that sends the input path through + * the delegator. + */ + protected FileStatus statWithDelegator(final PathFragment path, final boolean followSymlinks) + throws IOException { + Preconditions.checkState(delegator != null); + return delegator.stat(getDelegatedPath(path), followSymlinks); + } + + /** + * Returns true if the given path is within this file system's scope, false + * otherwise. + * + * @param parentDepth the number of segments in the path's parent directory + * (only meaningful for paths that begin with ".."). The parent directory + * itself is assumed to be in scope. + * @param normalizedPath input path, expected to be normalized such that all + * ".." and "." segments are removed (with the exception of a possible + * prefix sequence of contiguous ".." segments) + */ + protected boolean inScope(int parentDepth, PathFragment normalizedPath) { + if (scopeRoot == null || !enableScopeChecking) { + return true; + } else if (normalizedPath.isAbsolute()) { + return normalizedPath.startsWith(scopeRoot); + } else { + // Efficiency note: we're not accounting for "/scope/root/../root" paths here, i.e. paths + // that appear to go out of scope but ultimately stay within scope. This may result in + // unnecessary re-delegation back into the same FS. we're choosing to forgo that + // optimization under the assumption that such scenarios are rare and unimportant to + // overall performance. We can always enhance this if needed. + return parentDepth - leadingParentReferences(normalizedPath) >= scopeRoot.segmentCount(); + } + } + + /** + * Given a path that's normalized (no ".." or "." segments), except for a possible + * prefix sequence of contiguous ".." segments, returns the size of that prefix + * sequence. + * + * <p>Example allowed inputs: "/absolute/path", "relative/path", "../../relative/path". + * Example disallowed inputs: "/absolute/path/../path2", "relative/../path", "../relative/../p". + */ + protected int leadingParentReferences(PathFragment normalizedPath) { + int leadingParentReferences = 0; + for (int i = 0; i < normalizedPath.segmentCount() && + normalizedPath.getSegment(i).equals(".."); i++) { + leadingParentReferences++; + } + return leadingParentReferences; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Symlinks.java b/src/main/java/com/google/devtools/build/lib/vfs/Symlinks.java new file mode 100644 index 0000000..ceb353a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/Symlinks.java
@@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +/** + * An enumeration for selecting between {@code stat}- and {@code lstat}-like + * behavior in various {@link Path} operations. + */ +public enum Symlinks { + + /** Follow symbolic links; stat(2)-like behaviour. */ + FOLLOW, + + /** Do not follow symbolic links; lstat(2)-like behaviour. */ + NOFOLLOW; + + boolean toBoolean() { return this == FOLLOW; } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java new file mode 100644 index 0000000..b349b53 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java
@@ -0,0 +1,419 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.vfs; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.util.StringTrie; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Collection; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Presents a unified view of multiple virtual {@link FileSystem} instances, to which requests are + * delegated based on a {@link PathFragment} prefix mapping. + * If multiple prefixes apply to a given path, the *longest* (i.e. most specific) match is used. + * The order in which the delegates are specified does not influence the mapping. + * + * <p>Paths are preserved absolutely, contrary to how "mount" works, e.g.: + * /foo/bar maps to /foo/bar on the delegate, even if it is mounted at /foo. + * + * <p>For example: + * "/in" maps to InFileSystem, "/" maps to OtherFileSystem. + * Reading from "/in/base/BUILD" through the UnionFileSystem will delegate the read operation to + * InFileSystem, which will read "/in/base/BUILD" relative to its root. + * ("mount" behavior would remap it to "/base/BUILD" on the delegate). + * + * <p>Intra-filesystem symbolic links are resolved to their ultimate targets. + * Cross-filesystem links are not currently supported. + */ +@ThreadSafety.ThreadSafe +public class UnionFileSystem extends FileSystem { + + // Prefix trie index, allowing children to easily inherit prefix mappings + // of their parents. + // This does not currently handle unicode filenames. + private StringTrie<FileSystem> pathDelegate; + + // True iff the filesystem can be modified. If false, mutating operations + // will throw UnsupportedOperationExceptions. + private final boolean readOnly; + + /** + * Creates a new modifiable UnionFileSystem with prefix mappings + * specified by a map. + * + * @param prefixMapping map of path prefixes to {@link FileSystem}s + */ + public UnionFileSystem(Map<PathFragment, FileSystem> prefixMapping, + FileSystem rootFileSystem) { + this(prefixMapping, rootFileSystem, /* readOnly */ false); + } + + /** + * Creates a new modifiable or read-only UnionFileSystem with prefix mappings + * specified by a map. + * + * @param prefixMapping map of path prefixes to delegate {@link FileSystem}s + * @param rootFileSystem root for default requests; i.e. mapping of "/" + * @param readOnly if true, mutating operations will throw + */ + public UnionFileSystem(Map<PathFragment, FileSystem> prefixMapping, + FileSystem rootFileSystem, boolean readOnly) { + super(); + Preconditions.checkNotNull(prefixMapping); + Preconditions.checkNotNull(rootFileSystem); + Preconditions.checkArgument(rootFileSystem != this, "Circular root filesystem."); + Preconditions.checkArgument( + !prefixMapping.containsKey(PathFragment.EMPTY_FRAGMENT), + "Attempted to specify an explicit root prefix mapping; " + + "please use the rootFileSystem argument instead."); + + this.readOnly = readOnly; + this.pathDelegate = new StringTrie<FileSystem>(); + + for (Map.Entry<PathFragment, FileSystem> prefix : prefixMapping.entrySet()) { + FileSystem delegate = prefix.getValue(); + PathFragment prefixPath = prefix.getKey(); + + // Extra slash prevents within-directory mappings, which Path can't handle. + String path = prefixPath.getPathString(); + pathDelegate.put(path, delegate); + } + pathDelegate.put(PathFragment.EMPTY_FRAGMENT.getPathString(), rootFileSystem); + } + + /** + * Retrieves the filesystem delegate of a path mapping. + * Does not follow symlinks (but you can call on a path preprocessed with + * {@link #resolveSymbolicLinks} to support this use case). + * + * @param path the {@link Path} to map to a filesystem + * @throws IllegalArgumentException if no delegate exists for the path + */ + protected FileSystem getDelegate(Path path) { + Preconditions.checkNotNull(path); + + String pathString = path.getPathString(); + FileSystem immediateDelegate = pathDelegate.get(pathString); + + // Should never actually happen if the root delegate is present. + Preconditions.checkArgument(immediateDelegate != null, "No delegate filesystem exists for %s", + pathString); + return immediateDelegate; + } + + // Associates the path with the root of the given delegate filesystem. + // Necessary to avoid null pointer problems inside of the delegates. + protected Path adjustPath(Path path, FileSystem delegate) { + return delegate.getPath(path.asFragment()); + } + + /** + * Follow a symbolic link once using the appropriate delegate filesystem, also + * resolving parent directory symlinks. + * + * @param path {@link Path} to the symbolic link + */ + @Override + protected PathFragment readSymbolicLink(Path path) throws IOException { + Preconditions.checkNotNull(path); + FileSystem delegate = getDelegate(path); + return delegate.readSymbolicLink(adjustPath(path, delegate)); + } + + @Override + protected PathFragment resolveOneLink(Path path) throws IOException { + Preconditions.checkNotNull(path); + FileSystem delegate = getDelegate(path); + return delegate.resolveOneLink(adjustPath(path, delegate)); + } + + private void checkModifiable() { + if (!supportsModifications()) { + throw new UnsupportedOperationException( + "Modifications to this " + getClass().getSimpleName() + " are disabled."); + } + } + + @Override + public boolean supportsModifications() { + return !readOnly; + } + + @Override + public boolean supportsSymbolicLinks() { + return true; + } + + @Override + public String getFileSystemType(Path path) { + FileSystem delegate = getDelegate(path); + return delegate.getFileSystemType(path); + } + + @Override + protected byte[] getMD5Digest(Path path) throws IOException { + FileSystem delegate = getDelegate(path); + return delegate.getMD5Digest(adjustPath(path, delegate)); + } + + @Override + protected boolean createDirectory(Path path) throws IOException { + checkModifiable(); + // When creating the exact directory that is mapped, + // create it on both the parent's delegate and the path's delegate. + // This is necessary both for the parent to see the directory and for the + // delegate to use it. + // This is present to address this problematic case: + // / -> RootFs + // /foo -> FooFs + // mkdir /foo + // ls / ("foo" would be missing if not created on the parent) + // ls /foo (would fail if foo weren't also present on the child) + FileSystem delegate = getDelegate(path); + Path parent = path.getParentDirectory(); + if (parent != null) { + FileSystem parentDelegate = getDelegate(parent); + if (parentDelegate != delegate) { + // There's a possibility it already exists on the parent, so don't die + // if the directory can't be created there. + parentDelegate.createDirectory(adjustPath(path, parentDelegate)); + } + } + return delegate.createDirectory(adjustPath(path, delegate)); + } + + @Override + protected long getFileSize(Path path, boolean followSymlinks) throws IOException { + FileSystem delegate = getDelegate(path); + return delegate.getFileSize(adjustPath(path, delegate), followSymlinks); + } + + @Override + protected boolean delete(Path path) throws IOException { + checkModifiable(); + FileSystem delegate = getDelegate(path); + return delegate.delete(adjustPath(path, delegate)); + } + + @Override + protected long getLastModifiedTime(Path path, boolean followSymlinks) throws IOException { + FileSystem delegate = getDelegate(path); + return delegate.getLastModifiedTime(adjustPath(path, delegate), followSymlinks); + } + + @Override + protected void setLastModifiedTime(Path path, long newTime) throws IOException { + checkModifiable(); + FileSystem delegate = getDelegate(path); + delegate.setLastModifiedTime(adjustPath(path, delegate), newTime); + } + + @Override + protected boolean isSymbolicLink(Path path) { + FileSystem delegate = getDelegate(path); + path = adjustPath(path, delegate); + return delegate.isSymbolicLink(path); + } + + @Override + protected boolean isDirectory(Path path, boolean followSymlinks) { + FileSystem delegate = getDelegate(path); + return delegate.isDirectory(adjustPath(path, delegate), followSymlinks); + } + + @Override + protected boolean isFile(Path path, boolean followSymlinks) { + FileSystem delegate = getDelegate(path); + return delegate.isFile(adjustPath(path, delegate), followSymlinks); + } + + @Override + protected void createSymbolicLink(Path linkPath, PathFragment targetFragment) throws IOException { + checkModifiable(); + if (!supportsSymbolicLinks()) { + throw new UnsupportedOperationException( + "Attempted to create a symlink, but symlink support is disabled."); + } + + FileSystem delegate = getDelegate(linkPath); + delegate.createSymbolicLink(adjustPath(linkPath, delegate), targetFragment); + } + + @Override + protected boolean exists(Path path, boolean followSymlinks) { + FileSystem delegate = getDelegate(path); + return delegate.exists(adjustPath(path, delegate), followSymlinks); + } + + @Override + protected FileStatus stat(final Path path, final boolean followSymlinks) throws IOException { + FileSystem delegate = getDelegate(path); + return delegate.stat(adjustPath(path, delegate), followSymlinks); + } + + // Needs to be overridden for the delegation logic, because the + // UnixFileSystem implements statNullable and stat as separate codepaths. + // More generally, we wish to delegate all filesystem operations. + @Override + protected FileStatus statNullable(Path path, boolean followSymlinks) { + FileSystem delegate = getDelegate(path); + return delegate.statNullable(adjustPath(path, delegate), followSymlinks); + } + + @Override + @Nullable + protected FileStatus statIfFound(Path path, boolean followSymlinks) throws IOException { + FileSystem delegate = getDelegate(path); + return delegate.statIfFound(adjustPath(path, delegate), followSymlinks); + } + + /** + * Retrieves the directory entries for the specified path under the assumption + * that {@code resolvedPath} is the resolved path of {@code path} in one of the + * underlying file systems. + * + * @param path the {@link Path} whose children are to be retrieved + */ + @Override + protected Collection<Path> getDirectoryEntries(Path path) throws IOException { + FileSystem delegate = getDelegate(path); + Path resolvedPath = adjustPath(path, delegate); + Collection<Path> entries = resolvedPath.getDirectoryEntries(); + Collection<Path> result = Lists.newArrayListWithCapacity(entries.size()); + for (Path entry : entries) { + result.add(path.getChild(entry.getBaseName())); + } + return result; + } + + // No need for the more complex logic of getDirectoryEntries; it calls it implicitly. + @Override + protected Collection<Dirent> readdir(Path path, boolean followSymlinks) throws IOException { + FileSystem delegate = getDelegate(path); + return delegate.readdir(adjustPath(path, delegate), followSymlinks); + } + + @Override + protected boolean isReadable(Path path) throws IOException { + FileSystem delegate = getDelegate(path); + return delegate.isReadable(adjustPath(path, delegate)); + } + + @Override + protected void setReadable(Path path, boolean readable) throws IOException { + checkModifiable(); + FileSystem delegate = getDelegate(path); + delegate.setReadable(adjustPath(path, delegate), readable); + } + + @Override + protected boolean isWritable(Path path) throws IOException { + if (!supportsModifications()) { + return false; + } + FileSystem delegate = getDelegate(path); + return delegate.isWritable(adjustPath(path, delegate)); + } + + @Override + protected void setWritable(Path path, boolean writable) throws IOException { + checkModifiable(); + FileSystem delegate = getDelegate(path); + delegate.setWritable(adjustPath(path, delegate), writable); + } + + @Override + protected boolean isExecutable(Path path) throws IOException { + FileSystem delegate = getDelegate(path); + return delegate.isExecutable(adjustPath(path, delegate)); + } + + @Override + protected void setExecutable(Path path, boolean executable) throws IOException { + checkModifiable(); + FileSystem delegate = getDelegate(path); + delegate.setExecutable(adjustPath(path, delegate), executable); + } + + @Override + protected String getFastDigestFunctionType(Path path) { + FileSystem delegate = getDelegate(path); + return delegate.getFastDigestFunctionType(adjustPath(path, delegate)); + } + + @Override + protected byte[] getFastDigest(Path path) throws IOException { + FileSystem delegate = getDelegate(path); + return delegate.getFastDigest(adjustPath(path, delegate)); + } + + @Override + protected byte[] getxattr(Path path, String name, boolean followSymlinks) throws IOException { + FileSystem delegate = getDelegate(path); + return delegate.getxattr(adjustPath(path, delegate), name, followSymlinks); + } + + @Override + protected InputStream getInputStream(Path path) throws IOException { + FileSystem delegate = getDelegate(path); + return delegate.getInputStream(adjustPath(path, delegate)); + } + + @Override + protected OutputStream getOutputStream(Path path, boolean append) throws IOException { + checkModifiable(); + FileSystem delegate = getDelegate(path); + return delegate.getOutputStream(adjustPath(path, delegate), append); + } + + @Override + protected void renameTo(Path sourcePath, Path targetPath) throws IOException { + checkModifiable(); + FileSystem sourceDelegate = getDelegate(sourcePath); + if (!sourceDelegate.supportsModifications()) { + throw new UnsupportedOperationException( + "The filesystem for the source path " + + sourcePath.getPathString() + " does not support modifications."); + } + sourcePath = adjustPath(sourcePath, sourceDelegate); + + FileSystem targetDelegate = getDelegate(targetPath); + if (!targetDelegate.supportsModifications()) { + throw new UnsupportedOperationException( + "The filesystem for the target path " + + targetPath.getPathString() + " does not support modifications."); + } + targetPath = adjustPath(targetPath, targetDelegate); + + if (sourceDelegate == targetDelegate) { + // Easy, same filesystem. + sourceDelegate.renameTo(sourcePath, targetPath); + return; + } else { + // Copy across filesystems, then delete. + // copyFile throws on failure, so delete will never be reached if it fails. + FileSystemUtils.copyFile(sourcePath, targetPath); + sourceDelegate.delete(sourcePath); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/UnixFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/UnixFileSystem.java new file mode 100644 index 0000000..c7dd3a8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/UnixFileSystem.java
@@ -0,0 +1,414 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.unix.ErrnoFileStatus; +import com.google.devtools.build.lib.unix.FilesystemUtils; +import com.google.devtools.build.lib.unix.FilesystemUtils.Dirents; +import com.google.devtools.build.lib.unix.FilesystemUtils.ReadTypes; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * This class implements the FileSystem interface using direct calls to the + * UNIX filesystem. + */ +// Not final only for testing. +@ThreadSafe +public class UnixFileSystem extends AbstractFileSystem { + + public static final UnixFileSystem INSTANCE = new UnixFileSystem(); + /** + * Eager implementation of FileStatus for file systems that have an atomic + * stat(2) syscall. A proxy for {@link com.google.devtools.build.lib.unix.FileStatus}. + * Note that isFile and getLastModifiedTime have slightly different meanings + * between UNIX and VFS. + */ + @VisibleForTesting + protected static class UnixFileStatus implements FileStatus { + + private final com.google.devtools.build.lib.unix.FileStatus status; + + UnixFileStatus(com.google.devtools.build.lib.unix.FileStatus status) { + this.status = status; + } + + @Override + public boolean isFile() { return !isDirectory() && !isSymbolicLink(); } + + @Override + public boolean isDirectory() { return status.isDirectory(); } + + @Override + public boolean isSymbolicLink() { return status.isSymbolicLink(); } + + @Override + public long getSize() { return status.getSize(); } + + @Override + public long getLastModifiedTime() { + return (status.getLastModifiedTime() * 1000) + + (status.getFractionalLastModifiedTime() / 1000000); + } + + @Override + public long getLastChangeTime() { + return (status.getLastChangeTime() * 1000) + + (status.getFractionalLastChangeTime() / 1000000); + } + + @Override + public long getNodeId() { + // Note that we may want to include more information in this id number going forward, + // especially the device number. + return status.getInodeNumber(); + } + + int getPermissions() { return status.getPermissions(); } + + @Override + public String toString() { return status.toString(); } + } + + @Override + protected Collection<Path> getDirectoryEntries(Path path) throws IOException { + String name = path.getPathString(); + String[] entries; + long startTime = Profiler.nanoTimeMaybe(); + try { + entries = FilesystemUtils.readdir(name); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_DIR, name); + } + Collection<Path> result = new ArrayList<>(entries.length); + for (String entry : entries) { + result.add(path.getChild(entry)); + } + return result; + } + + @Override + protected PathFragment resolveOneLink(Path path) throws IOException { + // Beware, this seemingly simple code belies the complex specification of + // FileSystem.resolveOneLink(). + return stat(path, false).isSymbolicLink() + ? readSymbolicLink(path) + : null; + } + + /** + * Converts from {@link com.google.devtools.build.lib.unix.FilesystemUtils.Dirents.Type} to + * {@link com.google.devtools.build.lib.vfs.Dirent.Type}. + */ + private static Dirent.Type convertToDirentType(Dirents.Type type) { + switch (type) { + case FILE: + return Dirent.Type.FILE; + case DIRECTORY: + return Dirent.Type.DIRECTORY; + case SYMLINK: + return Dirent.Type.SYMLINK; + case UNKNOWN: + return Dirent.Type.UNKNOWN; + default: + throw new IllegalArgumentException("Unknown type " + type); + } + } + + @Override + protected Collection<Dirent> readdir(Path path, boolean followSymlinks) throws IOException { + String name = path.getPathString(); + long startTime = Profiler.nanoTimeMaybe(); + try { + Dirents unixDirents = FilesystemUtils.readdir(name, + followSymlinks ? ReadTypes.FOLLOW : ReadTypes.NOFOLLOW); + Preconditions.checkState(unixDirents.hasTypes()); + List<Dirent> dirents = Lists.newArrayListWithCapacity(unixDirents.size()); + for (int i = 0; i < unixDirents.size(); i++) { + dirents.add(new Dirent(unixDirents.getName(i), + convertToDirentType(unixDirents.getType(i)))); + } + return dirents; + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_DIR, name); + } + } + + @Override + protected UnixFileStatus stat(Path path, boolean followSymlinks) throws IOException { + String name = path.getPathString(); + long startTime = Profiler.nanoTimeMaybe(); + try { + return new UnixFileStatus(followSymlinks + ? FilesystemUtils.stat(name) + : FilesystemUtils.lstat(name)); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, name); + } + } + + // Like stat(), but returns null instead of throwing. + // This is a performance optimization in the case where clients + // catch and don't re-throw. + @Override + protected UnixFileStatus statNullable(Path path, boolean followSymlinks) { + String name = path.getPathString(); + long startTime = Profiler.nanoTimeMaybe(); + try { + ErrnoFileStatus stat = followSymlinks + ? FilesystemUtils.errnoStat(name) + : FilesystemUtils.errnoLstat(name); + return stat.hasError() ? null : new UnixFileStatus(stat); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, name); + } + } + + @Override + protected boolean exists(Path path, boolean followSymlinks) { + return statNullable(path, followSymlinks) != null; + } + + /** + * Return true iff the {@code stat} of {@code path} resulted in an {@code ENOENT} + * or {@code ENOTDIR} error. + */ + @Override + protected FileStatus statIfFound(Path path, boolean followSymlinks) throws IOException { + String name = path.getPathString(); + long startTime = Profiler.nanoTimeMaybe(); + try { + ErrnoFileStatus stat = followSymlinks + ? FilesystemUtils.errnoStat(name) + : FilesystemUtils.errnoLstat(name); + if (!stat.hasError()) { + return new UnixFileStatus(stat); + } + int errno = stat.getErrno(); + if (errno == ErrnoFileStatus.ENOENT || errno == ErrnoFileStatus.ENOTDIR) { + return null; + } + // This should not return -- we are calling stat here just to throw the proper exception. + // However, since there may be transient IO errors, we cannot guarantee that an exception will + // be thrown. + // TODO(bazel-team): Extract the exception-construction code and make it visible separately in + // FilesystemUtils to avoid having to do a duplicate stat call. + return stat(path, followSymlinks); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, name); + } + } + + @Override + protected boolean isDirectory(Path path, boolean followSymlinks) { + UnixFileStatus stat = statNullable(path, followSymlinks); + return stat != null && stat.isDirectory(); + } + + @Override + protected boolean isFile(Path path, boolean followSymlinks) { + // Note, FileStatus.isFile means *regular* file whereas Path.isFile may + // mean special file too, so we don't return FileStatus.isFile here. + UnixFileStatus status = statNullable(path, followSymlinks); + return status != null && !(status.isSymbolicLink() || status.isDirectory()); + } + + @Override + protected boolean isReadable(Path path) throws IOException { + return (stat(path, true).getPermissions() & 0400) != 0; + } + + @Override + protected boolean isWritable(Path path) throws IOException { + return (stat(path, true).getPermissions() & 0200) != 0; + } + + @Override + protected boolean isExecutable(Path path) throws IOException { + return (stat(path, true).getPermissions() & 0100) != 0; + } + + /** + * Adds or remove the bits specified in "permissionBits" to the permission + * mask of the file specified by {@code path}. If the argument {@code add} is + * true, the specified permissions are added, otherwise they are removed. + * + * @throws IOException if there was an error writing the file's metadata + */ + private void modifyPermissionBits(Path path, int permissionBits, boolean add) + throws IOException { + synchronized (path) { + int oldMode = stat(path, true).getPermissions(); + int newMode = add ? (oldMode | permissionBits) : (oldMode & ~permissionBits); + FilesystemUtils.chmod(path.toString(), newMode); + } + } + + @Override + protected void setReadable(Path path, boolean readable) throws IOException { + modifyPermissionBits(path, 0400, readable); + } + + @Override + protected void setWritable(Path path, boolean writable) throws IOException { + modifyPermissionBits(path, 0200, writable); + } + + @Override + protected void setExecutable(Path path, boolean executable) throws IOException { + modifyPermissionBits(path, 0111, executable); + } + + @Override + protected void chmod(Path path, int mode) throws IOException { + synchronized (path) { + FilesystemUtils.chmod(path.toString(), mode); + } + } + + @Override + public boolean supportsModifications() { + return true; + } + + @Override + public boolean supportsSymbolicLinks() { + return true; + } + + @Override + protected boolean createDirectory(Path path) throws IOException { + synchronized (path) { + // Note: UNIX mkdir(2), FilesystemUtils.mkdir() and createDirectory all + // have different ways of representing failure! + if (FilesystemUtils.mkdir(path.toString(), 0777)) { + return true; // successfully created + } + + // false => EEXIST: something is already in the way (file/dir/symlink) + if (isDirectory(path, false)) { + return false; // directory already existed + } else { + throw new IOException(path + " (File exists)"); + } + } + } + + @Override + protected void createSymbolicLink(Path linkPath, PathFragment targetFragment) + throws IOException { + synchronized (linkPath) { + FilesystemUtils.symlink(targetFragment.toString(), linkPath.toString()); + } + } + + @Override + protected PathFragment readSymbolicLink(Path path) throws IOException { + String name = path.toString(); + long startTime = Profiler.nanoTimeMaybe(); + try { + return new PathFragment(FilesystemUtils.readlink(name)); + } catch (IOException e) { + // EINVAL => not a symbolic link. Anything else is a real error. + throw e.getMessage().endsWith("(Invalid argument)") ? new NotASymlinkException(path) : e; + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_LINK, name); + } + } + + @Override + protected void renameTo(Path sourcePath, Path targetPath) throws IOException { + synchronized (sourcePath) { + FilesystemUtils.rename(sourcePath.toString(), targetPath.toString()); + } + } + + @Override + protected long getFileSize(Path path, boolean followSymlinks) throws IOException { + return stat(path, followSymlinks).getSize(); + } + + @Override + protected boolean delete(Path path) throws IOException { + String name = path.toString(); + long startTime = Profiler.nanoTimeMaybe(); + synchronized (path) { + try { + return FilesystemUtils.remove(name); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_DELETE, name); + } + } + } + + @Override + protected long getLastModifiedTime(Path path, boolean followSymlinks) throws IOException { + return stat(path, followSymlinks).getLastModifiedTime(); + } + + @Override + protected boolean isSymbolicLink(Path path) { + UnixFileStatus stat = statNullable(path, false); + return stat != null && stat.isSymbolicLink(); + } + + @Override + protected void setLastModifiedTime(Path path, long newTime) throws IOException { + synchronized (path) { + if (newTime == -1L) { // "now" + FilesystemUtils.utime(path.toString(), true, 0, 0); + } else { + // newTime > MAX_INT => -ve unixTime + int unixTime = (int) (newTime / 1000); + FilesystemUtils.utime(path.toString(), false, unixTime, unixTime); + } + } + } + + @Override + protected byte[] getxattr(Path path, String name, boolean followSymlinks) throws IOException { + String pathName = path.toString(); + long startTime = Profiler.nanoTimeMaybe(); + try { + return followSymlinks + ? FilesystemUtils.getxattr(pathName, name) : FilesystemUtils.lgetxattr(pathName, name); + } catch (UnsupportedOperationException e) { + // getxattr() syscall is not supported by the underlying filesystem (it returned ENOTSUP). + // Per method contract, treat this as ENODATA. + return null; + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_XATTR, pathName); + } + } + + @Override + protected byte[] getMD5Digest(Path path) throws IOException { + String name = path.toString(); + long startTime = Profiler.nanoTimeMaybe(); + try { + return FilesystemUtils.md5sum(name).asBytes(); + } finally { + profiler.logSimpleTask(startTime, ProfilerTask.VFS_MD5, name); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java b/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java new file mode 100644 index 0000000..d512abc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/UnixGlob.java
@@ -0,0 +1,785 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.vfs; + +import com.google.common.base.Joiner; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.base.Splitter; +import com.google.common.base.Throwables; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.AbstractFuture; +import com.google.common.util.concurrent.Futures; +import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; + +/** + * Implementation of a subset of UNIX-style file globbing, expanding "*" and "?" as wildcards, but + * not [a-z] ranges. + * + * <p><code>**</code> gets special treatment in include patterns. If it is used as a complete path + * segment it matches the filenames in subdirectories recursively. + */ +public final class UnixGlob { + private UnixGlob() {} + + private static List<Path> globInternal(Path base, Collection<String> patterns, + Collection<String> excludePatterns, + boolean excludeDirectories, + Predicate<Path> dirPred, + boolean checkForInterruption, + FilesystemCalls syscalls, + ThreadPoolExecutor threadPool) + throws IOException, InterruptedException { + GlobVisitor visitor = (threadPool == null) + ? new GlobVisitor(checkForInterruption) + : new GlobVisitor(threadPool, checkForInterruption); + return visitor.glob(base, patterns, excludePatterns, excludeDirectories, dirPred, syscalls); + } + + private static Future<List<Path>> globAsyncInternal(Path base, Collection<String> patterns, + Collection<String> excludePatterns, + boolean excludeDirectories, + Predicate<Path> dirPred, + FilesystemCalls syscalls, + boolean checkForInterruption, + ThreadPoolExecutor threadPool) { + GlobVisitor visitor = (threadPool == null) + ? new GlobVisitor(checkForInterruption) + : new GlobVisitor(threadPool, checkForInterruption); + return visitor.globAsync(base, patterns, excludePatterns, excludeDirectories, dirPred, + syscalls); + } + + /** + * Checks that each pattern is valid, splits it into segments and checks + * that each segment contains only valid wildcards. + * + * @return list of segment arrays + */ + private static List<String[]> checkAndSplitPatterns(Collection<String> patterns) { + List<String[]> list = Lists.newArrayListWithCapacity(patterns.size()); + for (String pattern : patterns) { + String error = checkPatternForError(pattern); + if (error != null) { + throw new IllegalArgumentException(error + " (in glob pattern '" + pattern + "')"); + } + Iterable<String> segments = Splitter.on('/').split(pattern); + list.add(Iterables.toArray(segments, String.class)); + } + return list; + } + + /** + * @return whether or not {@code pattern} contains illegal characters + */ + public static String checkPatternForError(String pattern) { + if (pattern.isEmpty()) { + return "pattern cannot be empty"; + } + if (pattern.charAt(0) == '/') { + return "pattern cannot be absolute"; + } + for (int i = 0; i < pattern.length(); i++) { + char c = pattern.charAt(i); + switch (c) { + case '(': case ')': + case '{': case '}': + case '[': case ']': + return "illegal character '" + c + "'"; + } + } + Iterable<String> segments = Splitter.on('/').split(pattern); + for (String segment : segments) { + if (segment.isEmpty()) { + return "empty segment not permitted"; + } + if (segment.equals(".") || segment.equals("..")) { + return "segment '" + segment + "' not permitted"; + } + if (segment.contains("**") && !segment.equals("**")) { + return "recursive wildcard must be its own segment"; + } + } + return null; + } + + private static boolean excludedOnMatch(Path path, List<String[]> excludePatterns, + int idx, Cache<String, Pattern> cache, + Predicate<Path> dirPred) { + for (String[] excludePattern : excludePatterns) { + String text = path.getBaseName(); + if (idx == excludePattern.length + && matches(excludePattern[idx - 1], text, cache)) { + return true; + } + } + return false; + } + + /** + * Returns the exclude patterns in {@code excludePatterns} which could + * apply to the children of {@code base} + * + * @param idx index into {@code excludePatterns} for the part of the pattern + * which might match {@code base} + */ + private static List<String[]> getRelevantExcludes( + final Path base, List<String[]> excludePatterns, final int idx, + final Cache<String, Pattern> cache) { + if (excludePatterns.isEmpty()) { + return excludePatterns; + } + List<String[]> list = new ArrayList<>(); + for (String[] patterns : excludePatterns) { + if (excludePatternMatches(patterns, idx, base, cache)) { + list.add(patterns); + } + } + return list; + } + + /** + * @param patterns a list of patterns + * @param idx index into {@code patterns} + */ + private static boolean excludePatternMatches(String[] patterns, int idx, + Path base, + Cache<String, Pattern> cache) { + if (idx == 0) { + return true; + } + String text = base.getBaseName(); + return patterns.length > idx && matches(patterns[idx - 1], text, cache); + } + + /** + * Calls {@link #matches(String, String, Cache) matches(pattern, str, null)} + */ + public static boolean matches(String pattern, String str) { + return matches(pattern, str, null); + } + + /** + * Returns whether {@code str} matches the glob pattern {@code pattern}. This + * method may use the {@code patternCache} to speed up the matching process. + * + * @param pattern a glob pattern + * @param str the string to match + * @param patternCache a cache from patterns to compiled Pattern objects, or + * {@code null} to skip caching + */ + public static boolean matches(String pattern, String str, + Cache<String, Pattern> patternCache) { + if (pattern.length() == 0 || str.length() == 0) { + return false; + } + + // Common case: ** + if (pattern.equals("**")) { + return true; + } + + // Common case: * + if (pattern.equals("*")) { + return true; + } + + // If a filename starts with '.', this char must be matched explicitly. + if (str.charAt(0) == '.' && pattern.charAt(0) != '.') { + return false; + } + + // Common case: *.xyz + if (pattern.charAt(0) == '*' && pattern.lastIndexOf('*') == 0) { + return str.endsWith(pattern.substring(1)); + } + // Common case: xyz* + int lastIndex = pattern.length() - 1; + // The first clause of this if statement is unnecessary, but is an + // optimization--charAt runs faster than indexOf. + if (pattern.charAt(lastIndex) == '*' && pattern.indexOf('*') == lastIndex) { + return str.startsWith(pattern.substring(0, lastIndex)); + } + + Pattern regex = patternCache == null ? null : patternCache.getIfPresent(pattern); + if (regex == null) { + regex = makePatternFromWildcard(pattern); + if (patternCache != null) { + patternCache.put(pattern, regex); + } + } + return regex.matcher(str).matches(); + } + + /** + * Returns a regular expression implementing a matcher for "pattern", in which + * "*" and "?" are wildcards. + * + * <p>e.g. "foo*bar?.java" -> "foo.*bar.\\.java" + */ + private static Pattern makePatternFromWildcard(String pattern) { + StringBuilder regexp = new StringBuilder(); + for(int i = 0, len = pattern.length(); i < len; i++) { + char c = pattern.charAt(i); + switch(c) { + case '*': + regexp.append(".*"); + break; + case '?': + regexp.append('.'); + break; + //escape the regexp special characters that are allowed in wildcards + case '^': case '$': case '|': case '+': + case '{': case '}': case '[': case ']': + case '\\': case '.': + regexp.append('\\'); + regexp.append(c); + break; + default: + regexp.append(c); + break; + } + } + return Pattern.compile(regexp.toString()); + } + + /** + * Filesystem calls required for glob(). + */ + public static interface FilesystemCalls { + /** + * Get directory entries and their types. + */ + Collection<Dirent> readdir(Path path, Symlinks symlinks) throws IOException; + + /** + * Return the stat() for the given path, or null. + */ + FileStatus statNullable(Path path, Symlinks symlinks); + } + + public static FilesystemCalls DEFAULT_SYSCALLS = new FilesystemCalls() { + @Override + public Collection<Dirent> readdir(Path path, Symlinks symlinks) throws IOException { + return path.readdir(symlinks); + } + + @Override + public FileStatus statNullable(Path path, Symlinks symlinks) { + return path.statNullable(symlinks); + } + }; + + public static final AtomicReference<FilesystemCalls> DEFAULT_SYSCALLS_REF = + new AtomicReference<FilesystemCalls>(DEFAULT_SYSCALLS); + + public static Builder forPath(Path path) { + return new Builder(path); + } + + /** + * Builder class for UnixGlob. + * + * + */ + public static class Builder { + private Path base; + private List<String> patterns; + private List<String> excludes; + private boolean excludeDirectories; + private Predicate<Path> pathFilter; + private ThreadPoolExecutor threadPool; + private AtomicReference<? extends FilesystemCalls> syscalls = + new AtomicReference<>(DEFAULT_SYSCALLS); + + /** + * Creates a glob builder with the given base path. + */ + public Builder(Path base) { + this.base = base; + this.patterns = Lists.newArrayList(); + this.excludes = Lists.newArrayList(); + this.excludeDirectories = false; + this.pathFilter = Predicates.alwaysTrue(); + } + + /** + * Adds a pattern to include to the glob builder. + * + * <p>For a description of the syntax of the patterns, see {@link UnixGlob}. + */ + public Builder addPattern(String pattern) { + this.patterns.add(pattern); + return this; + } + + /** + * Adds a pattern to include to the glob builder. + * + * <p>For a description of the syntax of the patterns, see {@link UnixGlob}. + */ + public Builder addPatterns(String... patterns) { + for (String pattern : patterns) { + this.patterns.add(pattern); + } + return this; + } + + /** + * Adds a pattern to include to the glob builder. + * + * <p>For a description of the syntax of the patterns, see {@link UnixGlob}. + */ + public Builder addPatterns(Collection<String> patterns) { + this.patterns.addAll(patterns); + return this; + } + + /** + * Sets the FilesystemCalls interface to use on this glob(). + */ + public Builder setFilesystemCalls(AtomicReference<? extends FilesystemCalls> syscalls) { + this.syscalls = (syscalls == null) + ? new AtomicReference<FilesystemCalls>(DEFAULT_SYSCALLS) + : syscalls; + return this; + } + + /** + * Adds patterns to exclude from the results to the glob builder. + * + * <p>For a description of the syntax of the patterns, see {@link UnixGlob}. + */ + public Builder addExcludes(String... excludes) { + this.excludes.addAll(Arrays.asList(excludes)); + return this; + } + + /** + * Adds patterns to exclude from the results to the glob builder. + * + * <p>For a description of the syntax of the patterns, see {@link UnixGlob}. + */ + public Builder addExcludes(Collection<String> excludes) { + this.excludes.addAll(excludes); + return this; + } + + /** + * If set to true, directories are not returned in the glob result. + */ + public Builder setExcludeDirectories(boolean excludeDirectories) { + this.excludeDirectories = excludeDirectories; + return this; + } + + + /** + * Sets the threadpool to use for parallel glob evaluation. + * If unset, evaluation is done in-thread. + */ + public Builder setThreadPool(ThreadPoolExecutor pool) { + this.threadPool = pool; + return this; + } + + + /** + * If set, the given predicate is called for every directory + * encountered. If it returns false, the corresponding item is not + * returned in the output and directories are not traversed either. + */ + public Builder setDirectoryFilter(Predicate<Path> pathFilter) { + this.pathFilter = pathFilter; + return this; + } + + /** + * Executes the glob. + */ + public List<Path> glob() throws IOException { + try { + return globInternal(base, patterns, excludes, excludeDirectories, pathFilter, false, + syscalls.get(), threadPool); + } catch (InterruptedException e) { + // cannot happen, since we told globInternal not to throw + throw new IllegalStateException(e); + } + } + + /** + * Executes the glob. + * + * @throws InterruptedException if the thread is interrupted. + */ + public List<Path> globInterruptible() throws IOException, InterruptedException { + return globInternal(base, patterns, excludes, excludeDirectories, pathFilter, true, + syscalls.get(), threadPool); + } + + /** + * Executes the glob asynchronously. + * + * @param checkForInterrupt if the returned future may throw + * InterruptedException. + */ + public Future<List<Path>> globAsync(boolean checkForInterrupt) { + return globAsyncInternal(base, patterns, excludes, excludeDirectories, pathFilter, + syscalls.get(), checkForInterrupt, threadPool); + } + } + + /** + * Adapts the result of the glob visitation as a Future. + */ + private static class GlobFuture extends AbstractFuture<List<Path>> { + private final GlobVisitor visitor; + private final boolean checkForInterrupt; + private final Object completionLock = new Object(); + + public GlobFuture(GlobVisitor visitor, boolean checkForInterrupt) { + this.visitor = visitor; + this.checkForInterrupt = checkForInterrupt; + } + + private List<Path> getSafe() throws InterruptedException, ExecutionException { + boolean interrupted = false; + try { + while (true) { + try { + return super.get(); + } catch (InterruptedException e) { + if (checkForInterrupt) { + throw e; + } + interrupted = true; + } catch (ExecutionException e) { + // The checkForInterrupt logic is already handled in + // GlobVisitor#waitForCompletion(). + Throwables.propagateIfInstanceOf(e.getCause(), InterruptedException.class); + throw e; + } + } + } finally { + if (!checkForInterrupt && interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + @Override + public List<Path> get() throws InterruptedException, ExecutionException { + synchronized (completionLock) { + if (isDone()) { + return getSafe(); + } + + try { + visitor.waitForCompletion(); + super.set(Lists.newArrayList(visitor.results)); + } catch (Throwable t) { + super.setException(t); + } + List<Path> result = getSafe(); + return result; + } + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + synchronized (completionLock) { + if (isDone()) { + return false; + } + + visitor.interrupt(); + return true; + } + } + } + + /** + * GlobVisitor executes a glob using parallelism, which is useful when + * the glob() requires many readdir() calls on high latency filesystems. + */ + private static final class GlobVisitor extends AbstractQueueVisitor { + // These collections are used across workers and must therefore be + // thread-safe. + + private final static String THREAD_NAME = "GlobVisitor"; + + private final Collection<Path> results = + Collections.synchronizedSet(Sets.<Path>newTreeSet()); + private final Cache<String, Pattern> cache = CacheBuilder.newBuilder().build( + new CacheLoader<String, Pattern>() { + @Override + public Pattern load(String wildcard) { + return makePatternFromWildcard(wildcard); + } + }); + + private final GlobFuture result; + private final boolean failFastOnInterrupt; + + public GlobVisitor(ThreadPoolExecutor executor, boolean failFastOnInterrupt) { + super(executor, /*shutdownOnCompletion=*/false, /*failFastOnException=*/true, + /*failFastOnInterrupt=*/failFastOnInterrupt); + this.result = new GlobFuture(this, failFastOnInterrupt); + this.failFastOnInterrupt = failFastOnInterrupt; + } + + public GlobVisitor(boolean failFastOnInterrupt) { + super(/*concurrent=*/false, 0, 0, 0, null, /*failFastOnException=*/true, + /*failFastOnInterrupt=*/failFastOnInterrupt, THREAD_NAME); + this.result = new GlobFuture(this, failFastOnInterrupt); + this.failFastOnInterrupt = failFastOnInterrupt; + } + + /** + * Performs wildcard globbing: returns the sorted list of filenames that match any of + * {@code patterns} relative to {@code base}, but which do not match {@code excludePatterns}. + * Directories are traversed if and only if they match {@code dirPred}. The predicate is also + * called for the root of the traversal. + * + * <p>Patterns may include "*" and "?", but not "[a-z]". + * + * <p><code>**</code> gets special treatment in include patterns. If it is + * used as a complete path segment it matches the filenames in + * subdirectories recursively. + * + * @throws IllegalArgumentException if any glob or exclude pattern + * {@linkplain #checkPatternForError(String) contains errors} or if + * any exclude pattern segment contains <code>**</code> or if any + * include pattern segment contains <code>**</code> but not equal to + * it. + */ + public List<Path> glob(Path base, Collection<String> patterns, + Collection<String> excludePatterns, boolean excludeDirectories, + Predicate<Path> dirPred, FilesystemCalls syscalls) + throws IOException, InterruptedException { + try { + return globAsync(base, patterns, excludePatterns, excludeDirectories, + dirPred, syscalls).get(); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + Throwables.propagateIfPossible(cause, IOException.class); + throw new RuntimeException(e); + } + } + + public Future<List<Path>> globAsync(Path base, Collection<String> patterns, + Collection<String> excludePatterns, boolean excludeDirectories, + Predicate<Path> dirPred, FilesystemCalls syscalls) { + + FileStatus baseStat = syscalls.statNullable(base, Symlinks.FOLLOW); + if (baseStat == null || patterns.isEmpty()) { + return Futures.immediateFuture(Collections.<Path>emptyList()); + } + + List<String[]> splitPatterns = checkAndSplitPatterns(patterns); + List<String[]> splitExcludes = checkAndSplitPatterns(excludePatterns); + + // We do a dumb loop, even though it will likely duplicate work + // (e.g., readdir calls). In order to optimize, we would need + // to keep track of which patterns shared sub-patterns and which did not + // (for example consider the glob [*/*.java, sub/*.java, */*.txt]). + for (String[] splitPattern : splitPatterns) { + queueGlob(base, baseStat.isDirectory(), splitPattern, 0, excludeDirectories, + splitExcludes, 0, results, cache, dirPred, syscalls); + } + + return result; + } + + protected void waitForCompletion() throws IOException, InterruptedException { + try { + super.work(failFastOnInterrupt); + } catch (InterruptedException e) { + if (failFastOnInterrupt) { + throw e; + } else { + Thread.currentThread().interrupt(); + } + } catch (IORuntimeException e) { + if (Thread.interrupted()) { + // As per the contract of AbstractQueueVisitor#work, if an unchecked exception is thrown + // and the build is interrupted, the thrown exception is what will be rethrown. Since the + // user presumably wanted to interrupt the build, we ignore the thrown IORuntimeException + // (which doesn't indicate a programming bug) and throw an InterruptedException. + if (failFastOnInterrupt) { + throw new InterruptedException(); + } + Thread.currentThread().interrupt(); + } + throw e.getCauseIOException(); + } + } + + private void queueGlob(final Path base, final boolean baseIsDir, + final String[] patternParts, final int idx, + final boolean excludeDirectories, + final List<String[]> excludePatterns, + final int excludeIdx, + final Collection<Path> results, final Cache<String, Pattern> cache, + final Predicate<Path> dirPred, final FilesystemCalls syscalls) { + enqueue(new Runnable() { + @Override + public void run() { + Profiler.instance().startTask(ProfilerTask.VFS_GLOB, this); + try { + reallyGlob(base, baseIsDir, patternParts, idx, excludeDirectories, + excludePatterns, excludeIdx, results, cache, dirPred, syscalls); + } catch (IOException e) { + throw new IORuntimeException(e); + } catch (InterruptedException e) { + // When we get to this point, the main thread already knows that the + // globbing has been interrupted, so we do not need to report the + // error condition. + } finally { + Profiler.instance().completeTask(ProfilerTask.VFS_GLOB); + } + } + + @Override + public String toString() { + return String.format( + "%s glob(include=[%s], exclude=[%s], exclude_directories=%s)", + base.getPathString(), + "\"" + Joiner.on("\", \"").join(patternParts) + "\"", + "\"" + Joiner.on("\", \"").join(excludePatterns) + "\"", + excludeDirectories); + } + }); + + } + + /** + * Expressed in Haskell: + * <pre> + * reallyGlob base [] = { base } + * reallyGlob base [x:xs] = union { reallyGlob(f, xs) | f results "base/x" } + * </pre> + */ + private void reallyGlob(Path base, boolean baseIsDir, String[] patternParts, int idx, + boolean excludeDirectories, + List<String[]> excludePatterns, + int excludeIdx, + Collection<Path> results, Cache<String, Pattern> cache, + Predicate<Path> dirPred, + FilesystemCalls syscalls) throws IOException, InterruptedException { + if (failFastOnInterrupt && Thread.interrupted()) { + throw new InterruptedException(); + } + + if (baseIsDir && !dirPred.apply(base)) { + return; + } + + if (idx == patternParts.length) { // Base case. + if (!(excludeDirectories && baseIsDir) && + !excludedOnMatch(base, excludePatterns, excludeIdx, cache, dirPred)) { + results.add(base); + } + + return; + } + + if (!baseIsDir) { + // Nothing to find here. + return; + } + + List<String[]> relevantExcludes + = getRelevantExcludes(base, excludePatterns, excludeIdx, cache); + final String pattern = patternParts[idx]; + + // ** is special: it can match nothing at all. + // For example, x/** matches x, **/y matches y, and x/**/y matches x/y. + if ("**".equals(pattern)) { + queueGlob(base, baseIsDir, patternParts, idx + 1, excludeDirectories, + excludePatterns, excludeIdx, results, cache, dirPred, syscalls); + } + + if (!pattern.contains("*") && !pattern.contains("?")) { + // We do not need to do a readdir in this case, just a stat. + Path child = base.getChild(pattern); + FileStatus status = syscalls.statNullable(child, Symlinks.FOLLOW); + if (status == null || (!status.isDirectory() && !status.isFile())) { + // The file is a dangling symlink, fifo, does not exist, etc. + return; + } + + boolean childIsDir = status.isDirectory(); + + queueGlob(child, childIsDir, patternParts, idx + 1, excludeDirectories, + relevantExcludes, excludeIdx + 1, results, cache, dirPred, syscalls); + return; + } + + Collection<Dirent> dents = syscalls.readdir(base, Symlinks.FOLLOW); + + for (Dirent dent : dents) { + Dirent.Type type = dent.getType(); + if (type == Dirent.Type.UNKNOWN) { + // The file is a dangling symlink, fifo, etc. + continue; + } + boolean childIsDir = (type == Dirent.Type.DIRECTORY); + String text = dent.getName(); + Path child = base.getChild(text); + + if ("**".equals(pattern)) { + // Recurse without shifting the pattern. + if (childIsDir) { + queueGlob(child, childIsDir, patternParts, idx, excludeDirectories, + relevantExcludes, excludeIdx + 1, results, cache, dirPred, syscalls); + } + } + if (matches(pattern, text, cache)) { + // Recurse and consume one segment of the pattern. + if (childIsDir) { + queueGlob(child, childIsDir, patternParts, idx + 1, excludeDirectories, + relevantExcludes, excludeIdx + 1, results, cache, dirPred, syscalls); + } else { + // Instead of using an async call, just repeat the base case above. + if (idx + 1 == patternParts.length && + !excludedOnMatch(child, relevantExcludes, excludeIdx + 1, cache, dirPred)) { + results.add(child); + } + } + } + } + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/ZipFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/ZipFileSystem.java new file mode 100644 index 0000000..558263d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/ZipFileSystem.java
@@ -0,0 +1,253 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.common.base.Predicate; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * A FileSystem that provides a read-only filesystem view on a zip file. + * Inherits the constraints imposed by ReadonlyFileSystem. + */ +@ThreadSafe +public class ZipFileSystem extends ReadonlyFileSystem { + + private final ZipFile zipFile; + + /** + * The sole purpose of this field is to hold a strong reference to all leaf + * {@link Path}s which have a non-null "entry" field, preventing them from + * being garbage-collected. (The leaf paths hold string references to their + * parents, so we don't need to include them here.) + * + * <p>This is necessary because {@link Path}s may be recycled when they + * become unreachable, but the ZipFileSystem uses them to hold the {@link + * ZipEntry} for that path, if any. Without this additional strong + * reference, ZipEntries would seem to "disappear" during garbage collection. + */ + @SuppressWarnings("unused") + private final Object paths; + + /** + * Constructs a ZipFileSystem from a zip file identified with a given path. + */ + public ZipFileSystem(Path zipPath) throws IOException { + // Throw some more specific exceptions than ZipFile does. + // We do this using File instead of Path, in case zipPath points to an + // InMemoryFileSystem. This case is not really supported but + // can occur in tests. + File file = zipPath.getPathFile(); + if (!file.exists()) { + throw new FileNotFoundException(String.format("File '%s' does not exist", zipPath)); + } + if (!file.isFile()) { + throw new IOException(String.format("'%s' is not a file", zipPath)); + } + if (!file.canRead()) { + throw new IOException(String.format("File '%s' is not readable", zipPath)); + } + + this.zipFile = new ZipFile(file); + this.paths = populatePathTree(); + } + + // ZipPath extends Path with a set-once ZipEntry field. + // TODO(bazel-team): (2009) Delete class ZipPath, and perform the + // Path-to-ZipEntry lookup in {@link #zipEntry} and {@link + // #getDirectoryEntries}. Then this field becomes redundant. + @ThreadSafe + private static class ZipPath extends Path { + /** + * Non-null iff this file/directory exists. Set by setZipEntry for files + * explicitly mentioned in the zipfile's table of contents, or implicitly + * an ancestor of them. + */ + ZipEntry entry = null; + + // Root path. + ZipPath(ZipFileSystem fileSystem) { + super(fileSystem); + } + + // Non-root paths. + ZipPath(ZipFileSystem fileSystem, String name, ZipPath parent) { + super(fileSystem, name, parent); + } + + void setZipEntry(ZipEntry entry) { + if (this.entry != null) { + throw new IllegalStateException("setZipEntry(" + entry + + ") called twice!"); + } + this.entry = entry; + + // Ensure all parents of this path have a directory ZipEntry: + for (ZipPath path = (ZipPath) getParentDirectory(); + path != null && path.entry == null; + path = (ZipPath) path.getParentDirectory()) { + // Note, the ZipEntry for the root path is called "//", but that's ok. + path.setZipEntry(new ZipEntry(path + "/")); // trailing "/" => isDir + } + } + + @Override + protected ZipPath createChildPath(String childName) { + return new ZipPath((ZipFileSystem) getFileSystem(), childName, this); + } + } + + /** + * Scans the Zip file and associates a ZipEntry with each filename + * (ZipPath) that is mentioned in the table of contents. Returns a + * collection of all corresponding Paths. + */ + private Collection<Path> populatePathTree() { + Collection<Path> paths = new ArrayList<>(); + for (ZipEntry entry : Collections.list(zipFile.entries())) { + PathFragment frag = new PathFragment(entry.getName()); + Path path = rootPath.getRelative(frag); + paths.add(path); + ((ZipPath) path).setZipEntry(entry); + } + return paths; + } + + @Override + public String getFileSystemType(Path path) { + return "zipfs"; + } + + @Override + protected Path createRootPath() { + return new ZipPath(this); + } + + /** Returns the ZipEntry associated with a given path name, if any. */ + private static ZipEntry zipEntry(Path path) { + return ((ZipPath) path).entry; + } + + /** Like zipEntry, but throws FileNotFoundException unless path exists. */ + private static ZipEntry zipEntryNonNull(Path path) + throws FileNotFoundException { + ZipEntry zipEntry = zipEntry(path); + if (zipEntry == null) { + throw new FileNotFoundException(path + " (No such file or directory)"); + } + return zipEntry; + } + + @Override + protected InputStream getInputStream(Path path) throws IOException { + return zipFile.getInputStream(zipEntryNonNull(path)); + } + + @Override + protected Collection<Path> getDirectoryEntries(Path path) + throws IOException { + zipEntryNonNull(path); + final Collection<Path> result = new ArrayList<>(); + ((ZipPath) path).applyToChildren(new Predicate<Path>() { + @Override + public boolean apply(Path child) { + if (zipEntry(child) != null) { + result.add(child); + } + return true; + } + }); + return result; + } + + @Override + protected boolean exists(Path path, boolean followSymlinks) { + return zipEntry(path) != null; + } + + @Override + protected boolean isDirectory(Path path, boolean followSymlinks) { + ZipEntry entry = zipEntry(path); + return entry != null && entry.isDirectory(); + } + + @Override + protected boolean isFile(Path path, boolean followSymlinks) { + ZipEntry entry = zipEntry(path); + return entry != null && !entry.isDirectory(); + } + + @Override + protected boolean isReadable(Path path) throws IOException { + zipEntryNonNull(path); + return true; + } + + @Override + protected boolean isWritable(Path path) throws IOException { + zipEntryNonNull(path); + return false; + } + + @Override + protected boolean isExecutable(Path path) throws IOException { + zipEntryNonNull(path); + return false; + } + + @Override + protected PathFragment readSymbolicLink(Path path) throws IOException { + zipEntryNonNull(path); + throw new NotASymlinkException(path); + } + + @Override + protected long getFileSize(Path path, boolean followSymlinks) + throws IOException { + return zipEntryNonNull(path).getSize(); + } + + @Override + protected long getLastModifiedTime(Path path, boolean followSymlinks) + throws FileNotFoundException { + return zipEntryNonNull(path).getTime(); + } + + @Override + protected boolean isSymbolicLink(Path path) { + return false; + } + + @Override + protected FileStatus statIfFound(Path path, boolean followSymlinks) { + try { + return stat(path, followSymlinks); + } catch (FileNotFoundException e) { + return null; + } catch (IOException e) { + // getLastModifiedTime can only throw FileNotFoundException, which is what stat uses. + throw new IllegalStateException (e); + } + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/FileInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/FileInfo.java new file mode 100644 index 0000000..fff562f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/FileInfo.java
@@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.inmemoryfs; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.util.Clock; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * This interface represents a mutable file stored in an InMemoryFileSystem. + */ +@ThreadSafe +public abstract class FileInfo extends InMemoryContentInfo { + protected FileInfo(Clock clock) { + super(clock); + } + + @Override + public boolean isDirectory() { + return false; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public boolean isFile() { + return true; + } + + protected abstract byte[] readContent() throws IOException; + + protected abstract OutputStream getOutputStream(boolean append) throws IOException; +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfo.java new file mode 100644 index 0000000..0e7de71 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfo.java
@@ -0,0 +1,212 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.inmemoryfs; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; + +/** + * This interface defines the function directly supported by the "files" stored + * in a InMemoryFileSystem. This corresponds to a file or inode in UNIX: it + * doesn't have a path (it could have many paths due to hard links, or none if + * it's unlinked, i.e. garbage). + * + * <p>This class is thread-safe: instances may be accessed and modified from + * concurrent threads. Subclasses must preserve this property. + */ +@ThreadSafe +public abstract class InMemoryContentInfo implements ScopeEscapableStatus { + + private final Clock clock; + + /** + * Stores the time when the file was last modified. This is atomically updated + * whenever the file changes, so all accesses must be synchronized. + */ + private long lastModifiedTime; + + /** + * Stores the time when the file information was changed. This is atomically updated + * whenever the file changes, so all accesses must be synchronized. + */ + private long lastChangeTime; + + /** + * Modifications to the isWritable field do not update the lastModifiedTime, + * so we don't need to synchronize; using volatile is enough. + */ + private volatile boolean isWritable = true; + private volatile boolean isExecutable = false; + private volatile boolean isReadable = true; + + protected InMemoryContentInfo(Clock clock) { + this(clock, true); + } + + protected InMemoryContentInfo(Clock clock, boolean isMutable) { + this.clock = clock; + // When we create the file, it is modified. + if (isMutable) { + markModificationTime(); + } + } + + /** + * Returns true if the current object is a directory. + */ + @Override + public abstract boolean isDirectory(); + + /** + * Returns true if the current object is a symbolic link. + */ + @Override + public abstract boolean isSymbolicLink(); + + /** + * Returns true if the current object is a regular file. + */ + @Override + public abstract boolean isFile(); + + /** + * Returns the size of the entity denoted by the current object. For files, + * this is the length in bytes, for directories the number of children. The + * size of links is unspecified. + */ + @Override + public abstract long getSize() throws IOException; + + /** + * Returns the time when the entity denoted by the current object was last + * modified. + */ + @Override + public synchronized long getLastModifiedTime() { + return lastModifiedTime; + } + + /** + * Returns the time when the entity denoted by the current object was last + * changed. + */ + @Override + public synchronized long getLastChangeTime() { + return lastChangeTime; + } + + /** + * Returns the file node id for the given instance, emulated by the + * identity hash code. + */ + @Override + public long getNodeId() { + return System.identityHashCode(this); + } + + /** + * Sets the time that denotes when the entity denoted by this object was last + * modified. + */ + synchronized void setLastModifiedTime(long newTime) { + lastModifiedTime = newTime; + markChangeTime(); + } + + /** + * Sets the last modification and change times to the current time. + */ + protected synchronized void markModificationTime() { + Preconditions.checkState(clock != null); + lastModifiedTime = clock.currentTimeMillis(); + lastChangeTime = lastModifiedTime; + } + + /** + * Sets the last change time to the current time. + */ + protected synchronized void markChangeTime() { + Preconditions.checkState(clock != null); + lastChangeTime = clock.currentTimeMillis(); + } + + /** + * Sets whether the current file is readable. + */ + boolean isReadable() { + return isReadable; + } + + /** + * Returns whether the current file is readable. + */ + void setReadable(boolean readable) { + isReadable = readable; + } + + + /** + * Sets whether the current file is writable. + */ + void setWritable(boolean writable) { + isWritable = writable; + markChangeTime(); + } + + /** + * Returns whether the current file is writable. + */ + boolean isWritable() { + return isWritable; + } + + /** + * Sets whether the current file is executable. + */ + void setExecutable(boolean executable) { + isExecutable = executable; + markChangeTime(); + } + + /** + * Returns whether the current file is executable. + */ + boolean isExecutable() { + return isExecutable; + } + + @Override + public boolean outOfScope() { + return false; + } + + @Override + public PathFragment getEscapingPath() { + return null; + } + + /** + * Called just before this inode is moved. + * + * @param targetPath where the inode is relocated. + * @throws IOException + */ + protected void movedTo(Path targetPath) throws IOException { + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryDirectoryInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryDirectoryInfo.java new file mode 100644 index 0000000..400490b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryDirectoryInfo.java
@@ -0,0 +1,108 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.inmemoryfs; + +import com.google.common.collect.MapMaker; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.util.Clock; + +import java.util.Set; +import java.util.concurrent.ConcurrentMap; + +/** + * This class represents a directory stored in an {@link InMemoryFileSystem}. + */ +@ThreadSafe +class InMemoryDirectoryInfo extends InMemoryContentInfo { + + private final ConcurrentMap<String, InMemoryContentInfo> directoryContent = + new MapMaker().makeMap(); + + InMemoryDirectoryInfo(Clock clock) { + this(clock, true); + } + + protected InMemoryDirectoryInfo(Clock clock, boolean isMutable) { + super(clock, isMutable); + if (isMutable) { + setExecutable(true); + } + } + + /** + * Adds a new child to this directory under the name "name". Callers must + * ensure that no entry of that name exists already. + */ + synchronized void addChild(String name, InMemoryContentInfo inode) { + if (name == null) { throw new NullPointerException(); } + if (inode == null) { throw new NullPointerException(); } + if (directoryContent.put(name, inode) != null) { + throw new IllegalArgumentException("File already exists: " + name); + } + markModificationTime(); + } + + /** + * Does a directory lookup, and returns the "inode" for the specified name. + * Returns null if the child is not found. + */ + synchronized InMemoryContentInfo getChild(String name) { + return directoryContent.get(name); + } + + /** + * Removes a previously existing child from the directory specified by this + * object. + */ + synchronized void removeChild(String name) { + if (directoryContent.remove(name) == null) { + throw new IllegalArgumentException(name + " is not a member of this directory"); + } + markModificationTime(); + } + + /** + * This function returns the content of a directory. For now, it returns a set + * to reflect the semantics of the value returned (ie. unordered, no + * duplicates). If thats too slow, it should be changed later. + */ + Set<String> getAllChildren() { + return directoryContent.keySet(); + } + + @Override + public boolean isDirectory() { + return true; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public boolean isFile() { + return false; + } + + /** + * In the InMemory hierarchy, the getSize on a directory always returns the + * number of children in the directory. + */ + @Override + public long getSize() { + return directoryContent.size(); + } + +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java new file mode 100644 index 0000000..f88285d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileInfo.java
@@ -0,0 +1,97 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.inmemoryfs; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.util.Clock; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * InMemoryFileInfo manages file contents by storing them entirely in memory. + */ +@ThreadSafe +public class InMemoryFileInfo extends FileInfo { + + /** + * Updates to the content must atomically update the lastModifiedTime. So all + * accesses to this field must be synchronized. + */ + protected byte[] content; + + protected InMemoryFileInfo(Clock clock) { + super(clock); + content = new byte[0]; // New files start out empty. + } + + @Override + public synchronized long getSize() { + return content.length; + } + + @Override + public synchronized byte[] readContent() { + return content.clone(); + } + + private synchronized void setContent(byte[] newContent) { + content = newContent; + markModificationTime(); + } + + @Override + protected synchronized OutputStream getOutputStream(boolean append) + throws IOException { + OutputStream out = new ByteArrayOutputStream() { + private boolean closed = false; + + @Override + public void write(byte[] data) throws IOException { + Preconditions.checkState(!closed); + super.write(data); + } + + @Override + public void write(int dataByte) { + Preconditions.checkState(!closed); + super.write(dataByte); + } + + @Override + public void write(byte[] data, int offset, int length) { + Preconditions.checkState(!closed); + super.write(data, offset, length); + } + + @Override + public void close() { + flush(); + closed = true; + } + + @Override + public void flush() { + setContent(toByteArray().clone()); + } + }; + + if (append) { + out.write(readContent()); + } + return out; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystem.java new file mode 100644 index 0000000..8a3b823 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystem.java
@@ -0,0 +1,920 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.inmemoryfs; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.unix.FileAccessException; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.JavaClock; +import com.google.devtools.build.lib.vfs.FileStatus; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.ScopeEscapableFileSystem; +import com.google.devtools.build.lib.vfs.Symlinks; + +import java.io.ByteArrayInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.Stack; + +import javax.annotation.Nullable; + +/** + * This class provides a complete in-memory file system. + * + * <p>Naming convention: we use "path" for all {@link Path} variables, since these + * represent *names* and we use "node" or "inode" for InMemoryContentInfo + * variables, since these correspond to inodes in the UNIX file system. + * + * <p>The code is structured to be as similar to the implementation of UNIX "namei" + * as is reasonably possibly. This provides a firm reference point for many + * concepts and makes compatibility easier to achieve. + * + * <p>As a scope-escapable file system, this class supports re-delegation of symbolic links + * that escape its root. This is done through the use of {@link OutOfScopeFileStatus} + * and {@link OutOfScopeDirectoryStatus} objects, which may be returned by + * getDirectory, pathWalk, and scopeLimitedStat. Any code that calls one of these + * methods (either directly or indirectly) is obligated to check the possibility + * that its info represents an out-of-scope path. Lack of such a check will result + * in unchecked runtime exceptions upon any request for status data (as well as + * possible logical errors). + */ +@ThreadSafe +public class InMemoryFileSystem extends ScopeEscapableFileSystem { + + private final Clock clock; + + // The root inode (a directory). + private final InMemoryDirectoryInfo rootInode; + + // Maximum number of traversals before ELOOP is thrown. + private static final int MAX_TRAVERSALS = 256; + + /** + * Creates a new InMemoryFileSystem with scope checking disabled (all paths are considered to be + * within scope) and a default clock. + */ + public InMemoryFileSystem() { + this(new JavaClock()); + } + + /** + * Creates a new InMemoryFileSystem with scope checking disabled (all + * paths are considered to be within scope). + */ + public InMemoryFileSystem(Clock clock) { + this(clock, null); + } + + /** + * Creates a new InMemoryFileSystem with scope checking bound to + * scopeRoot, i.e. any path that's not below scopeRoot is considered + * to be out of scope. + */ + protected InMemoryFileSystem(Clock clock, PathFragment scopeRoot) { + super(scopeRoot); + this.clock = clock; + this.rootInode = new InMemoryDirectoryInfo(clock); + rootInode.addChild(".", rootInode); + rootInode.addChild("..", rootInode); + } + + /** + * The errors that {@link InMemoryFileSystem} might issue for different sorts of IO failures. + */ + public enum Error { + ENOENT("No such file or directory"), + EACCES("Permission denied"), + ENOTDIR("Not a directory"), + EEXIST("File exists"), + EBUSY("Device or resource busy"), + ENOTEMPTY("Directory not empty"), + EISDIR("Is a directory"), + ELOOP("Too many levels of symbolic links"); + + private final String message; + + private Error(String message) { + this.message = message; + } + + @Override + public String toString() { + return message; + } + + /** Implemented by exceptions that contain the extra info of which Error caused them. */ + private static interface WithError { + Error getError(); + } + + /** + * The exceptions below extend their parent classes in order to additionally store the error + * that caused them. However, they must impersonate their parents to any outside callers, + * including in their toString() method, which prints the class name followed by the exception + * method. This method returns the same value as the toString() method of a {@link Throwable}'s + * parent would, so that the child class can have the same toString() value. + */ + private static String parentThrowableToString(Throwable obj) { + String s = obj.getClass().getSuperclass().getName(); + String message = obj.getLocalizedMessage(); + return (message != null) ? (s + ": " + message) : s; + } + + private static class IOExceptionWithError extends IOException implements WithError { + private final Error errorCode; + + private IOExceptionWithError(String message, Error errorCode) { + super(message); + this.errorCode = errorCode; + } + + @Override + public Error getError() { + return errorCode; + } + + @Override + public String toString() { + return parentThrowableToString(this); + } + } + + + private static class FileNotFoundExceptionWithError + extends FileNotFoundException implements WithError { + private final Error errorCode; + + private FileNotFoundExceptionWithError(String message, Error errorCode) { + super(message); + this.errorCode = errorCode; + } + + @Override + public Error getError() { + return errorCode; + } + + @Override + public String toString() { + return parentThrowableToString(this); + } + } + + + private static class FileAccessExceptionWithError + extends FileAccessException implements WithError { + private final Error errorCode; + + private FileAccessExceptionWithError(String message, Error errorCode) { + super(message); + this.errorCode = errorCode; + } + + @Override + public Error getError() { + return errorCode; + } + + @Override + public String toString() { + return parentThrowableToString(this); + } + } + + /** + * Returns a new IOException for the error. The exception message + * contains 'path', and is consistent with the messages returned by + * c.g.common.unix.FilesystemUtils. + */ + public IOException exception(Path path) throws IOException { + String m = path + " (" + message + ")"; + if (this == EACCES) { + throw new FileAccessExceptionWithError(m, this); + } else if (this == ENOENT) { + throw new FileNotFoundExceptionWithError(m, this); + } else { + throw new IOExceptionWithError(m, this); + } + } + } + + /** + * {@inheritDoc} + * + * <p>If <code>/proc/mounts</code> does not exist return {@code "inmemoryfs"}. + */ + @Override + public String getFileSystemType(Path path) { + return path.getRelative("/proc/mounts").exists() ? super.getFileSystemType(path) : "inmemoryfs"; + } + + /**************************************************************************** + * "Kernel" primitives: basic directory lookup primitives, in topological + * order. + */ + + /** + * Unlinks the entry 'child' from its existing parent directory 'dir'. Dual to + * insert. This succeeds even if 'child' names a non-empty directory; we need + * that for renameTo. 'child' must be a member of its parent directory, + * however. Fails if the directory was read-only. + */ + private void unlink(InMemoryDirectoryInfo dir, String child, Path errorPath) + throws IOException { + if (!dir.isWritable()) { throw Error.EACCES.exception(errorPath); } + dir.removeChild(child); + } + + /** + * Inserts inode 'childInode' into the existing directory 'dir' under the + * specified 'name'. Dual to unlink. Fails if the directory was read-only. + */ + private void insert(InMemoryDirectoryInfo dir, String child, + InMemoryContentInfo childInode, Path errorPath) + throws IOException { + if (!dir.isWritable()) { throw Error.EACCES.exception(errorPath); } + dir.addChild(child, childInode); + } + + /** + * Given an existing directory 'dir', looks up 'name' within it and returns + * its inode. Assumes the file exists, unless 'create', in which case it will + * try to create it. May fail with ENOTDIR, EACCES, ENOENT. Error messages + * will be reported against file 'path'. + */ + private InMemoryContentInfo directoryLookup(InMemoryContentInfo dir, + String name, + boolean create, + Path path) throws IOException { + if (!dir.isDirectory()) { throw Error.ENOTDIR.exception(path); } + InMemoryDirectoryInfo imdi = (InMemoryDirectoryInfo) dir; + if (!imdi.isExecutable()) { throw Error.EACCES.exception(path); } + InMemoryContentInfo child = imdi.getChild(name); + if (child == null) { + if (!create) { + throw Error.ENOENT.exception(path); + } else { + child = makeFileInfo(clock, path.asFragment()); + insert(imdi, name, child, path); + } + } + return child; + } + + /** + * Low-level path-to-inode lookup routine. Analogous to path_walk() in many + * UNIX kernels. Given 'path', walks the directory tree from the root, + * resolving all symbolic links, and returns the designated inode. + * + * <p>If 'create' is false, the inode must exist; otherwise, it will be created + * and added to its parent directory, which must exist. + * + * <p>Iff the given path escapes this file system's scope, the returned value + * is an {@link OutOfScopeFileStatus} instance. Any code that calls this method + * needs to check for that possibility (via {@link ScopeEscapableStatus#outOfScope}). + * + * <p>May fail with ENOTDIR, ENOENT, EACCES, ELOOP. + */ + private synchronized InMemoryContentInfo pathWalk(Path path, boolean create) + throws IOException { + // Implementation note: This is where we check for out-of-scope symlinks and + // trigger re-delegation to another file system accordingly. This code handles + // both absolute and relative symlinks. Some assumptions we make: First, only + // symlink targets as read from getNormalizedLinkContent() can escape our scope. + // This is because Path objects are all canonicalized (see {@link Path#getRelative}, + // etc.) and symlink target segments that get added to the stack are in-scope by + // definition. Second, symlink targets with relative segments must have the form + // [".."]*[standard segment]+, i.e. only the ".." non-standard segment is allowed + // and it may only appear as part of a contiguous prefix sequence. + + Stack<String> stack = new Stack<>(); + PathFragment rootPathFragment = rootPath.asFragment(); + for (Path p = path; !p.asFragment().equals(rootPathFragment); p = p.getParentDirectory()) { + stack.push(p.getBaseName()); + } + + InMemoryContentInfo inode = rootInode; + int parentDepth = -1; + int traversals = 0; + + while (!stack.isEmpty()) { + traversals++; + + String name = stack.pop(); + parentDepth += name.equals("..") ? -1 : 1; + + // ENOENT on last segment with 'create' => create a new file. + InMemoryContentInfo child = directoryLookup(inode, name, create && stack.isEmpty(), path); + if (child.isSymbolicLink()) { + PathFragment linkTarget = ((InMemoryLinkInfo) child).getNormalizedLinkContent(); + if (!inScope(parentDepth, linkTarget)) { + return outOfScopeStatus(linkTarget, parentDepth, stack); + } + if (linkTarget.isAbsolute()) { + inode = rootInode; + parentDepth = -1; + } + if (traversals > MAX_TRAVERSALS) { + throw Error.ELOOP.exception(path); + } + for (int ii = linkTarget.segmentCount() - 1; ii >= 0; --ii) { + stack.push(linkTarget.getSegment(ii)); // Note this may include ".." segments. + } + } else { + inode = child; + } + } + return inode; + } + + /** + * Helper routine for pathWalk: given a symlink target known to escape this file system's + * scope (and that has the form [".."]*[standard segment]+), the number of segments + * in the directory containing the symlink, and the remaining path segments following + * the symlink in the original input to pathWalk, returns an OutofScopeFileStatus + * initialized with an appropriate out-of-scope reformulation of pathWalk's original + * input. + */ + private OutOfScopeFileStatus outOfScopeStatus(PathFragment linkTarget, int parentDepth, + Stack<String> descendantSegments) { + + PathFragment escapingPath; + if (linkTarget.isAbsolute()) { + escapingPath = linkTarget; + } else { + // Relative out-of-scope paths must look like "../../../a/b/c". Find the target's + // parent path depth by subtracting one from parentDepth for each ".." reference. + // Then use that to retrieve a prefix of the scope root, which is the target's + // canonicalized parent path. + int leadingParentRefs = leadingParentReferences(linkTarget); + int baseDepth = parentDepth - leadingParentRefs; + Preconditions.checkState(baseDepth < scopeRoot.segmentCount()); + escapingPath = baseDepth > 0 + ? scopeRoot.subFragment(0, baseDepth) + : scopeRoot.subFragment(0, 0); + // Now add in everything that comes after the ".." sequence. + for (int i = leadingParentRefs; i < linkTarget.segmentCount(); i++) { + escapingPath = escapingPath.getRelative(linkTarget.getSegment(i)); + } + } + + // We've now converted the symlink to its target in canonicalized absolute path + // form. Since the symlink wasn't necessarily the final segment in the original + // input sent to pathWalk, now add in every segment that came after. + while (!descendantSegments.empty()) { + escapingPath = escapingPath.getRelative(descendantSegments.pop()); + } + + return new OutOfScopeFileStatus(escapingPath); + } + + /** + * Given 'path', returns the existing directory inode it designates, + * following symbolic links. + * + * <p>May fail with ENOTDIR, or any exception from pathWalk. + * + * <p>Iff the given path escapes this file system's scope, this method skips + * ENOTDIR checking and returns an OutOfScopeDirectoryStatus instance. Any + * code that calls this method needs to check for that possibility + * (via {@link ScopeEscapableStatus#outOfScope}). + */ + private InMemoryDirectoryInfo getDirectory(Path path) throws IOException { + InMemoryContentInfo dirInfo = pathWalk(path, false); + if (dirInfo.outOfScope()) { + return new OutOfScopeDirectoryStatus(dirInfo.getEscapingPath()); + } else if (!dirInfo.isDirectory()) { + throw Error.ENOTDIR.exception(path); + } else { + return (InMemoryDirectoryInfo) dirInfo; + } + } + + /** + * Helper method for stat, scopeLimitedStat: lock the internal state and return the + * path's (no symlink-followed) stat if the path's parent directory is within scope, + * else return an "out of scope" reference to the path's parent directory (which will + * presumably be re-delegated to another FS). + */ + private synchronized InMemoryContentInfo getNoFollowStatOrOutOfScopeParent(Path path) + throws IOException { + InMemoryDirectoryInfo dirInfo = getDirectory(path.getParentDirectory()); + return dirInfo.outOfScope() + ? dirInfo + : directoryLookup(dirInfo, path.getBaseName(), /*create=*/false, path); + } + + /** + * Given 'path', returns the existing inode it designates, optionally + * following symbolic links. Analogous to UNIX stat(2)/lstat(2), except that + * it returns a mutable inode we can modify directly. + */ + @Override + public FileStatus stat(Path path, boolean followSymlinks) throws IOException { + if (followSymlinks) { + InMemoryContentInfo status = scopeLimitedStat(path, true); + return status.outOfScope() + ? statWithDelegator(status.getEscapingPath(), true) + : status; + } else { + if (path.equals(rootPath)) { + return rootInode; + } else { + InMemoryContentInfo status = getNoFollowStatOrOutOfScopeParent(path); + // If out of scope, status references the path's parent directory. Else it references the + // path itself. + return status.outOfScope() + ? getDelegatedPath(status.getEscapingPath().getRelative( + path.getBaseName())).stat(Symlinks.NOFOLLOW) + : status; + } + } + } + + @Override + @Nullable + public FileStatus statIfFound(Path path, boolean followSymlinks) throws IOException { + try { + return stat(path, followSymlinks); + } catch (IOException e) { + if (e instanceof Error.WithError) { + Error errorCode = ((Error.WithError) e).getError(); + if (errorCode == Error.ENOENT || errorCode == Error.ENOTDIR) { + return null; + } + } + throw e; + } + } + + /** + * Version of stat that returns an inode if the input path stays entirely within + * this file system's scope, otherwise an {@link OutOfScopeFileStatus}. + * + * <p>Any code that calls this method needs to check for either possibility via + * {@link ScopeEscapableStatus#outOfScope}. + */ + protected InMemoryContentInfo scopeLimitedStat(Path path, boolean followSymlinks) + throws IOException { + if (followSymlinks) { + return pathWalk(path, false); + } else { + if (path.equals(rootPath)) { + return rootInode; + } else { + InMemoryContentInfo status = getNoFollowStatOrOutOfScopeParent(path); + // If out of scope, status references the path's parent directory. Else it references the + // path itself. + return status.outOfScope() + ? new OutOfScopeFileStatus(status.getEscapingPath().getRelative(path.getBaseName())) + : status; + } + } + } + + /**************************************************************************** + * FileSystem methods + */ + + /** + * This is a helper routing for {@link #resolveSymbolicLinks(Path)}, i.e. + * the "user-mode" routing for canonicalising paths. It is analogous to the + * code in glibc's realpath(3). + * + * <p>Just like realpath, resolveSymbolicLinks requires a quadratic number of + * directory lookups: n path segments are statted, and each stat requires a + * linear amount of work in the "kernel" routine. + */ + @Override + protected PathFragment resolveOneLink(Path path) throws IOException { + // Beware, this seemingly simple code belies the complex specification of + // FileSystem.resolveOneLink(). + InMemoryContentInfo status = scopeLimitedStat(path, false); + if (status.outOfScope()) { + return resolveOneLinkWithDelegator(status.getEscapingPath()); + } else { + return status.isSymbolicLink() + ? ((InMemoryLinkInfo) status).getLinkContent() + : null; + } + } + + @Override + protected boolean isDirectory(Path path, boolean followSymlinks) { + try { + return stat(path, followSymlinks).isDirectory(); + } catch (IOException e) { + return false; + } + } + + @Override + protected boolean isFile(Path path, boolean followSymlinks) { + try { + return stat(path, followSymlinks).isFile(); + } catch (IOException e) { + return false; + } + } + + @Override + protected boolean isSymbolicLink(Path path) { + try { + return stat(path, false).isSymbolicLink(); + } catch (IOException e) { + return false; + } + } + + @Override + protected boolean exists(Path path, boolean followSymlinks) { + try { + stat(path, followSymlinks); + return true; + } catch (IOException e) { + return false; + } + } + + /** + * Like {@link #exists}, but checks for existence within this filesystem's scope. + */ + protected boolean scopeLimitedExists(Path path, boolean followSymlinks) { + try { + // Path#asFragment() always returns an absolute path, so inScope() is called with + // parentDepth = 0. + return inScope(0, path.asFragment()) && !scopeLimitedStat(path, followSymlinks).outOfScope(); + } catch (IOException e) { + return false; + } + } + + @Override + protected boolean isReadable(Path path) throws IOException { + InMemoryContentInfo status = scopeLimitedStat(path, true); + return status.outOfScope() + ? getDelegatedPath(status.getEscapingPath()).isReadable() + : status.isReadable(); + } + + @Override + protected void setReadable(Path path, boolean readable) throws IOException { + InMemoryContentInfo status; + synchronized (this) { + status = scopeLimitedStat(path, true); + if (!status.outOfScope()) { + status.setReadable(readable); + return; + } + } + // If we get here, we're out of scope. + getDelegatedPath(status.getEscapingPath()).setReadable(readable); + } + + @Override + protected boolean isWritable(Path path) throws IOException { + InMemoryContentInfo status = scopeLimitedStat(path, true); + return status.outOfScope() + ? getDelegatedPath(status.getEscapingPath()).isWritable() + : status.isWritable(); + } + + @Override + protected void setWritable(Path path, boolean writable) throws IOException { + InMemoryContentInfo status; + synchronized (this) { + status = scopeLimitedStat(path, true); + if (!status.outOfScope()) { + status.setWritable(writable); + return; + } + } + // If we get here, we're out of scope. + getDelegatedPath(status.getEscapingPath()).setWritable(writable); + } + + @Override + protected boolean isExecutable(Path path) throws IOException { + InMemoryContentInfo status = scopeLimitedStat(path, true); + return status.outOfScope() + ? getDelegatedPath(status.getEscapingPath()).isExecutable() + : status.isExecutable(); + } + + @Override + protected void setExecutable(Path path, boolean executable) + throws IOException { + InMemoryContentInfo status; + synchronized (this) { + status = scopeLimitedStat(path, true); + if (!status.outOfScope()) { + status.setExecutable(executable); + return; + } + } + // If we get here, we're out of scope. + getDelegatedPath(status.getEscapingPath()).setExecutable(executable); + } + + @Override + public boolean supportsModifications() { + return true; + } + + @Override + public boolean supportsSymbolicLinks() { + return true; + } + + /** + * Constructs a new inode. Provided so that subclasses of InMemoryFileSystem + * can inject subclasses of FileInfo properly. + */ + protected FileInfo makeFileInfo(Clock clock, PathFragment frag) { + return new InMemoryFileInfo(clock); + } + + /** + * Returns a new path constructed by appending the child's base name to the + * escaped parent path. For example, assume our file system root is /foo + * and /foo/link1 -> /bar. This method can be used on child = /foo/link1/link2/name + * and parent = /bar/link2 to return /bar/link2/name, which is a semi-resolved + * path bound to a different file system. + */ + private Path getDelegatedPath(PathFragment escapedParent, Path child) { + return getDelegatedPath(escapedParent.getRelative(child.getBaseName())); + } + + @Override + protected boolean createDirectory(Path path) throws IOException { + if (path.equals(rootPath)) { throw Error.EACCES.exception(path); } + + InMemoryDirectoryInfo parent; + synchronized (this) { + parent = getDirectory(path.getParentDirectory()); + if (!parent.outOfScope()) { + InMemoryContentInfo child = parent.getChild(path.getBaseName()); + if (child != null) { // already exists + if (child.isDirectory()) { + return false; + } else { + throw Error.EEXIST.exception(path); + } + } + + InMemoryDirectoryInfo newDir = new InMemoryDirectoryInfo(clock); + newDir.addChild(".", newDir); + newDir.addChild("..", parent); + insert(parent, path.getBaseName(), newDir, path); + + return true; + } + } + + // If we get here, we're out of scope. + return getDelegatedPath(parent.getEscapingPath(), path).createDirectory(); + } + + @Override + protected void createSymbolicLink(Path path, PathFragment targetFragment) + throws IOException { + if (path.equals(rootPath)) { throw Error.EACCES.exception(path); } + + InMemoryDirectoryInfo parent; + synchronized (this) { + parent = getDirectory(path.getParentDirectory()); + if (!parent.outOfScope()) { + if (parent.getChild(path.getBaseName()) != null) { throw Error.EEXIST.exception(path); } + insert(parent, path.getBaseName(), new InMemoryLinkInfo(clock, targetFragment), path); + return; + } + } + + // If we get here, we're out of scope. + getDelegatedPath(parent.getEscapingPath(), path).createSymbolicLink(targetFragment); + } + + @Override + protected PathFragment readSymbolicLink(Path path) throws IOException { + InMemoryContentInfo status = scopeLimitedStat(path, false); + if (status.outOfScope()) { + return getDelegatedPath(status.getEscapingPath()).readSymbolicLink(); + } else if (status.isSymbolicLink()) { + Preconditions.checkState(status instanceof InMemoryLinkInfo); + return ((InMemoryLinkInfo) status).getLinkContent(); + } else { + throw new NotASymlinkException(path); + } + } + + @Override + protected long getFileSize(Path path, boolean followSymlinks) + throws IOException { + return stat(path, followSymlinks).getSize(); + } + + @Override + protected Collection<Path> getDirectoryEntries(Path path) throws IOException { + InMemoryDirectoryInfo dirInfo; + synchronized (this) { + dirInfo = getDirectory(path); + if (!dirInfo.outOfScope()) { + FileStatus status = stat(path, false); + Preconditions.checkState(status instanceof InMemoryContentInfo); + if (!((InMemoryContentInfo) status).isReadable()) { + throw new IOException("Directory is not readable"); + } + + Set<String> allChildren = dirInfo.getAllChildren(); + List<Path> result = new ArrayList<>(allChildren.size()); + for (String child : allChildren) { + if (!(child.equals(".") || child.equals(".."))) { + result.add(path.getChild(child)); + } + } + return result; + } + } + + // If we get here, we're out of scope. + return getDelegatedPath(dirInfo.getEscapingPath()).getDirectoryEntries(); + } + + @Override + protected boolean delete(Path path) throws IOException { + if (path.equals(rootPath)) { throw Error.EBUSY.exception(path); } + if (!exists(path, false)) { return false; } + + InMemoryDirectoryInfo parent; + synchronized (this) { + parent = getDirectory(path.getParentDirectory()); + if (!parent.outOfScope()) { + InMemoryContentInfo child = parent.getChild(path.getBaseName()); + if (child.isDirectory() && child.getSize() > 2) { throw Error.ENOTEMPTY.exception(path); } + unlink(parent, path.getBaseName(), path); + return true; + } + } + + // If we get here, we're out of scope. + return getDelegatedPath(parent.getEscapingPath(), path).delete(); + } + + @Override + protected long getLastModifiedTime(Path path, boolean followSymlinks) + throws IOException { + return stat(path, followSymlinks).getLastModifiedTime(); + } + + @Override + protected void setLastModifiedTime(Path path, long newTime) throws IOException { + InMemoryContentInfo status; + synchronized (this) { + status = scopeLimitedStat(path, true); + if (!status.outOfScope()) { + status.setLastModifiedTime(newTime == -1L + ? clock.currentTimeMillis() + : newTime); + return; + } + } + + // If we get here, we're out of scope. + getDelegatedPath(status.getEscapingPath()).setLastModifiedTime(newTime); + } + + @Override + protected InputStream getInputStream(Path path) throws IOException { + InMemoryContentInfo status; + synchronized (this) { + status = scopeLimitedStat(path, true); + if (!status.outOfScope()) { + if (status.isDirectory()) { throw Error.EISDIR.exception(path); } + if (!path.isReadable()) { throw Error.EACCES.exception(path); } + Preconditions.checkState(status instanceof FileInfo); + return new ByteArrayInputStream(((FileInfo) status).readContent()); + } + } + + // If we get here, we're out of scope. + return getDelegatedPath(status.getEscapingPath()).getInputStream(); + } + + /** + * Creates a new file at the given path and returns its inode. If the path + * escapes this file system's scope, trivially returns an "out of scope" status. + * Calling code should check for both possibilities via + * {@link ScopeEscapableStatus#outOfScope}. + */ + protected InMemoryContentInfo getOrCreateWritableInode(Path path) + throws IOException { + // open(WR_ONLY) of a dangling link writes through the link. That means + // that the usual path lookup operations have to behave differently when + // resolving a path with the intent to create it: instead of failing with + // ENOENT they have to return an open file. This is exactly how UNIX + // kernels do it, which is what we're trying to emulate. + InMemoryContentInfo child = pathWalk(path, /*create=*/true); + Preconditions.checkNotNull(child); + if (child.outOfScope()) { + return child; + } else if (child.isDirectory()) { + throw Error.EISDIR.exception(path); + } else { // existing or newly-created file + if (!child.isWritable()) { throw Error.EACCES.exception(path); } + return child; + } + } + + @Override + protected OutputStream getOutputStream(Path path, boolean append) + throws IOException { + InMemoryContentInfo status; + synchronized (this) { + status = getOrCreateWritableInode(path); + if (!status.outOfScope()) { + return ((FileInfo) getOrCreateWritableInode(path)).getOutputStream(append); + } + } + // If we get here, we're out of scope. + return getDelegatedPath(status.getEscapingPath()).getOutputStream(append); + } + + @Override + protected void renameTo(Path sourcePath, Path targetPath) + throws IOException { + if (sourcePath.equals(rootPath)) { throw Error.EACCES.exception(sourcePath); } + if (targetPath.equals(rootPath)) { throw Error.EACCES.exception(targetPath); } + + InMemoryDirectoryInfo sourceParent; + InMemoryDirectoryInfo targetParent; + + synchronized (this) { + sourceParent = getDirectory(sourcePath.getParentDirectory()); + targetParent = getDirectory(targetPath.getParentDirectory()); + + // Handle the rename if both paths are within our scope. + if (!sourceParent.outOfScope() && !targetParent.outOfScope()) { + InMemoryContentInfo sourceInode = sourceParent.getChild(sourcePath.getBaseName()); + if (sourceInode == null) { throw Error.ENOENT.exception(sourcePath); } + InMemoryContentInfo targetInode = targetParent.getChild(targetPath.getBaseName()); + + unlink(sourceParent, sourcePath.getBaseName(), sourcePath); + try { + // TODO(bazel-team): (2009) test with symbolic links. + + // Precondition checks: + if (targetInode != null) { // already exists + if (targetInode.isDirectory()) { + if (!sourceInode.isDirectory()) { + throw new IOException(sourcePath + " -> " + targetPath + " (" + Error.EISDIR + ")"); + } + if (targetInode.getSize() > 2) { + throw Error.ENOTEMPTY.exception(targetPath); + } + } else if (sourceInode.isDirectory()) { + throw new IOException(sourcePath + " -> " + targetPath + " (" + Error.ENOTDIR + ")"); + } + unlink(targetParent, targetPath.getBaseName(), targetPath); + } + sourceInode.movedTo(targetPath); + insert(targetParent, targetPath.getBaseName(), sourceInode, targetPath); + return; + + } catch (IOException e) { + sourceInode.movedTo(sourcePath); + insert(sourceParent, sourcePath.getBaseName(), sourceInode, sourcePath); // restore source + throw e; + } + } + } + + // If we get here, either one or both paths is out of scope. + if (sourceParent.outOfScope() && targetParent.outOfScope()) { + Path delegatedSource = getDelegatedPath(sourceParent.getEscapingPath(), sourcePath); + Path delegatedTarget = getDelegatedPath(targetParent.getEscapingPath(), targetPath); + delegatedSource.renameTo(delegatedTarget); + } else { + // We don't support cross-file system renaming. + throw Error.EACCES.exception(targetPath); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryLinkInfo.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryLinkInfo.java new file mode 100644 index 0000000..f8837ee --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryLinkInfo.java
@@ -0,0 +1,76 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.inmemoryfs; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * This interface represents a symbolic link to an absolute or relative path, + * stored in an InMemoryFileSystem. + */ +@ThreadSafe @Immutable +class InMemoryLinkInfo extends InMemoryContentInfo { + + private final PathFragment linkContent; + private final PathFragment normalizedLinkContent; + + InMemoryLinkInfo(Clock clock, PathFragment linkContent) { + super(clock); + this.linkContent = linkContent; + this.normalizedLinkContent = linkContent.normalize(); + } + + @Override + public boolean isDirectory() { + return false; + } + + @Override + public boolean isSymbolicLink() { + return true; + } + + @Override + public boolean isFile() { + return false; + } + + @Override + public long getSize() { + return linkContent.toString().length(); + } + + /** + * Returns the content of the symbolic link. + */ + PathFragment getLinkContent() { + return linkContent; + } + + /** + * Returns the content of the symbolic link, with ".." and "." removed + * (except for the possibility of necessary ".." segments at the beginning). + */ + PathFragment getNormalizedLinkContent() { + return normalizedLinkContent; + } + + @Override + public String toString() { + return super.toString() + " -> " + linkContent; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/OutOfScopeDirectoryStatus.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/OutOfScopeDirectoryStatus.java new file mode 100644 index 0000000..b757acd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/OutOfScopeDirectoryStatus.java
@@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.inmemoryfs; + +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Set; + +/** + * A directory status that signifies a path has left this file system's + * scope. All methods beside {@link #outOfScope} and {@link #getEscapingPath} + * are disabled. + */ +final class OutOfScopeDirectoryStatus extends InMemoryDirectoryInfo { + /** + * Contains the requested path resolved up to the point where it + * first escapes the scope. See + * {@link ScopeEscapableStatus#getEscapingPath} for an example. + */ + private final PathFragment escapingPath; + + public OutOfScopeDirectoryStatus(PathFragment escapingPath) { + super(null, false); + this.escapingPath = escapingPath; + } + + @Override + public boolean outOfScope() { + return true; + } + + @Override + public PathFragment getEscapingPath() { + return escapingPath; + } + + private static UnsupportedOperationException failure() { + return new UnsupportedOperationException(); + } + + @Override public boolean isDirectory() { throw failure(); } + @Override public boolean isSymbolicLink() { throw failure(); } + @Override public boolean isFile() { throw failure(); } + @Override public long getSize() { throw failure(); } + @Override protected void markModificationTime() { throw failure(); } + @Override public synchronized long getLastModifiedTime() { throw failure(); } + @Override void setLastModifiedTime(long newTime) { throw failure(); } + @Override public synchronized long getLastChangeTime() { throw failure(); } + @Override boolean isReadable() { throw failure(); } + @Override void setReadable(boolean readable) { throw failure(); } + @Override void setWritable(boolean writable) { throw failure(); } + @Override void setExecutable(boolean executable) { throw failure(); } + @Override boolean isWritable() { throw failure(); } + @Override boolean isExecutable() { throw failure(); } + @Override void addChild(String name, InMemoryContentInfo inode) { throw failure(); } + @Override InMemoryContentInfo getChild(String name) { throw failure(); } + @Override void removeChild(String name) { throw failure(); } + @Override Set<String> getAllChildren() { throw failure(); } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/OutOfScopeFileStatus.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/OutOfScopeFileStatus.java new file mode 100644 index 0000000..177ac11 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/OutOfScopeFileStatus.java
@@ -0,0 +1,65 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.inmemoryfs; + +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * A file status that signifies a path has left this file system's + * scope. All methods beside {@link #outOfScope} and {@link #getEscapingPath} + * are disabled. + */ +final class OutOfScopeFileStatus extends InMemoryContentInfo { + + /** + * Contains the requested path resolved up to the point where it + * first escapes the scope. See + * {@link ScopeEscapableStatus#getEscapingPath} for an example. + */ + private final PathFragment escapingPath; + + public OutOfScopeFileStatus(PathFragment escapingPath) { + super(null, false); + this.escapingPath = escapingPath; + } + + @Override + public boolean outOfScope() { + return true; + } + + @Override + public PathFragment getEscapingPath() { + return escapingPath; + } + + private static UnsupportedOperationException failure() { + return new UnsupportedOperationException(); + } + + @Override public boolean isDirectory() { throw failure(); } + @Override public boolean isSymbolicLink() { throw failure(); } + @Override public boolean isFile() { throw failure(); } + @Override public long getSize() { throw failure(); } + @Override protected void markModificationTime() { throw failure(); } + @Override public synchronized long getLastModifiedTime() { throw failure(); } + @Override void setLastModifiedTime(long newTime) { throw failure(); } + @Override public synchronized long getLastChangeTime() { throw failure(); } + @Override boolean isReadable() { throw failure(); } + @Override void setReadable(boolean readable) { throw failure(); } + @Override void setWritable(boolean writable) { throw failure(); } + @Override boolean isWritable() { throw failure(); } + @Override void setExecutable(boolean executable) { throw failure(); } + @Override boolean isExecutable() { throw failure(); } +}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/ScopeEscapableStatus.java b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/ScopeEscapableStatus.java new file mode 100644 index 0000000..4afec78 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs/ScopeEscapableStatus.java
@@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.inmemoryfs; + +import com.google.devtools.build.lib.vfs.FileStatus; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.ScopeEscapableFileSystem; + +/** + * Interface definition for a file status that may signify that the + * referenced path falls outside the scope of the file system (see + * {@link ScopeEscapableFileSystem}) and can provide the "escaped" + * version of that path suitable for re-delegation to another file + * system. + */ +interface ScopeEscapableStatus extends FileStatus { + + /** + * Returns true if this status corresponds to a path that leaves + * the file system's scope, false otherwise. + */ + boolean outOfScope(); + + /** + * If this status represents a path that leaves the file system's scope, + * returns the requested path resolved up to the point where it first + * escapes the file system. For example: if the file system is mapped to + * /foo, the requested path is /foo/link1/link2/link3, and link1 -> /bar, + * this returns /bar/link2/link3. + * + * <p>If this status doesn't represent a scope-escaping path, returns + * null. + */ + PathFragment getEscapingPath(); +}
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/IndexPageHandler.java b/src/main/java/com/google/devtools/build/lib/webstatusserver/IndexPageHandler.java new file mode 100644 index 0000000..c9eb3ed --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/IndexPageHandler.java
@@ -0,0 +1,82 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.webstatusserver; + +import com.google.common.collect.ImmutableList; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + + +/** + * Handlers for displaying the index page of server. + * + */ +public class IndexPageHandler { + private List<TestStatusHandler> testHandlers = new ArrayList<>(); + private IndexPageJsonData dataHandler; + private StaticResourceHandler frontendHandler; + + public IndexPageHandler(HttpServer server, List<TestStatusHandler> testHandlers) { + this.testHandlers = testHandlers; + this.dataHandler = new IndexPageJsonData(this); + this.frontendHandler = + StaticResourceHandler.createFromRelativePath("static/index.html", "text/html"); + server.createContext("/", frontendHandler); + server.createContext("/tests/list", dataHandler); + } + + /** + * Puts data from the build log into json suitable for frontend. + * + */ + private class IndexPageJsonData implements HttpHandler { + private IndexPageHandler pageHandler; + private Gson gson = new Gson(); + public IndexPageJsonData(IndexPageHandler indexPageHandler) { + this.pageHandler = indexPageHandler; + } + + @Override + public void handle(HttpExchange exchange) throws IOException { + exchange.getResponseHeaders().put("Content-Type", ImmutableList.of("application/json")); + JsonArray response = new JsonArray(); + for (TestStatusHandler handler : this.pageHandler.testHandlers) { + WebStatusBuildLog buildLog = handler.getBuildLog(); + JsonObject test = new JsonObject(); + test.add("targets", gson.toJsonTree(buildLog.getTargetList())); + test.addProperty("startTime", buildLog.getStartTime()); + test.addProperty("finished", buildLog.finished()); + test.addProperty("uuid", buildLog.getCommandId().toString()); + response.add(test); + } + String serializedResponse = response.toString(); + exchange.sendResponseHeaders(200, serializedResponse.length()); + OutputStream os = exchange.getResponseBody(); + os.write(serializedResponse.getBytes()); + os.close(); + } + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/StaticResourceHandler.java b/src/main/java/com/google/devtools/build/lib/webstatusserver/StaticResourceHandler.java new file mode 100644 index 0000000..cd9eb5f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/StaticResourceHandler.java
@@ -0,0 +1,80 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.webstatusserver; + +import com.google.common.collect.ImmutableList; +import com.google.common.io.CharStreams; +import com.google.devtools.build.lib.util.ResourceFileLoader; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.URL; +import java.util.List; + +/** + * Handler for static resources (JS, html, css...) + */ +public class StaticResourceHandler implements HttpHandler { + private String response; + private List<String> contentType; + private int httpCode; + + public static StaticResourceHandler createFromAbsolutePath(String path, String contentType) { + return new StaticResourceHandler(path, contentType, true); + } + + public static StaticResourceHandler createFromRelativePath(String path, String contentType) { + return new StaticResourceHandler(path, contentType, false); + } + + private StaticResourceHandler(String path, String contentType, boolean absolutePath) { + try { + if (absolutePath) { + InputStream resourceStream = loadFromAbsolutePath(WebStatusServerModule.class, path); + response = CharStreams.toString(new InputStreamReader(resourceStream)); + + } else { + response = ResourceFileLoader.loadResource(WebStatusServerModule.class, path); + } + httpCode = 200; + } catch (IOException e) { + throw new IllegalArgumentException("resource " + path + " not found"); + } + this.contentType = ImmutableList.of(contentType); + } + + @Override + public void handle(HttpExchange exchange) throws IOException { + exchange.getResponseHeaders().put("Content-Type", contentType); + exchange.sendResponseHeaders(httpCode, response.length()); + OutputStream os = exchange.getResponseBody(); + os.write(response.getBytes()); + os.close(); + } + + public static InputStream loadFromAbsolutePath(Class<?> loadingClass, String path) + throws IOException { + URL resourceUrl = loadingClass.getClassLoader().getResource(path); + if (resourceUrl == null) { + throw new IllegalArgumentException("resource " + path + " not found"); + } + return resourceUrl.openStream(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/TestStatusHandler.java b/src/main/java/com/google/devtools/build/lib/webstatusserver/TestStatusHandler.java new file mode 100644 index 0000000..41cb06d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/TestStatusHandler.java
@@ -0,0 +1,148 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.webstatusserver; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.reflect.TypeToken; + +import com.sun.net.httpserver.HttpContext; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; + +import java.io.IOException; +import java.io.OutputStream; +import java.lang.reflect.Type; +import java.util.Map; +import java.util.Map.Entry; + +/** + * Collection of handlers for displaying the test data. + */ +class TestStatusHandler { + private StaticResourceHandler frontendHandler; + private WebStatusBuildLog buildLog; + private HttpHandler detailsHandler; + private HttpServer server; + private ImmutableList<HttpContext> contexts; + private CommandJsonData commandHandler; + private Gson gson = new Gson(); + + public TestStatusHandler(HttpServer server, WebStatusBuildLog buildLog) { + Builder<HttpContext> builder = ImmutableList.builder(); + this.buildLog = buildLog; + this.server = server; + detailsHandler = new TestStatusResultJsonData(this); + commandHandler = new CommandJsonData(this); + frontendHandler = StaticResourceHandler.createFromRelativePath("static/test.html", "text/html"); + builder.add( + server.createContext("/tests/" + buildLog.getCommandId() + "/details", detailsHandler)); + builder.add( + server.createContext("/tests/" + buildLog.getCommandId() + "/info", commandHandler)); + builder.add(server.createContext("/tests/" + buildLog.getCommandId(), frontendHandler)); + contexts = builder.build(); + } + + public WebStatusBuildLog getBuildLog() { + return buildLog; + } + + + /** + * Serves JSON objects containing command info, which will be rendered by frontend. + */ + private class CommandJsonData implements HttpHandler { + private TestStatusHandler testStatusHandler; + + public CommandJsonData(TestStatusHandler testStatusHandler) { + this.testStatusHandler = testStatusHandler; + } + + @Override + public void handle(HttpExchange exchange) throws IOException { + exchange.getResponseHeaders().put("Content-Type", ImmutableList.of("application/json")); + Type commandInfoType = new TypeToken<Map<String, JsonElement>>() {}.getType(); + JsonObject response = gson.toJsonTree(testStatusHandler.buildLog.getCommandInfo(), + commandInfoType).getAsJsonObject(); + response.addProperty("startTime", testStatusHandler.buildLog.getStartTime()); + response.addProperty("finished", testStatusHandler.buildLog.finished()); + + String serializedResponse = response.toString(); + exchange.sendResponseHeaders(200, serializedResponse.length()); + OutputStream os = exchange.getResponseBody(); + os.write(serializedResponse.getBytes()); + os.close(); + } + } + + /** + * Serves JSON objects containing test cases, which will be rendered by frontend. + */ + private class TestStatusResultJsonData implements HttpHandler { + private TestStatusHandler testStatusHandler; + + public TestStatusResultJsonData(TestStatusHandler testStatusHandler) { + this.testStatusHandler = testStatusHandler; + } + + @Override + public void handle(HttpExchange exchange) throws IOException { + Map<String, JsonObject> testInfo = testStatusHandler.buildLog.getTestCases(); + exchange.getResponseHeaders().put("Content-Type", ImmutableList.of("application/json")); + JsonObject response = new JsonObject(); + for (Entry<String, JsonObject> testCase : testInfo.entrySet()) { + response.add(testCase.getKey(), testCase.getValue()); + } + + String serializedResponse = response.toString(); + exchange.sendResponseHeaders(200, serializedResponse.length()); + OutputStream os = exchange.getResponseBody(); + os.write(serializedResponse.getBytes()); + os.close(); + } + } + + /** + * Adds another URI for existing test data. If specified URI is already used by some other + * handler, the previous handler will be removed. + */ + public void overrideURI(String uri) { + String detailsPath = uri + "/details"; + String commandPath = uri + "/info"; + try { + this.server.removeContext(detailsPath); + this.server.removeContext(commandPath); + } catch (IllegalArgumentException e) { + // There was nothing to remove, so proceed with creation (unfortunately the server api doesn't + // have "hasContext" method) + } + this.server.createContext(detailsPath, this.detailsHandler); + this.server.createContext(commandPath, this.commandHandler); + } + + /** + * Deregisters all the handlers associated with the test. + */ + public void deregister() { + for (HttpContext c : this.contexts) { + this.server.removeContext(c); + } + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusBuildLog.java b/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusBuildLog.java new file mode 100644 index 0000000..86eed88 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusBuildLog.java
@@ -0,0 +1,200 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.webstatusserver; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; +import com.google.devtools.build.lib.view.test.TestStatus.TestCase; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.logging.Logger; + +/** + * Stores information about one build command. The data is stored in JSON so that it can be + * can be easily fed to frontend. + * + * <p> The information is grouped into following structures: + * <ul> + * <li> {@link #commandInfo} contain information about the build known when it starts but before + * anything is actually compiled/run + * <li> {@link #testCases} contain detailed information about each test case ran, for now they're + * + * </ul> + */ +public class WebStatusBuildLog { + private Gson gson = new Gson(); + private boolean complete = false; + private static final Logger LOG = + Logger.getLogger(WebStatusEventCollector.class.getCanonicalName()); + private Map<String, JsonElement> commandInfo = new HashMap<String, JsonElement>(); + private Map<String, JsonObject> testCases = new HashMap<String, JsonObject>(); + private long startTime; + private ImmutableList<String> targetList; + private UUID commandId; + + public WebStatusBuildLog(UUID commandId) { + this.commandId = commandId; + } + + public WebStatusBuildLog addInfo(String key, Object value) { + commandInfo.put(key, gson.toJsonTree(value)); + return this; + } + + public void addStartTime(long startTime) { + this.startTime = startTime; + } + + public void addTargetList(List<String> targets) { + this.targetList = ImmutableList.copyOf(targets); + } + + public void finish() { + commandInfo = ImmutableMap.copyOf(commandInfo); + complete = true; + } + + public Map<String, JsonElement> getCommandInfo() { + return commandInfo; + } + + public ImmutableMap<String, JsonObject> getTestCases() { + // TODO(bazel-team): not really immutable, since one can do addProperty on + // values (unfortunately gson doesn't support immutable JsonObjects) + return ImmutableMap.copyOf(testCases); + } + + public boolean finished() { + return complete; + } + + public List<String> getTargetList() { + return targetList; + } + + public long getStartTime() { + return startTime; + } + + public void addTestTarget(Label label) { + String targetName = label.toShorthandString(); + if (!testCases.containsKey(targetName)) { + JsonObject summary = createTestCaseEmptyJsonNode(targetName); + summary.addProperty("finished", false); + summary.addProperty("status", "started"); + testCases.put(targetName, summary); + } else { + // TODO(bazel-team): figure out if there are any situations it can happen + } + } + + public void addTestSummary(Label label, BlazeTestStatus status, List<Long> testTimes, + boolean isCached) { + JsonObject testCase = testCases.get(label.toShorthandString()); + testCase.addProperty("status", status.toString()); + testCase.add("times", gson.toJsonTree(testTimes)); + testCase.addProperty("cached", isCached); + testCase.addProperty("finished", true); + } + + public void addTargetBuilt(Label label, boolean success) { + if (testCases.containsKey(label.toShorthandString())) { + if (success) { + testCases.get(label.toShorthandString()).addProperty("status", "built"); + } else { + testCases.get(label.toShorthandString()).addProperty("status", "build failure"); + } + } else { + LOG.info("Unhandled target: " + label); + } + } + + @VisibleForTesting + static JsonObject createTestCaseEmptyJsonNode(String fullName) { + JsonObject currentNode = new JsonObject(); + currentNode.addProperty("fullName", fullName); + currentNode.addProperty("name", ""); + currentNode.addProperty("className", ""); + currentNode.add("results", new JsonObject()); + currentNode.add("times", new JsonObject()); + currentNode.add("children", new JsonObject()); + currentNode.add("failures", new JsonObject()); + currentNode.add("errors", new JsonObject()); + return currentNode; + } + + private static JsonObject createTestCaseEmptyJsonNode(String fullName, TestCase testCase) { + JsonObject currentNode = createTestCaseEmptyJsonNode(fullName); + currentNode.addProperty("name", testCase.getName()); + currentNode.addProperty("className", testCase.getClassName()); + return currentNode; + } + + private JsonObject mergeTestCases(JsonObject currentNode, String fullName, TestCase testCase, + int shardNumber) { + if (currentNode == null) { + currentNode = createTestCaseEmptyJsonNode(fullName, testCase); + } + + if (testCase.getRun()) { + JsonObject results = (JsonObject) currentNode.get("results"); + JsonObject times = (JsonObject) currentNode.get("times"); + + if (testCase.hasResult()) { + results.addProperty(Integer.toString(shardNumber), testCase.getResult()); + } + + if (testCase.hasStatus()) { + results.addProperty(Integer.toString(shardNumber), testCase.getStatus().toString()); + } + + if (testCase.hasRunDurationMillis()) { + times.addProperty(Integer.toString(shardNumber), testCase.getRunDurationMillis()); + } + } + JsonObject children = (JsonObject) currentNode.get("children"); + + for (TestCase child : testCase.getChildList()) { + String fullChildName = child.getClassName() + "." + child.getName(); + JsonObject childNode = mergeTestCases((JsonObject) children.get(fullChildName), fullChildName, + child, shardNumber); + if (!children.has(fullChildName)) { + children.add(fullChildName, childNode); + } + } + return currentNode; + } + + public void addTestResult(Label label, TestCase testCase, int shardNumber) { + String testResultFullName = label.toShorthandString(); + if (!testCases.containsKey(testResultFullName)) { + testCases.put(testResultFullName, createTestCaseEmptyJsonNode(testResultFullName, testCase)); + } + mergeTestCases(testCases.get(testResultFullName), testResultFullName, testCase, shardNumber); + } + + public UUID getCommandId() { + return commandId; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusEventCollector.java b/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusEventCollector.java new file mode 100644 index 0000000..40b0908 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusEventCollector.java
@@ -0,0 +1,135 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.webstatusserver; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.TargetCompleteEvent; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent; +import com.google.devtools.build.lib.buildtool.buildevent.TestFilteringCompleteEvent; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.rules.test.TestResult; +import com.google.devtools.build.lib.runtime.CommandCompleteEvent; +import com.google.devtools.build.lib.runtime.CommandStartEvent; +import com.google.devtools.build.lib.runtime.TestSummary; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.logging.Logger; + +/** + * This class monitors the build progress, collects events and preprocesses them for use by + * frontend. + * + */ +public class WebStatusEventCollector { + private static final Logger LOG = + Logger.getLogger(WebStatusEventCollector.class.getCanonicalName()); + private final EventBus eventBus; + private final Reporter reporter; + private final int port; + private WebStatusBuildLog currentBuild; + private WebStatusServerModule serverModule; + + public WebStatusEventCollector(EventBus eventBus, Reporter reporter, + WebStatusServerModule webStatusServerModule) { + this.eventBus = eventBus; + this.eventBus.register(this); + this.reporter = reporter; + this.port = webStatusServerModule.getPort(); + this.serverModule = webStatusServerModule; + LOG.info("Created new status collector"); + } + + @Subscribe + public void buildStarted(BuildStartingEvent startingEvent) { + BuildRequest request = startingEvent.getRequest(); + BlazeVersionInfo versionInfo = BlazeVersionInfo.instance(); + currentBuild.addStartTime(request.getStartTime()); + currentBuild.addTargetList(request.getTargets()); + currentBuild + .addInfo("version", versionInfo) + .addInfo("commandName", request.getCommandName()) + .addInfo("outputFs", startingEvent.getOutputFileSystem()) + .addInfo("symlinkPrefix", request.getSymlinkPrefix()) + .addInfo("optionsDescription", request.getOptionsDescription()) + .addInfo("targets", request.getTargets()) + .addInfo("viewOptions", request.getViewOptions()); + } + + @Subscribe + @SuppressWarnings("unused") + public void commandComplete(CommandCompleteEvent completeEvent) { + currentBuild.addInfo("endTime", completeEvent.getEventTimeInEpochTime()); + currentBuild.finish(); + } + + @Subscribe + @SuppressWarnings("unused") + public void commandStarted(CommandStartEvent event) { + this.currentBuild = new WebStatusBuildLog(event.getCommandId()); + this.serverModule.commandStarted(); + String webStatusServerUrl = "http://localhost:" + port; + this.reporter.handle(Event.info("Status page: " + webStatusServerUrl + "/tests/" + + this.currentBuild.getCommandId() + " (alternative link: " + webStatusServerUrl + + WebStatusServerModule.LAST_TEST_URI + " )")); + } + + @Subscribe + public void doneTestFiltering(TestFilteringCompleteEvent event) { + if (event.getTestTargets() != null) { + Builder<Label> builder = ImmutableList.builder(); + for (ConfiguredTarget target : event.getTestTargets()) { + builder.add(target.getLabel()); + } + doneTestFiltering(builder.build()); + } + } + + @VisibleForTesting + public void doneTestFiltering(Iterable<Label> testLabels) { + for (Label label : testLabels) { + currentBuild.addTestTarget(label); + } + } + + @Subscribe + public void testTargetComplete(TestSummary summary) { + currentBuild.addTestSummary(summary.getTarget().getLabel(), summary.getStatus(), + summary.getTestTimes(), summary.isCached()); + } + + @Subscribe + public void testTargetResult(TestResult result) { + currentBuild.addTestResult(result.getTestAction().getOwner().getLabel(), + result.getData().getTestCase(), result.getShardNum()); + } + + @Subscribe + public void targetComplete(TargetCompleteEvent event) { + // TODO(bazel-team): would getting more details about failure be useful? + currentBuild.addTargetBuilt(event.getTarget().getTarget().getLabel(), !event.failed()); + } + + public WebStatusBuildLog getBuildLog() { + return this.currentBuild; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusServerModule.java b/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusServerModule.java new file mode 100644 index 0000000..13d4c8b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/WebStatusServerModule.java
@@ -0,0 +1,159 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.webstatusserver; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.BlazeServerStartupOptions; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsProvider; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.util.LinkedList; +import java.util.UUID; +import java.util.logging.Logger; + +/** + * Web server for monitoring blaze commands status. + */ +public class WebStatusServerModule extends BlazeModule { + static final String LAST_TEST_URI = "/tests/last"; + // 100 is an arbitrary limit; it seems like a reasonable size for history and it's okay to change + // it + private static final int MAX_TESTS_STORED = 100; + + private HttpServer server; + private boolean running = false; + private BlazeServerStartupOptions serverOptions; + private static final Logger LOG = + Logger.getLogger(WebStatusServerModule.class.getCanonicalName()); + private int port; + private LinkedList<TestStatusHandler> testsRan = new LinkedList<>(); + @SuppressWarnings("unused") + private WebStatusEventCollector collector; + @SuppressWarnings("unused") + private IndexPageHandler indexHandler; + + @Override + public Iterable<Class<? extends OptionsBase>> getStartupOptions() { + return ImmutableList.<Class<? extends OptionsBase>>of(BlazeServerStartupOptions.class); + } + + @Override + public void blazeStartup(OptionsProvider startupOptions, BlazeVersionInfo versionInfo, + UUID instanceId, BlazeDirectories directories, Clock clock) throws AbruptExitException { + serverOptions = startupOptions.getOptions(BlazeServerStartupOptions.class); + if (serverOptions.useWebStatusServer <= 0) { + LOG.info("web status server disabled"); + return; + } + port = serverOptions.useWebStatusServer; + try { + server = HttpServer.create(new InetSocketAddress(port), 0); + serveStaticContent(); + TextHandler lastCommandHandler = new TextHandler("No commands ran yet."); + server.createContext("/last", lastCommandHandler); + server.setExecutor(null); + server.start(); + indexHandler = new IndexPageHandler(server, this.testsRan); + running = true; + LOG.info("Running web status server on port " + port); + } catch (IOException e) { + // TODO(bazel-team): Display information about why it failed + running = false; + LOG.warning("Unable to run web status server on port " + port); + } + } + + @Override + public void beforeCommand(BlazeRuntime blazeRuntime, Command command) throws AbruptExitException { + if (!running) { + return; + } + collector = + new WebStatusEventCollector(blazeRuntime.getEventBus(), blazeRuntime.getReporter(), this); + } + + public void commandStarted() { + WebStatusBuildLog currentBuild = collector.getBuildLog(); + + if (testsRan.size() == MAX_TESTS_STORED) { + TestStatusHandler oldestTest = testsRan.removeLast(); + oldestTest.deregister(); + } + + TestStatusHandler lastTest = new TestStatusHandler(server, currentBuild); + testsRan.add(lastTest); + + lastTest.overrideURI(LAST_TEST_URI); + } + + private void serveStaticContent() { + StaticResourceHandler testjs = + StaticResourceHandler.createFromRelativePath("static/test.js", "application/javascript"); + StaticResourceHandler indexjs = + StaticResourceHandler.createFromRelativePath("static/index.js", "application/javascript"); + StaticResourceHandler style = + StaticResourceHandler.createFromRelativePath("static/style.css", "text/css"); + StaticResourceHandler d3 = StaticResourceHandler.createFromAbsolutePath( + "third_party/javascript/d3/d3-js.js", "application/javascript"); + StaticResourceHandler jquery = StaticResourceHandler.createFromAbsolutePath( + "third_party/javascript/jquery/v2_0_3/jquery_uncompressed.jslib", + "application/javascript"); + StaticResourceHandler testFrontend = + StaticResourceHandler.createFromRelativePath("static/test.html", "text/html"); + + server.createContext("/css/style.css", style); + server.createContext("/js/test.js", testjs); + server.createContext("/js/index.js", indexjs); + server.createContext("/js/lib/d3.js", d3); + server.createContext("/js/lib/jquery.js", jquery); + server.createContext(LAST_TEST_URI, testFrontend); + } + + private static class TextHandler implements HttpHandler { + private String response; + + private TextHandler(String response) { + this.response = response; + } + + @Override + public void handle(HttpExchange exchange) throws IOException { + exchange.getResponseHeaders().put("Content-Type", ImmutableList.of("text/plain")); + exchange.sendResponseHeaders(200, response.length()); + OutputStream os = exchange.getResponseBody(); + os.write(response.getBytes()); + os.close(); + } + } + + public int getPort() { + return port; + } +} +
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/static/index.html b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/index.html new file mode 100644 index 0000000..f57bc30 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/index.html
@@ -0,0 +1,14 @@ +<html> +<head> + <title> Bazel web server </title> + <link rel="stylesheet" type="text/css" href="/css/style.css"></link> + <script src="/js/lib/d3.js" type="application/javascript"></script> + <script src="/js/lib/jquery.js" type="application/javascript"></script> + <script src="/js/index.js" type="application/javascript"></script> +</head> +<body onload="showData()"> + <h1> Bazel web server status page </h1> + <div id="testsList"> + </div> +</body> +</html>
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/static/index.js b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/index.js new file mode 100644 index 0000000..4ef9671 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/index.js
@@ -0,0 +1,76 @@ +var icons = { + running: '\u25B6', + finished: '\u2611' +}; + +function showData() { + renderTestList(getTestsData()); +} + +function getTestsData() { + // TODO(bazel-team): change it to async callback retrieving data in background + // (for simplicity this is synchronous now) + return $.ajax({ + type: 'GET', + url: document.URL + 'tests/list', + async: false + }).responseJSON; +} + +function renderTestList(tests) { + var rows = d3.select('#testsList') + .selectAll() + .data(tests) + .enter().append('div') + .classed('info-cell', true); + + // status + rows.append('div').classed('info-detail', true).text(function(j) { + return j.finished ? icons.finished : icons.running; + }); + + // target(s) name(s) + rows.append('div').classed('info-detail', true).text(function(j) { + if (j.targets.length == 1) { + return j.targets[0]; + } + if (j.targets.length == 0) { + return 'Unknown target.'; + } + return j.targets; + }); + + // start time + rows.append('div').classed('info-detail', true).text(function(j) { + // Pad value with 2 zeroes + function pad(value) { + return value < 10 ? '0' + value : value; + } + + var + date = new Date(j.startTime), + today = new Date(Date.now()), + h = pad(date.getHours()), + m = pad(date.getMinutes()), + dd = pad(date.getDay()), + mm = pad(date.getMonth()), + yy = date.getYear(), + day; + + // don't show date if ran today + if (dd != today.getDay() && mm != today.getMonth() && + yy != today.getYear()) { + day = ' on ' + yy + '-' + mm + '-' + dd; + } else { + day = ''; + } + return h + ':' + m; + }); + + // link + rows.append('div').classed('info-detail', true).classed('button', true) + .append('a').attr('href', function(datum, index) { + return '/tests/' + datum.uuid; + }) + .text('link'); +}
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/static/test.html b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/test.html new file mode 100644 index 0000000..04a6fb7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/test.html
@@ -0,0 +1,28 @@ +<html> +<head><title>Tests Result Page</title> + <link rel="stylesheet" type="text/css" href="/css/style.css"></link> + <script src="/js/lib/d3.js" type="application/javascript"></script> + <script src="/js/lib/jquery.js" type="application/javascript"></script> + <script src="/js/test.js" type="application/javascript"></script> +</head> +<body> +<h1> Bazel web status server </h1> +<div id="testInfo"> + No test info to display. +</div> +<br> +<div id="testFilters"> + <div class="info-cell"> + <input placeholder="Filter by name" type=text id="search"></input> + <!-- TODO(bazel-team) this is very simplistic view of tests, + we probably need more filters --> + <input type=checkbox checked=true id="boxPassed">passed</input> + <input type=checkbox checked=true id="boxFailed">failed</input> + <button id="clearFilters"> clear filters </button> + </div> +</div> +<div id="testDetails"> + No test details to display. +</div> +</body> +</html>
diff --git a/src/main/java/com/google/devtools/build/lib/webstatusserver/static/test.js b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/test.js new file mode 100644 index 0000000..406dcab --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/webstatusserver/static/test.js
@@ -0,0 +1,384 @@ +var icons = { + running: '?', + passed: '\u2705', + errors: '\u274c' +}; + + +function showData() { + renderDetails(getDetailsData(), false); + renderInfo(getCommandInfo()); +} + +function getCommandInfo() { + var url = document.URL; + if (url[url.length - 1] != '/') { + url += '/'; + } + return $.ajax({ + type: 'GET', + url: url + 'info', + async: false + }).responseJSON; +} + +function getDetailsData() { + // TODO(bazel-team): auto refresh, async callback + var url = document.URL; + if (url[url.length - 1] != '/') { + url += '/'; + } + return $.ajax({ + type: 'GET', + url: url + 'details', + async: false + }).responseJSON; +} + + +function showDate(d) { + function pad(x) { + return x < 10 ? '0' + x : '' + x; + } + var today = new Date(); + var result = pad(d.getHours()) + ':' + pad(d.getMinutes()) + ':' + + pad(d.getSeconds()); + if (d.getDate() === today.getDate() && d.getMonth() === today.getMonth() && + d.getYear() === today.getYear()) { + result += ' today'; + } else { + result += pad(d.getDate()) + ' ' + pad(d.getMonth()) + ' ' + d.getYear(); + } + return result; +} + +function renderInfo(info) { + $('#testInfo').empty(); + var data = [ + ['Targets: ', info['targets']], + ['Started at: ', showDate(new Date(info['startTime']))], + ]; + if (info['finished']) { + data.push(['Finished at: ', showDate(new Date(info['endTime']))]); + } else { + data.push(['Still running']); + } + var selection = d3.select('#testInfo').selectAll() + .data(data) + .enter().append('div') + .classed('info-cell', true); + selection + .append('div') + .classed('info-detail', true) + .text(function(d) { return d[0]; }); + selection + .append('div') + .classed('info-detail', true) + .text(function(d) { return d[1]; }); +} + +// predicate is either a predicate function or null - in the latter case +// everything is shown +function renderDetails(tests, predicate) { + $('#testDetails').empty(); + if (tests.length == 0) { + $('#testDetails').text('No test details to display.'); + return; + } + // flatten object to array and set visibility + tests = $.map(tests, function(element) { + if (predicate) { + setVisibility(predicate, element); + } + return element; + }); + var rows = d3.select('#testDetails').selectAll() + .data(tests) + .enter().append('div') + .classed('test-case', true); + + function addTestDetail(selection, toplevel) { + function fullName() { + selection.append('div').classed('test-detail', true).text(function(j) { + return j.fullName; + }); + } + function propagateStatus(j) { + var result = ''; + var failures = []; + var errors = []; + $.each(j.results, function(key, value) { + if (value == 'FAILED') { + failures.push(key); + } + if (value == 'ERROR') { + errors.push(key); + } + }); + if (failures.length > 0) { + var s = failures.length > 1 ? 's' : ''; + result += 'Failed on ' + failures.length + ' shard' + s + ': ' + + failures.join(); + } + if (errors.length > 0) { + var s = failures.length > 1 ? 's' : ''; + result += 'Errors on ' + errors.length + ' shard' + s + ': ' + + errors.join(); + } + if (result == '') { + return j.status; + } + return result; + } + function testCaseStatus() { + selection.append('div') + .classed('test-detail', true) + .text(propagateStatus); + } + function testTargetStatus() { + selection.append('div') + .classed('test-detail', true) + .text(function(target) { + var childStatus = propagateStatus(target); + if (target.finished = false) { + return target.status + ' ' + stillRunning; + } else { + if (childStatus == 'PASSED') { + return target.status; + } else { + return target.status + ' ' + childStatus; + } + } + }); + } + function testTargetStatusIcon() { + selection.append('div') + .classed('test-detail', true) + .attr('color', function(target) { + var childStatus = propagateStatus(target); + if (target.finished == false) { + return 'running'; + } else { + if (childStatus == 'PASSED') { + return 'passed'; + } else { + return 'errors'; + } + }}) + .text(function(target) { + var childStatus = propagateStatus(target); + if (target.finished == false) { + return icons.running; + } else { + if (childStatus == 'PASSED') { + return icons.passed; + } else { + return icons.errors; + } + } + }); + } + function testCaseTime() { + selection.append('div').classed('test-detail', true).text(function(j) { + var times = $.map(j.times, function(element, key) { return element }); + if (times.length < 1) { + return '?'; + } else { + return Math.max.apply(Math, times) / 1000 + ' s'; + } + }); + } + + function visibilityFilter() { + selection.attr('show', function(datum) { + return ('show' in datum) ? datum['show'] : true; + }); + } + + // Toplevel nodes represent test targets, so they look a bit different + if (toplevel) { + testTargetStatusIcon(); + fullName(); + } else { + testTargetStatusIcon(); + fullName(); + testCaseStatus(); + testCaseTime(); + } + visibilityFilter(); + } + + function addNestedDetails(table, toplevel) { + table.sort(function(data1, data2) { + if (data1.fullName < data2.fullName) { + return -1; + } + if (data1.fullName > data2.fullName) { + return 1; + } + return 0; + }); + + addTestDetail(table, toplevel); + + // Add children nodes + show/hide button + var nonLeafNodes = table.filter(function(data, index) { + return !($.isEmptyObject(data.children)); + }); + var nextLevelNodes = nonLeafNodes.selectAll().data(function(d) { + return $.map(d.children, function(element, key) { return element }); + }); + + if (nextLevelNodes.enter().empty()) { + return; + } + + nonLeafNodes + .append('div') + .classed('test-detail', true) + .classed('button', true) + .text(function(j) { + return 'Show details'; + }) + .attr('toggle', 'off') + .on('click', function(datum) { + if ($(this).attr('toggle') == 'on') { + $(this).siblings('.test-case').not('[show=false]').hide(); + $(this).attr('toggle', 'off'); + $(this).text('Show details'); + } else { + $(this).siblings('.test-case').not('[show=false]').show(); + $(this).attr('toggle', 'on'); + $(this).text('Hide details'); + } + }); + nextLevelNodes.enter().append('div').classed('test-case', true); + addNestedDetails(nextLevelNodes, false); + } + + addNestedDetails(rows, true); + $('.button').siblings('.test-case').hide(); + if (predicate) { + toggleVisibility(); + } +} + +function toggleVisibility() { + $('#testDetails > [show=false]').hide(); + $('#testDetails > [show=true]').show(); + $('[toggle=on]').siblings('[show=false]').hide(); + $('[toggle=on]').siblings('[show=true]').show(); +} + +function setVisibility(predicate, object) { + var show = predicate(object); + var childrenPredicate = predicate; + // It rarely makes sense to show a non-leaf node and hide its children, so + // we just show all children + if (show) { + childrenPredicate = function() { return true; }; + } + if ('children' in object) { + for (var child in object.children) { + setVisibility(childrenPredicate, object.children[child]); + show = object.children[child]['show'] || show; + } + } + object['show'] = show; +} + +// given a list of predicates, return a function +function intersectFilters(filterList) { + var filters = filterList.filter(function(x) { return x }); + return function(x) { + for (var i = 0; i < filters.length; i++) { + if (!filters[i](x)) { + return false; + } + } + return true; + } +} + +function textFilterActive() { + return $('#search').val(); +} + +function getTestFilters() { + var statusFilter = null; + var textFilter = null; + var filters = []; + var passed = $('#boxPassed').prop('checked'); + var failed = $('#boxFailed').prop('checked'); + // add checkbox filters only when necessary (ie. something is unchecked - when + // everything is checked this means user wants to see everything). + if (!(passed && failed)) { + var checkBoxFilters = []; + if (passed) { + checkBoxFilters.push(function(object) { + return object.status == 'PASSED'; + }); + } + if (failed) { + checkBoxFilters.push(function(object) { + return 'status' in object && object.status != 'PASSED'; + }); + } + filters.push(function(object) { + return checkBoxFilters.some(function(f) { return f(object); }); + }); + } + if (textFilterActive()) { + filters.push(function(object) { + // TODO(bazel-team): would case insentive search make more sense? + return ('fullName' in object && + object.fullName.indexOf($('#search').val()) != -1); + }); + } + return filters; +} + +function redraw() { + renderDetails(getDetailsData(), intersectFilters(getTestFilters())); +} + +function updateVisibleCases() { + var predicate = intersectFilters(getTestFilters()); + var parentCases = d3.selectAll('#testDetails > div').data(); + parentCases.forEach(function(element, index) { + setVisibility(predicate, element); + }); + d3.selectAll('.test-detail').attr('show', function(datum) { + return ('show' in datum) ? datum['show'] : true; + }); + d3.selectAll('.test-case').attr('show', function(datum) { + return ('show' in datum) ? datum['show'] : true; + }); + toggleVisibility(); + if (textFilterActive()) { + // expand nodes to save some clicking - if user searched for something that + // is leaf of the tree, she definitely wants to see it + $('#testDetails > [show=true]').find('[toggle=off]').click(); + } +} + +function enableControls() { + var redrawTimeout = null; + $('#boxPassed').click(updateVisibleCases); + $('#boxFailed').click(updateVisibleCases); + $('#search').keyup(function() { + clearTimeout(redrawTimeout); + redrawTimeout = setTimeout(updateVisibleCases, 500); + }); + $('#clearFilters').click(function() { + $('#boxPassed').prop('checked', true); + $('#boxFailed').prop('checked', true); + $('#search').val(''); + updateVisibleCases(); + }); +} + +$(function() { + showData(); + enableControls(); +});
diff --git a/src/main/java/com/google/devtools/build/skyframe/BuildDriver.java b/src/main/java/com/google/devtools/build/skyframe/BuildDriver.java new file mode 100644 index 0000000..938735b --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/BuildDriver.java
@@ -0,0 +1,32 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.skyframe; + +import com.google.devtools.build.lib.events.EventHandler; + +/** + * A BuildDriver wraps a MemoizingEvaluator, passing along the proper Version. + */ +public interface BuildDriver { + /** + * See {@link MemoizingEvaluator#evaluate}, which has the same semantics except for the + * inclusion of a {@link Version} value. + */ + <T extends SkyValue> EvaluationResult<T> evaluate( + Iterable<SkyKey> roots, boolean keepGoing, int numThreads, EventHandler reporter) + throws InterruptedException; + + MemoizingEvaluator getGraphForTesting(); +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/BuildingState.java b/src/main/java/com/google/devtools/build/skyframe/BuildingState.java new file mode 100644 index 0000000..21deec1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/BuildingState.java
@@ -0,0 +1,437 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.util.GroupedList; +import com.google.devtools.build.lib.util.GroupedList.GroupedListHelper; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + * Data the NodeEntry uses to maintain its state before it is done building. It allows the + * {@link NodeEntry} to keep the current state of the entry across invalidation and successive + * evaluations. A done node does not contain any of this data. However, if a node is marked dirty, + * its entry acquires a new {@code BuildingState} object, which persists until it is done again. + * + * <p>This class should be considered a private inner class of {@link NodeEntry} -- no other + * classes should instantiate a {@code BuildingState} object or call any of its methods directly. + * It is in a separate file solely to keep the {@link NodeEntry} class readable. In particular, the + * caller must synchronize access to this class. + */ +@ThreadCompatible +final class BuildingState { + enum DirtyState { + /** + * The node's dependencies need to be checked to see if it needs to be rebuilt. The + * dependencies must be obtained through calls to {@link #getNextDirtyDirectDeps} and checked. + */ + CHECK_DEPENDENCIES, + /** + * All of the node's dependencies are unchanged, and the value itself was not marked changed, + * so its current value is still valid -- it need not be rebuilt. + */ + VERIFIED_CLEAN, + /** + * A rebuilding is required or in progress, because either the node itself changed or one of + * its dependencies did. + */ + REBUILDING + } + + /** + * During its life, a node can go through states as follows: + * <ol> + * <li>Non-existent + * <li>Just created ({@code evaluating} is false) + * <li>Evaluating ({@code evaluating} is true) + * <li>Done (meaning this buildingState object is null) + * <li>Just created (when it is dirtied during evaluation) + * <li>Reset (just before it is re-evaluated) + * <li>Evaluating + * <li>Done + * </ol> + * + * <p>The "just created" state is there to allow the {@link EvaluableGraph#createIfAbsent} and + * {@link NodeEntry#addReverseDepAndCheckIfDone} methods to be separate. All callers have to + * call both methods in that order if they want to create a node. The second method calls + * {@link #startEvaluating}, which transitions the current node to the "evaluating" state and + * returns true only the first time it was called. A caller that gets "true" back from that call + * must start the evaluation of this node, while any subsequent callers must not. + * + * <p>An entry is set to "evaluating" as soon as it is scheduled for evaluation. Thus, even a + * node that is never actually built (for instance, a dirty node that is verified as clean) is + * in the "evaluating" state until it is done. + */ + private boolean evaluating = false; + + /** + * The state of a dirty node. A node is marked dirty in the BuildingState constructor, and goes + * into either the state {@link DirtyState#CHECK_DEPENDENCIES} or {@link DirtyState#REBUILDING}, + * depending on whether the caller specified that the node was itself changed or not. A non-null + * {@code dirtyState} indicates that the node {@link #isDirty} in some way. + */ + private DirtyState dirtyState = null; + + /** + * The number of dependencies that are known to be done in a {@link NodeEntry}. There is a + * potential check-then-act race here, so we need to make sure that when this is increased, we + * always check if the new value is equal to the number of required dependencies, and if so, we + * must re-schedule the node for evaluation. + * + * <p>There are two potential pitfalls here: 1) If multiple dependencies signal this node in + * close succession, this node should be scheduled exactly once. 2) If a thread is still working + * on this node, it should not be scheduled. + * + * <p>The first problem is solved by the {@link #signalDep} method, which also returns if the + * node needs to be re-scheduled, and ensures that only one thread gets a true return value. + * + * <p>The second problem is solved by first adding the newly discovered deps to a node's + * {@link #directDeps}, and then looping through the direct deps and registering this node as a + * reverse dependency. This ensures that the signaledDeps counter can only reach + * {@link #directDeps}.size() on the very last iteration of the loop, i.e., the thread is not + * working on the node anymore. Note that this requires that there is no code after the loop in + * {@code ParallelEvaluator.Evaluate#run}. + */ + private int signaledDeps = 0; + + /** + * Direct dependencies discovered during the build. They will be written to the immutable field + * {@code ValueEntry#directDeps} and the dependency group data to {@code ValueEntry#groupData} + * once the node is finished building. {@link SkyFunction}s can request deps in groups, and these + * groupings are preserved in this field. + */ + private final GroupedList<SkyKey> directDeps = new GroupedList<>(); + + /** + * The set of reverse dependencies that are registered before the node has finished building. + * Upon building, these reverse deps will be signaled and then stored in the permanent + * {@code ValueEntry#reverseDeps}. + */ + // TODO(bazel-team): Remove this field. With eager invalidation, all direct deps on this dirty + // node will be removed by the time evaluation starts, so reverse deps to signal can just be + // reverse deps in the main ValueEntry object. + private Object reverseDepsToSignal = ImmutableList.of(); + private List<SkyKey> reverseDepsToRemove = null; + private boolean reverseDepIsSingleObject = false; + + private static final ReverseDepsUtil<BuildingState> REVERSE_DEPS_UTIL = + new ReverseDepsUtil<BuildingState>() { + @Override + void setReverseDepsObject(BuildingState container, Object object) { + container.reverseDepsToSignal = object; + } + + @Override + void setSingleReverseDep(BuildingState container, boolean singleObject) { + container.reverseDepIsSingleObject = singleObject; + } + + @Override + void setReverseDepsToRemove(BuildingState container, List<SkyKey> object) { + container.reverseDepsToRemove = object; + } + + @Override + Object getReverseDepsObject(BuildingState container) { + return container.reverseDepsToSignal; + } + + @Override + boolean isSingleReverseDep(BuildingState container) { + return container.reverseDepIsSingleObject; + } + + @Override + List<SkyKey> getReverseDepsToRemove(BuildingState container) { + return container.reverseDepsToRemove; + } + }; + + // Below are fields that are used for dirty nodes. + + /** + * The dependencies requested (with group markers) last time the node was built (and below, the + * value last time the node was built). They will be compared to dependencies requested on this + * build to check whether this node has changed in {@link NodeEntry#setValue}. If they are null, + * it means that this node is being built for the first time. See {@link #directDeps} for more on + * dependency group storage. + */ + private final GroupedList<SkyKey> lastBuildDirectDeps; + private final SkyValue lastBuildValue; + + /** + * Which child should be re-evaluated next in the process of determining if this entry needs to + * be re-evaluated. Used by {@link #getNextDirtyDirectDeps} and {@link #signalDep(boolean)}. + */ + private Iterator<Iterable<SkyKey>> dirtyDirectDepIterator = null; + + BuildingState() { + lastBuildDirectDeps = null; + lastBuildValue = null; + } + + private BuildingState(boolean isChanged, GroupedList<SkyKey> lastBuildDirectDeps, + SkyValue lastBuildValue) { + this.lastBuildDirectDeps = lastBuildDirectDeps; + this.lastBuildValue = Preconditions.checkNotNull(lastBuildValue); + Preconditions.checkState(isChanged || !this.lastBuildDirectDeps.isEmpty(), + "is being marked dirty, not changed, but has no children that could have dirtied it", this); + dirtyState = isChanged ? DirtyState.REBUILDING : DirtyState.CHECK_DEPENDENCIES; + if (dirtyState == DirtyState.CHECK_DEPENDENCIES) { + // We need to iterate through the deps to see if they have changed. Initialize the iterator. + dirtyDirectDepIterator = lastBuildDirectDeps.iterator(); + } + } + + static BuildingState newDirtyState(boolean isChanged, + GroupedList<SkyKey> lastBuildDirectDeps, SkyValue lastBuildValue) { + return new BuildingState(isChanged, lastBuildDirectDeps, lastBuildValue); + } + + void markChanged() { + Preconditions.checkState(isDirty(), this); + Preconditions.checkState(!isChanged(), this); + Preconditions.checkState(!evaluating, this); + dirtyState = DirtyState.REBUILDING; + } + + void forceChanged() { + Preconditions.checkState(isDirty(), this); + Preconditions.checkState(!isChanged(), this); + Preconditions.checkState(evaluating, this); + Preconditions.checkState(isReady(), this); + dirtyState = DirtyState.REBUILDING; + } + + /** + * Returns whether all known children of this node have signaled that they are done. + */ + boolean isReady() { + int directDepsSize = directDeps.size(); + Preconditions.checkState(signaledDeps <= directDepsSize, "%s %s", directDepsSize, this); + return signaledDeps == directDepsSize; + } + + /** + * Returns true if the entry is marked dirty, meaning that at least one of its transitive + * dependencies is marked changed. + * + * @see NodeEntry#isDirty() + */ + boolean isDirty() { + return dirtyState != null; + } + + /** + * Returns true if the entry is known to require re-evaluation. + * + * @see NodeEntry#isChanged() + */ + boolean isChanged() { + return dirtyState == DirtyState.REBUILDING; + } + + private boolean rebuilding() { + return dirtyState == DirtyState.REBUILDING; + } + + /** + * Helper method to assert that node has finished building, as far as we can tell. We would + * actually like to check that the node has been evaluated, but that is not available in + * this context. + */ + private void checkNotProcessing() { + Preconditions.checkState(evaluating, "not started building %s", this); + Preconditions.checkState(!isDirty() || dirtyState == DirtyState.VERIFIED_CLEAN + || rebuilding(), "not done building %s", this); + Preconditions.checkState(isReady(), "not done building %s", this); + } + + /** + * Puts the node in the "evaluating" state if it is not already in it. Returns whether or not the + * node was already evaluating. Should only be called by + * {@link NodeEntry#addReverseDepAndCheckIfDone}. + */ + boolean startEvaluating() { + boolean result = !evaluating; + evaluating = true; + return result; + } + + /** + * Increments the number of children known to be finished. Returns true if the number of children + * finished is equal to the number of known children. + * + * <p>If the node is dirty and checking its deps for changes, this also updates {@link + * #dirtyState} as needed -- {@link DirtyState#REBUILDING} if the child has changed, + * and {@link DirtyState#VERIFIED_CLEAN} if the child has not changed and this was the last + * child to be checked (as determined by {@link #dirtyDirectDepIterator} == null, isReady(), and + * a flag set in {@link #getNextDirtyDirectDeps}). + * + * @see NodeEntry#signalDep(Version) + */ + boolean signalDep(boolean childChanged) { + signaledDeps++; + if (isDirty() && !rebuilding()) { + // Synchronization isn't needed here because the only caller is ValueEntry, which does it + // through the synchronized method signalDep(long). + if (childChanged) { + dirtyState = DirtyState.REBUILDING; + } else if (dirtyState == DirtyState.CHECK_DEPENDENCIES && isReady() + && dirtyDirectDepIterator == null) { + // No other dep already marked this as REBUILDING, no deps outstanding, and this was + // the last block of deps to be checked. + dirtyState = DirtyState.VERIFIED_CLEAN; + } + } + return isReady(); + } + + /** + * Returns true if {@code newValue}.equals the value from the last time this node was built, and + * the deps requested during this evaluation are exactly those requested the last time this node + * was built, in the same order. Should only be used by {@link NodeEntry#setValue}. + */ + boolean unchangedFromLastBuild(SkyValue newValue) { + checkNotProcessing(); + return lastBuildValue.equals(newValue) && lastBuildDirectDeps.equals(directDeps); + } + + boolean noDepsLastBuild() { + return lastBuildDirectDeps.isEmpty(); + } + + SkyValue getLastBuildValue() { + return Preconditions.checkNotNull(lastBuildValue, this); + } + + /** + * Gets the current state of checking this dirty entry to see if it must be re-evaluated. Must be + * called each time evaluation of a dirty entry starts to find the proper action to perform next, + * as enumerated by {@link DirtyState}. + * + * @see NodeEntry#getDirtyState() + */ + DirtyState getDirtyState() { + // Entry may not be ready if being built just for its errors. + Preconditions.checkState(isDirty(), "must be dirty to get dirty state %s", this); + Preconditions.checkState(evaluating, "must be evaluating to get dirty state %s", this); + return dirtyState; + } + + /** + * Gets the next children to be re-evaluated to see if this dirty node needs to be re-evaluated. + * + * <p>If this is the last group of children to be checked, then sets {@link + * #dirtyDirectDepIterator} to null so that the final call to {@link #signalDep(boolean)} will + * know to mark this entry as {@link DirtyState#VERIFIED_CLEAN} if no deps have changed. + * + * See {@link NodeEntry#getNextDirtyDirectDeps}. + */ + Collection<SkyKey> getNextDirtyDirectDeps() { + Preconditions.checkState(isDirty(), this); + Preconditions.checkState(dirtyState == DirtyState.CHECK_DEPENDENCIES, this); + Preconditions.checkState(evaluating, this); + List<SkyKey> nextDeps = ImmutableList.copyOf(dirtyDirectDepIterator.next()); + if (!dirtyDirectDepIterator.hasNext()) { + // Done checking deps. If this last group is clean, the state will become VERIFIED_CLEAN. + dirtyDirectDepIterator = null; + } + return nextDeps; + } + + void addDirectDeps(GroupedListHelper<SkyKey> depsThisRun) { + directDeps.append(depsThisRun); + } + + /** + * Returns the direct deps found so far on this build. Should only be called before the node has + * finished building. + * + * @see NodeEntry#getTemporaryDirectDeps() + */ + Set<SkyKey> getDirectDepsForBuild() { + return directDeps.toSet(); + } + + /** + * Returns the direct deps (in groups) found on this build. Should only be called when the node + * is done. + * + * @see NodeEntry#setStateFinishedAndReturnReverseDeps + */ + GroupedList<SkyKey> getFinishedDirectDeps() { + return directDeps; + } + + /** + * Returns reverse deps to signal that have been registered this build. + * + * @see NodeEntry#getReverseDeps() + */ + ImmutableSet<SkyKey> getReverseDepsToSignal() { + return REVERSE_DEPS_UTIL.getReverseDeps(this); + } + + /** + * Adds a reverse dependency that should be notified when this entry is done. + * + * @see NodeEntry#addReverseDepAndCheckIfDone(SkyKey) + */ + void addReverseDepToSignal(SkyKey newReverseDep) { + REVERSE_DEPS_UTIL.consolidateReverseDepsRemovals(this); + REVERSE_DEPS_UTIL.addReverseDeps(this, Collections.singleton(newReverseDep)); + } + + /** + * @see NodeEntry#removeReverseDep(SkyKey) + */ + void removeReverseDepToSignal(SkyKey reverseDep) { + REVERSE_DEPS_UTIL.removeReverseDep(this, reverseDep); + } + + /** + * Removes a set of deps from the set of known direct deps. This is complicated by the need + * to maintain the group data. If we remove a dep that ended a group, then its predecessor's + * group data must be changed to indicate that it now ends the group. + * + * @see NodeEntry#removeUnfinishedDeps + */ + void removeDirectDeps(Set<SkyKey> unfinishedDeps) { + directDeps.remove(unfinishedDeps); + } + + @Override + @SuppressWarnings("deprecation") + public String toString() { + return Objects.toStringHelper(this) // MoreObjects is not in Guava + .add("evaluating", evaluating) + .add("dirtyState", dirtyState) + .add("signaledDeps", signaledDeps) + .add("directDeps", directDeps) + .add("reverseDepsToSignal", REVERSE_DEPS_UTIL.toString(this)) + .add("lastBuildDirectDeps", lastBuildDirectDeps) + .add("lastBuildValue", lastBuildValue) + .add("dirtyDirectDepIterator", dirtyDirectDepIterator).toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/CycleDeduper.java b/src/main/java/com/google/devtools/build/skyframe/CycleDeduper.java new file mode 100644 index 0000000..f52333c --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/CycleDeduper.java
@@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +/** + * Dedupes C candidate cycles of size O(L) in O(CL) time and memory in the common case and + * O(C^2 * L) time and O(CL) memory in the extreme case. + * + * Two cycles are considered duplicates if they are exactly the same except for the entry point. + * For example, 'a' -> 'b' -> 'c' -> 'a' is the considered the same as 'b' -> 'c' -> 'a' -> 'b'. + */ +class CycleDeduper<T> { + + private HashMultimap<ImmutableSet<T>, ImmutableList<T>> knownCyclesByMembers = + HashMultimap.create(); + + /** + * Marks a non-empty list representing a cycle of unique values as being seen and returns true + * iff the cycle hasn't been seen before, accounting for logical equivalence of cycles. + * + * For example, the cycle 'a' -> 'b' -> 'c' -> 'a' is represented by the list ['a', 'b', 'c'] + * and is logically equivalent to the cycle represented by the list ['b', 'c', 'a']. + */ + public boolean seen(ImmutableList<T> cycle) { + ImmutableSet<T> cycleMembers = ImmutableSet.copyOf(cycle); + Preconditions.checkState(!cycle.isEmpty()); + Preconditions.checkState(cycle.size() == cycleMembers.size(), + "cycle doesn't have unique members: " + cycle); + + if (knownCyclesByMembers.containsEntry(cycleMembers, cycle)) { + return false; + } + + // Of the C cycles, suppose there are D cycles that have the same members (but are in an + // incompatible order). This code path takes O(D * L) time. The common case is that D is + // very small. + boolean found = false; + for (ImmutableList<T> candidateCycle : knownCyclesByMembers.get(cycleMembers)) { + int startPos = candidateCycle.indexOf(cycle.get(0)); + // The use of a multimap keyed by cycle members guarantees that the first element of 'cycle' + // is present in 'candidateCycle'. + Preconditions.checkState(startPos >= 0); + if (equalsWithSingleLoopFrom(cycle, candidateCycle, startPos)) { + found = true; + break; + } + } + // We add the cycle even if it's a duplicate so that future exact copies of this can be + // processed in O(L) time. We are already using O(CL) memory, and this optimization doesn't + // change that. + knownCyclesByMembers.put(cycleMembers, cycle); + return !found; + } + + /** + * Returns true iff + * listA[0], listA[1], ..., listA[listA.size()] + * is the same as + * listB[start], listB[start+1], ..., listB[listB.size()-1], listB[0], ..., listB[start-1] + */ + private boolean equalsWithSingleLoopFrom(ImmutableList<T> listA, ImmutableList<T> listB, + int start) { + if (listA.size() != listB.size()) { + return false; + } + int length = listA.size(); + for (int i = 0; i < length; i++) { + if (!listA.get(i).equals(listB.get((i + start) % length))) { + return false; + } + } + return true; + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/CycleInfo.java b/src/main/java/com/google/devtools/build/skyframe/CycleInfo.java new file mode 100644 index 0000000..a44d2fa --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/CycleInfo.java
@@ -0,0 +1,144 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + +import java.io.Serializable; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/** + * Data for a single cycle in the graph, together with the path to the cycle. For any value, the + * head of path to the cycle should be the value itself, or, if the value is actually in the cycle, + * the cycle should start with the value. + */ +public class CycleInfo implements Serializable { + private final ImmutableList<SkyKey> cycle; + private final ImmutableList<SkyKey> pathToCycle; + + @VisibleForTesting + public CycleInfo(Iterable<SkyKey> cycle) { + this(ImmutableList.<SkyKey>of(), cycle); + } + + CycleInfo(Iterable<SkyKey> pathToCycle, Iterable<SkyKey> cycle) { + this.pathToCycle = ImmutableList.copyOf(pathToCycle); + this.cycle = ImmutableList.copyOf(cycle); + } + + // If a cycle is already known, but we are processing a value in the middle of the cycle, we need + // to shift the cycle so that the value is at the head. + private CycleInfo(Iterable<SkyKey> cycle, int cycleStart) { + Preconditions.checkState(cycleStart >= 0, cycleStart); + ImmutableList.Builder<SkyKey> cycleTail = ImmutableList.builder(); + ImmutableList.Builder<SkyKey> cycleHead = ImmutableList.builder(); + int index = 0; + for (SkyKey key : cycle) { + if (index >= cycleStart) { + cycleHead.add(key); + } else { + cycleTail.add(key); + } + index++; + } + Preconditions.checkState(cycleStart < index, "%s >= %s ??", cycleStart, index); + this.cycle = cycleHead.addAll(cycleTail.build()).build(); + this.pathToCycle = ImmutableList.of(); + } + + public ImmutableList<SkyKey> getCycle() { + return cycle; + } + + public ImmutableList<SkyKey> getPathToCycle() { + return pathToCycle; + } + + // Given a cycle and a value, if the value is part of the cycle, shift the cycle. Otherwise, + // prepend the value to the head of pathToCycle. + private static CycleInfo normalizeCycle(final SkyKey value, CycleInfo cycle) { + int index = cycle.cycle.indexOf(value); + if (index > -1) { + if (!cycle.pathToCycle.isEmpty()) { + // The head value we are considering is already part of a cycle, but we have reached it by a + // roundabout way. Since we should have reached it directly as well, filter this roundabout + // way out. Example (c has a dependence on top): + // top + // / ^ + // a | + // / \ / + // b-> c + // In the traversal, we start at top, visit a, then c, then top. This yields the + // cycle {top,a,c}. Then we visit b, getting (b, {top,a,c}). Then we construct the full + // error for a. The error should just be the cycle {top,a,c}, but we have an extra copy of + // it via the path through b. + return null; + } + return new CycleInfo(cycle.cycle, index); + } + return new CycleInfo(Iterables.concat(ImmutableList.of(value), cycle.pathToCycle), + cycle.cycle); + } + + /** + * Normalize multiple cycles. This includes removing multiple paths to the same cycle, so that + * a value does not depend on the same cycle multiple ways through the same child value. Note that + * a value can still depend on the same cycle multiple ways, it's just that each way must be + * through a different child value (a path with a different first element). + */ + static Iterable<CycleInfo> prepareCycles(final SkyKey value, Iterable<CycleInfo> cycles) { + final Set<ImmutableList<SkyKey>> alreadyDoneCycles = new HashSet<>(); + return Iterables.filter(Iterables.transform(cycles, + new Function<CycleInfo, CycleInfo>() { + @Override + public CycleInfo apply(CycleInfo input) { + CycleInfo normalized = normalizeCycle(value, input); + if (normalized != null && alreadyDoneCycles.add(normalized.cycle)) { + return normalized; + } + return null; + } + }), Predicates.notNull()); + } + + @Override + public int hashCode() { + return Objects.hash(cycle, pathToCycle); + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + if (!(that instanceof CycleInfo)) { + return false; + } + + CycleInfo thatCycle = (CycleInfo) that; + return thatCycle.cycle.equals(this.cycle) && thatCycle.pathToCycle.equals(this.pathToCycle); + } + + @Override + public String toString() { + return Iterables.toString(pathToCycle) + " -> " + Iterables.toString(cycle); + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/CyclesReporter.java b/src/main/java/com/google/devtools/build/skyframe/CyclesReporter.java new file mode 100644 index 0000000..a9b0d8e --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/CyclesReporter.java
@@ -0,0 +1,102 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.events.EventHandler; + +/** + * An utility for custom reporting of errors from cycles in the the Skyframe graph. This class is + * stateful in order to differentiate between new cycles and cycles that have already been + * reported (do not reuse the instances or cache the results as it could end up printing + * inconsistent information or leak memory). It treats two cycles as the same if they contain the + * same {@link SkyKey}s in the same order, but perhaps with different starting points. See + * {@link CycleDeduper} for more information. + */ +public class CyclesReporter { + + /** + * Interface for reporting custom information about a single cycle. + */ + public interface SingleCycleReporter { + + /** + * Reports the given cycle and returns {@code true}, or return {@code false} if this + * {@link SingleCycleReporter} doesn't know how to report the cycle. + * + * @param topLevelKey the top level key that transitively depended on the cycle + * @param cycleInfo the cycle + * @param alreadyReported whether the cycle has already been reported to the + * {@link CyclesReporter}. + * @param eventHandler the eventHandler to which to report the error + */ + boolean maybeReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo, boolean alreadyReported, + EventHandler eventHandler); + } + + private final ImmutableList<SingleCycleReporter> cycleReporters; + private final CycleDeduper<SkyKey> cycleDeduper = new CycleDeduper<>(); + + /** + * Constructs a {@link CyclesReporter} that delegates to the given {@link SingleCycleReporter}s, + * in the given order, to report custom information about cycles. + */ + public CyclesReporter(SingleCycleReporter... cycleReporters) { + this.cycleReporters = ImmutableList.copyOf(cycleReporters); + } + + /** + * Reports the given cycles, differentiating between cycles that have already been reported. + * + * @param cycles The {@code Iterable} of cycles. + * @param topLevelKey This key represents the top level value key that returned cycle errors. + * @param eventHandler the eventHandler to which to report the error + */ + public void reportCycles(Iterable<CycleInfo> cycles, SkyKey topLevelKey, + EventHandler eventHandler) { + Preconditions.checkNotNull(eventHandler); + for (CycleInfo cycleInfo : cycles) { + boolean alreadyReported = false; + if (!cycleDeduper.seen(cycleInfo.getCycle())) { + alreadyReported = true; + } + boolean successfullyReported = false; + for (SingleCycleReporter cycleReporter : cycleReporters) { + if (cycleReporter.maybeReportCycle(topLevelKey, cycleInfo, alreadyReported, eventHandler)) { + successfullyReported = true; + break; + } + } + Preconditions.checkState(successfullyReported, + printArbitraryCycle(topLevelKey, cycleInfo, alreadyReported)); + } + } + + private String printArbitraryCycle(SkyKey topLevelKey, CycleInfo cycleInfo, + boolean alreadyReported) { + StringBuilder cycleMessage = new StringBuilder() + .append("topLevelKey: " + topLevelKey + "\n") + .append("alreadyReported: " + alreadyReported + "\n") + .append("path to cycle:\n"); + for (SkyKey skyKey : cycleInfo.getPathToCycle()) { + cycleMessage.append(skyKey + "\n"); + } + cycleMessage.append("cycle:\n"); + for (SkyKey skyKey : cycleInfo.getCycle()) { + cycleMessage.append(skyKey + "\n"); + } + return cycleMessage.toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/Differencer.java b/src/main/java/com/google/devtools/build/skyframe/Differencer.java new file mode 100644 index 0000000..f6433ac --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/Differencer.java
@@ -0,0 +1,45 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import java.util.Map; + +/** + * Calculate set of changed values in a graph. + */ +public interface Differencer { + + /** + * Represents a set of changed values. + */ + interface Diff { + /** + * Returns the value keys whose values have changed, but for which we don't have the new values. + */ + Iterable<SkyKey> changedKeysWithoutNewValues(); + + /** + * Returns the value keys whose values have changed, along with their new values. + * + * <p> The values in here cannot have any dependencies. This is required in order to prevent + * conflation of injected values and derived values. + */ + Map<SkyKey, ? extends SkyValue> changedKeysWithNewValues(); + } + + /** + * Returns the value keys that have changed between the two Versions. + */ + Diff getDiff(Version fromVersion, Version toVersion) throws InterruptedException; +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/DirtiableGraph.java b/src/main/java/com/google/devtools/build/skyframe/DirtiableGraph.java new file mode 100644 index 0000000..0781222 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/DirtiableGraph.java
@@ -0,0 +1,28 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +/** + * Interface for classes that need to remove values from graph. Currently just used by {@link + * EagerInvalidator}. + * + * <p>This class is not intended for direct use, and is only exposed as public for use in + * evaluation implementations outside of this package. + */ +public interface DirtiableGraph extends QueryableGraph { + /** + * Remove the value with given name from the graph. + */ + void remove(SkyKey key); +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/DirtyKeyTracker.java b/src/main/java/com/google/devtools/build/skyframe/DirtyKeyTracker.java new file mode 100644 index 0000000..b0b5074 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/DirtyKeyTracker.java
@@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +import java.util.Set; + +/** + * Interface for implementations that need to keep track of dirty SkyKeys. + */ +public interface DirtyKeyTracker { + + /** + * Marks the {@code skyKey} as dirty. + */ + @ThreadSafe + void dirty(SkyKey skyKey); + + /** + * Marks the {@code skyKey} as not dirty. + */ + @ThreadSafe + void notDirty(SkyKey skyKey); + + /** + * Returns the set of keys k for which there was a call to dirty(k) but not a subsequent call + * to notDirty(k). + */ + @ThreadSafe + Set<SkyKey> getDirtyKeys(); +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/DirtyKeyTrackerImpl.java b/src/main/java/com/google/devtools/build/skyframe/DirtyKeyTrackerImpl.java new file mode 100644 index 0000000..e3e070cb --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/DirtyKeyTrackerImpl.java
@@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; + +import java.util.Set; + +/** Encapsulates a thread-safe set of SkyKeys. */ +public class DirtyKeyTrackerImpl implements DirtyKeyTracker { + + private final Set<SkyKey> dirtyKeys = Sets.newConcurrentHashSet(); + + @Override + public void dirty(SkyKey skyKey) { + dirtyKeys.add(skyKey); + } + + @Override + public void notDirty(SkyKey skyKey) { + dirtyKeys.remove(skyKey); + } + + @Override + public Set<SkyKey> getDirtyKeys() { + return ImmutableSet.copyOf(dirtyKeys); + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/EagerInvalidator.java b/src/main/java/com/google/devtools/build/skyframe/EagerInvalidator.java new file mode 100644 index 0000000..fc2a2c7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/EagerInvalidator.java
@@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DeletingNodeVisitor; +import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DirtyingNodeVisitor; +import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.InvalidationState; + +/** + * Utility class for performing eager invalidation on Skyframe graphs. + * + * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations. + */ +public final class EagerInvalidator { + + private EagerInvalidator() {} + + /** + * Deletes given values. The {@code traverseGraph} parameter controls whether this method deletes + * (transitive) dependents of these nodes and relevant graph edges, or just the nodes themselves. + * Deleting just the nodes is inconsistent unless the graph will not be used for incremental + * builds in the future, but unfortunately there is a case where we delete nodes intra-build. As + * long as the full upward transitive closure of the nodes is specified for deletion, the graph + * remains consistent. + */ + public static void delete(DirtiableGraph graph, Iterable<SkyKey> diff, + EvaluationProgressReceiver invalidationReceiver, InvalidationState state, + boolean traverseGraph, DirtyKeyTracker dirtyKeyTracker) throws InterruptedException { + InvalidatingNodeVisitor visitor = + createVisitor(/*delete=*/true, graph, diff, invalidationReceiver, state, traverseGraph, + dirtyKeyTracker); + if (visitor != null) { + visitor.run(); + } + } + + /** + * Creates an invalidation visitor that is ready to run. Caller should call #run() on the visitor. + * Allows test classes to keep a reference to the visitor, and await exceptions/interrupts. + */ + @VisibleForTesting + static InvalidatingNodeVisitor createVisitor(boolean delete, DirtiableGraph graph, + Iterable<SkyKey> diff, EvaluationProgressReceiver invalidationReceiver, + InvalidationState state, boolean traverseGraph, DirtyKeyTracker dirtyKeyTracker) { + state.update(diff); + if (state.isEmpty()) { + return null; + } + return delete + ? new DeletingNodeVisitor(graph, invalidationReceiver, state, traverseGraph, + dirtyKeyTracker) + : new DirtyingNodeVisitor(graph, invalidationReceiver, state, dirtyKeyTracker); + } + + /** + * Invalidates given values and their upward transitive closure in the graph. + */ + public static void invalidate(DirtiableGraph graph, Iterable<SkyKey> diff, + EvaluationProgressReceiver invalidationReceiver, InvalidationState state, + DirtyKeyTracker dirtyKeyTracker) + throws InterruptedException { + // If we are invalidating, we must be in an incremental build by definition, so we must + // maintain a consistent graph state by traversing the graph and invalidating transitive + // dependencies. If edges aren't present, it would be impossible to check the dependencies of + // a dirty node in any case. + InvalidatingNodeVisitor visitor = + createVisitor(/*delete=*/false, graph, diff, invalidationReceiver, state, + /*traverseGraph=*/true, dirtyKeyTracker); + if (visitor != null) { + visitor.run(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/EdgelessNodeEntry.java b/src/main/java/com/google/devtools/build/skyframe/EdgelessNodeEntry.java new file mode 100644 index 0000000..98fb61e --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/EdgelessNodeEntry.java
@@ -0,0 +1,32 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.skyframe; + +/** + * NodeEntry that does not store edges (directDeps and reverseDeps) when the node is done. Used to + * save memory when it is known that the graph will not be reused. + * + * <p>Graph edges must be stored for incremental builds, but if this program will terminate after a + * single run, edges can be thrown away in order to save memory. The edges will be stored in the + * {@link BuildingState} as usual while the node is being built, but will not be stored once the + * node is done and written to the graph. Any attempt to access the edges once the node is done will + * fail the build fast. + */ +class EdgelessNodeEntry extends NodeEntry { + @Override + protected boolean keepEdges() { + return false; + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ErrorInfo.java b/src/main/java/com/google/devtools/build/skyframe/ErrorInfo.java new file mode 100644 index 0000000..6873d19 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/ErrorInfo.java
@@ -0,0 +1,157 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.skyframe.SkyFunctionException.ReifiedSkyFunctionException; + +import java.io.Serializable; +import java.util.Collection; + +import javax.annotation.Nullable; + +/** + * Information about why a {@link SkyValue} failed to evaluate successfully. + * + * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations. + */ +public class ErrorInfo implements Serializable { + /** + * The set of descendants of this value that failed to build + */ + private final NestedSet<SkyKey> rootCauses; + + /** + * An exception thrown upon a value's failure to build. The exception is used for reporting, and + * thus may ultimately be rethrown by the caller. As well, during a --nokeep_going evaluation, if + * an error value is encountered from an earlier --keep_going build, the exception to be thrown is + * taken from here. + */ + @Nullable private final Exception exception; + private final SkyKey rootCauseOfException; + + private final Iterable<CycleInfo> cycles; + + private final boolean isTransient; + private final boolean isCatastrophic; + + public ErrorInfo(ReifiedSkyFunctionException builderException) { + this.rootCauseOfException = builderException.getRootCauseSkyKey(); + this.rootCauses = NestedSetBuilder.create(Order.STABLE_ORDER, rootCauseOfException); + this.exception = Preconditions.checkNotNull(builderException.getCause(), builderException); + this.cycles = ImmutableList.of(); + this.isTransient = builderException.isTransient(); + this.isCatastrophic = builderException.isCatastrophic(); + } + + ErrorInfo(CycleInfo cycleInfo) { + this.rootCauses = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + this.exception = null; + this.rootCauseOfException = null; + this.cycles = ImmutableList.of(cycleInfo); + this.isTransient = false; + this.isCatastrophic = false; + } + + public ErrorInfo(SkyKey currentValue, Collection<ErrorInfo> childErrors) { + Preconditions.checkNotNull(currentValue); + Preconditions.checkState(!childErrors.isEmpty(), + "Error value %s with no exception must depend on another error value", currentValue); + NestedSetBuilder<SkyKey> builder = NestedSetBuilder.stableOrder(); + ImmutableList.Builder<CycleInfo> cycleBuilder = ImmutableList.builder(); + Exception firstException = null; + SkyKey firstChildKey = null; + boolean isTransient = false; + boolean isCatastrophic = false; + // Arbitrarily pick the first error. + for (ErrorInfo child : childErrors) { + if (firstException == null) { + firstException = child.getException(); + firstChildKey = child.getRootCauseOfException(); + } + builder.addTransitive(child.rootCauses); + cycleBuilder.addAll(CycleInfo.prepareCycles(currentValue, child.cycles)); + isTransient |= child.isTransient(); + isCatastrophic |= child.isCatastrophic(); + } + this.rootCauses = builder.build(); + this.exception = firstException; + this.rootCauseOfException = firstChildKey; + this.cycles = cycleBuilder.build(); + this.isTransient = isTransient; + this.isCatastrophic = isCatastrophic; + } + + @Override + public String toString() { + return String.format("<ErrorInfo exception=%s rootCauses=%s cycles=%s>", + exception, rootCauses, cycles); + } + + /** + * The root causes of a value that failed to build are its descendant values that failed to build. + * If a value's descendants all built successfully, but it failed to, its root cause will be + * itself. If a value depends on a cycle, but has no other errors, this method will return + * the empty set. + */ + public Iterable<SkyKey> getRootCauses() { + return rootCauses; + } + + /** + * The exception thrown when building a value. May be null if value's only error is depending + * on a cycle. + */ + @Nullable public Exception getException() { + return exception; + } + + public SkyKey getRootCauseOfException() { + return rootCauseOfException; + } + + /** + * Any cycles found when building this value. + * + * <p>If there are a large number of cycles, only a limited number are returned here. + * + * <p>If this value has a child through which there are multiple paths to the same cycle, only one + * path is returned here. However, if there are multiple paths to the same cycle, each of which + * goes through a different child, each of them is returned here. + */ + public Iterable<CycleInfo> getCycleInfo() { + return cycles; + } + + /** + * Returns true iff the error is transient, i.e. if retrying the same computation could lead to a + * different result. + */ + public boolean isTransient() { + return isTransient; + } + + + /** + * Returns true iff the error is catastrophic, i.e. it should halt even for a keepGoing update() + * call. + */ + public boolean isCatastrophic() { + return isCatastrophic; + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ErrorTransienceValue.java b/src/main/java/com/google/devtools/build/skyframe/ErrorTransienceValue.java new file mode 100644 index 0000000..c0c445d --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/ErrorTransienceValue.java
@@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +/** + * A value that represents "error transience", i.e. anything which may have caused an unexpected + * failure. + */ +public final class ErrorTransienceValue implements SkyValue { + public static final SkyFunctionName FUNCTION_NAME = + new SkyFunctionName("ERROR_TRANSIENCE", false); + + ErrorTransienceValue() {} + + public static SkyKey key() { + return new SkyKey(FUNCTION_NAME, "ERROR_TRANSIENCE"); + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/EvaluableGraph.java b/src/main/java/com/google/devtools/build/skyframe/EvaluableGraph.java new file mode 100644 index 0000000..3d9a934 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/EvaluableGraph.java
@@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +/** + * Interface between a single version of the graph and the evaluator. Supports mutation of that + * single version of the graph. + */ +interface EvaluableGraph extends QueryableGraph { + /** + * Creates a new node with the specified key if it does not exist yet. Returns the node entry + * (either the existing one or the one just created), never {@code null}. + */ + NodeEntry createIfAbsent(SkyKey key); +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/EvaluationProgressReceiver.java b/src/main/java/com/google/devtools/build/skyframe/EvaluationProgressReceiver.java new file mode 100644 index 0000000..7928878 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/EvaluationProgressReceiver.java
@@ -0,0 +1,77 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.devtools.build.lib.concurrent.ThreadSafety; + +/** + * Receiver to inform callers which values have been invalidated. Values may be invalidated and then + * re-validated if they have been found not to be changed. + */ +public interface EvaluationProgressReceiver { + /** + * New state of the value entry after evaluation. + */ + enum EvaluationState { + /** The value was successfully re-evaluated. */ + BUILT, + /** The value is clean or re-validated. */ + CLEAN, + } + + /** + * New state of the value entry after invalidation. + */ + enum InvalidationState { + /** The value is dirty, although it might get re-validated again. */ + DIRTY, + /** The value is dirty and got deleted, cannot get re-validated again. */ + DELETED, + } + + /** + * Notifies that {@code value} has been invalidated. + * + * <p>{@code state} indicates the new state of the value. + * + * <p>This method is not called on invalidation of values which do not have a value (usually + * because they are in error). + * + * <p>May be called concurrently from multiple threads, possibly with the same {@code value} + * object. + */ + @ThreadSafety.ThreadSafe + void invalidated(SkyValue value, InvalidationState state); + + /** + * Notifies that {@code skyKey} is about to get queued for evaluation. + * + * <p>Note that we don't guarantee that it actually got enqueued or will, only that if + * everything "goes well" (e.g. no interrupts happen) it will. + * + * <p>This guarantee is intentionally vague to encourage writing robust implementations. + */ + @ThreadSafety.ThreadSafe + void enqueueing(SkyKey skyKey); + + /** + * Notifies that {@code value} has been evaluated. + * + * <p>{@code state} indicates the new state of the value. + * + * <p>This method is not called if the value builder threw an error when building this value. + */ + @ThreadSafety.ThreadSafe + void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state); +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/EvaluationResult.java b/src/main/java/com/google/devtools/build/skyframe/EvaluationResult.java new file mode 100644 index 0000000..e518dca --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/EvaluationResult.java
@@ -0,0 +1,163 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * The result of a Skyframe {@link Evaluator#eval} call. Will contain all the + * successfully evaluated values, retrievable through {@link #get}. As well, the {@link ErrorInfo} + * for the first value that failed to evaluate (in the non-keep-going case), or any remaining values + * that failed to evaluate (in the keep-going case) will be retrievable. + * + * @param <T> The type of the values that the caller has requested. + */ +public class EvaluationResult<T extends SkyValue> { + + private final boolean hasError; + + private final Map<SkyKey, T> resultMap; + private final Map<SkyKey, ErrorInfo> errorMap; + + /** + * Constructor for the "completed" case. Used only by {@link Builder}. + */ + private EvaluationResult(Map<SkyKey, T> result, Map<SkyKey, ErrorInfo> errorMap, + boolean hasError) { + Preconditions.checkState(errorMap.isEmpty() || hasError, + "result=%s, errorMap=%s", result, errorMap); + this.resultMap = Preconditions.checkNotNull(result); + this.errorMap = Preconditions.checkNotNull(errorMap); + this.hasError = hasError; + } + + /** + * Get a successfully evaluated value. + */ + public T get(SkyKey key) { + Preconditions.checkNotNull(resultMap, key); + return resultMap.get(key); + } + + /** + * @return Whether or not the eval successfully evaluated all requested values. Note that this + * may return true even if all values returned are available in get(). This happens if a top-level + * value depends transitively on some value that recovered from a {@link SkyFunctionException}. + */ + public boolean hasError() { + return hasError; + } + + /** + * @return All successfully evaluated {@link SkyValue}s. + */ + public Collection<T> values() { + return Collections.unmodifiableCollection(resultMap.values()); + } + + /** + * Returns {@link Map} of {@link SkyKey}s to {@link ErrorInfo}. Note that currently some + * of the returned SkyKeys may not be the ones requested by the user. Moreover, the SkyKey + * is not necessarily the cause of the error -- it is just the value that was being evaluated + * when the error was discovered. For the cause of the error, use + * {@link ErrorInfo#getRootCauses()} on each ErrorInfo. + */ + public Map<SkyKey, ErrorInfo> errorMap() { + return ImmutableMap.copyOf(errorMap); + } + + /** + * @param key {@link SkyKey} to get {@link ErrorInfo} for. + */ + public ErrorInfo getError(SkyKey key) { + return Preconditions.checkNotNull(errorMap, key).get(key); + } + + /** + * @return Names of all values that were successfully evaluated. + */ + public <S> Collection<? extends S> keyNames() { + return this.<S>getNames(resultMap.keySet()); + } + + @SuppressWarnings("unchecked") + private <S> Collection<? extends S> getNames(Collection<SkyKey> keys) { + Collection<S> names = Lists.newArrayListWithCapacity(keys.size()); + for (SkyKey key : keys) { + names.add((S) key.argument()); + } + return names; + } + + /** + * Returns some error info. Convenience method equivalent to + * Iterables.getFirst({@link #errorMap()}, null).getValue(). + */ + public ErrorInfo getError() { + return Iterables.getFirst(errorMap.entrySet(), null).getValue(); + } + + @Override + @SuppressWarnings("deprecation") + public String toString() { + return Objects.toStringHelper(this) // MoreObjects is not in Guava + .add("hasError", hasError) + .add("errorMap", errorMap) + .add("resultMap", resultMap) + .toString(); + } + + public static <T extends SkyValue> Builder<T> builder() { + return new Builder<>(); + } + + /** + * Builder for {@link EvaluationResult}. + * + * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations. + */ + public static class Builder<T extends SkyValue> { + private final Map<SkyKey, T> result = new HashMap<>(); + private final Map<SkyKey, ErrorInfo> errors = new HashMap<>(); + private boolean hasError = false; + + @SuppressWarnings("unchecked") + public Builder<T> addResult(SkyKey key, SkyValue value) { + result.put(key, Preconditions.checkNotNull((T) value, key)); + return this; + } + + public Builder<T> addError(SkyKey key, ErrorInfo error) { + errors.put(key, Preconditions.checkNotNull(error, key)); + return this; + } + + public EvaluationResult<T> build() { + return new EvaluationResult<>(result, errors, hasError); + } + + public void setHasError(boolean hasError) { + this.hasError = hasError; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/Evaluator.java b/src/main/java/com/google/devtools/build/skyframe/Evaluator.java new file mode 100644 index 0000000..342eff1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/Evaluator.java
@@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.devtools.build.lib.events.EventHandler; + +/** + * An interface for the evaluator for a particular graph version. + */ +public interface Evaluator { + /** + * Factory to create Evaluator instances. + */ + interface Factory { + /** + * @param graph the graph to operate on + * @param graphVersion the version at which to write entries in the graph. + * @param reporter where to write warning/error/progress messages. + * @param keepGoing whether {@link #eval} should continue if building a {link Value} fails. + * Otherwise, we throw an exception on failure. + */ + Evaluator create(ProcessableGraph graph, long graphVersion, EventHandler reporter, + boolean keepGoing); + } + + /** + * Evaluates a set of values. Returns an {@link EvaluationResult}. All elements of skyKeys must + * be keys for Values of subtype T. + */ + <T extends SkyValue> EvaluationResult<T> eval(Iterable<SkyKey> skyKeys) + throws InterruptedException; +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ImmutableDiff.java b/src/main/java/com/google/devtools/build/skyframe/ImmutableDiff.java new file mode 100644 index 0000000..46ab29e --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/ImmutableDiff.java
@@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +/** + * Immutable implementation of {@link Differencer.Diff}. + */ +public class ImmutableDiff implements Differencer.Diff { + + private final ImmutableList<SkyKey> valuesToInvalidate; + private final ImmutableMap<SkyKey, SkyValue> valuesToInject; + + public ImmutableDiff(Iterable<SkyKey> valuesToInvalidate, Map<SkyKey, SkyValue> valuesToInject) { + this.valuesToInvalidate = ImmutableList.copyOf(valuesToInvalidate); + this.valuesToInject = ImmutableMap.copyOf(valuesToInject); + } + + @Override + public Iterable<SkyKey> changedKeysWithoutNewValues() { + return valuesToInvalidate; + } + + @Override + public Map<SkyKey, SkyValue> changedKeysWithNewValues() { + return valuesToInject; + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/InMemoryGraph.java b/src/main/java/com/google/devtools/build/skyframe/InMemoryGraph.java new file mode 100644 index 0000000..44956da --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/InMemoryGraph.java
@@ -0,0 +1,126 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.MapMaker; +import com.google.common.collect.Maps; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +import javax.annotation.Nullable; + +/** + * An in-memory graph implementation. All operations are thread-safe with ConcurrentMap semantics. + * Also see {@link NodeEntry}. + * + * <p>This class is public only for use in alternative graph implementations. + */ +public class InMemoryGraph implements ProcessableGraph { + + protected final ConcurrentMap<SkyKey, NodeEntry> nodeMap = + new MapMaker().initialCapacity(1024).concurrencyLevel(200).makeMap(); + private final boolean keepEdges; + + InMemoryGraph() { + this(/*keepEdges=*/true); + } + + public InMemoryGraph(boolean keepEdges) { + this.keepEdges = keepEdges; + } + + @Override + public void remove(SkyKey skyKey) { + nodeMap.remove(skyKey); + } + + @Override + public NodeEntry get(SkyKey skyKey) { + return nodeMap.get(skyKey); + } + + @Override + public NodeEntry createIfAbsent(SkyKey key) { + NodeEntry newval = keepEdges ? new NodeEntry() : new EdgelessNodeEntry(); + NodeEntry oldval = nodeMap.putIfAbsent(key, newval); + return oldval == null ? newval : oldval; + } + + /** Only done nodes exist to the outside world. */ + private static final Predicate<NodeEntry> NODE_DONE_PREDICATE = + new Predicate<NodeEntry>() { + @Override + public boolean apply(NodeEntry entry) { + return entry != null && entry.isDone(); + } + }; + + /** + * Returns a value, if it exists. If not, returns null. + */ + @Nullable public SkyValue getValue(SkyKey key) { + NodeEntry entry = get(key); + return NODE_DONE_PREDICATE.apply(entry) ? entry.getValue() : null; + } + + /** + * Returns a read-only live view of the nodes in the graph. All node are included. Dirty values + * include their Node value. Values in error have a null value. + */ + Map<SkyKey, SkyValue> getValues() { + return Collections.unmodifiableMap(Maps.transformValues( + nodeMap, + new Function<NodeEntry, SkyValue>() { + @Override + public SkyValue apply(NodeEntry entry) { + return entry.toValue(); + } + })); + } + + /** + * Returns a read-only live view of the done values in the graph. Dirty, changed, and error values + * are not present in the returned map + */ + Map<SkyKey, SkyValue> getDoneValues() { + return Collections.unmodifiableMap(Maps.filterValues(Maps.transformValues( + nodeMap, + new Function<NodeEntry, SkyValue>() { + @Override + public SkyValue apply(NodeEntry entry) { + return entry.isDone() ? entry.getValue() : null; + } + }), Predicates.notNull())); + } + + // Only for use by MemoizingEvaluator#delete + Map<SkyKey, NodeEntry> getAllValues() { + return Collections.unmodifiableMap(nodeMap); + } + + @VisibleForTesting + protected ConcurrentMap<SkyKey, NodeEntry> getNodeMap() { + return nodeMap; + } + + boolean keepsEdges() { + return keepEdges; + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java b/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java new file mode 100644 index 0000000..827cc7b --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java
@@ -0,0 +1,317 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.skyframe.Differencer.Diff; +import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DeletingInvalidationState; +import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DirtyingInvalidationState; +import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.InvalidationState; +import com.google.devtools.build.skyframe.NodeEntry.DependencyState; + +import java.io.PrintStream; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.annotation.Nullable; + +/** + * An inmemory implementation that uses the eager invalidation strategy. This class is, by itself, + * not thread-safe. Neither is it thread-safe to use this class in parallel with any of the + * returned graphs. However, it is allowed to access the graph from multiple threads as long as + * that does not happen in parallel with an {@link #evaluate} call. + * + * <p>This memoizing evaluator requires a sequential versioning scheme. Evaluations + * must pass in a monotonically increasing {@link IntVersion}. + */ +public final class InMemoryMemoizingEvaluator implements MemoizingEvaluator { + + private final ImmutableMap<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions; + @Nullable private final EvaluationProgressReceiver progressReceiver; + // Not final only for testing. + private InMemoryGraph graph; + private IntVersion lastGraphVersion = null; + + // State related to invalidation and deletion. + private Set<SkyKey> valuesToDelete = new LinkedHashSet<>(); + private Set<SkyKey> valuesToDirty = new LinkedHashSet<>(); + private Map<SkyKey, SkyValue> valuesToInject = new HashMap<>(); + private final DirtyKeyTracker dirtyKeyTracker = new DirtyKeyTrackerImpl(); + private final InvalidationState deleterState = new DeletingInvalidationState(); + private final Differencer differencer; + + // Keep edges in graph. Can be false to save memory, in which case incremental builds are + // not possible. + private final boolean keepEdges; + + // Values that the caller explicitly specified are assumed to be changed -- they will be + // re-evaluated even if none of their children are changed. + private final InvalidationState invalidatorState = new DirtyingInvalidationState(); + + private final EmittedEventState emittedEventState; + + private final AtomicBoolean evaluating = new AtomicBoolean(false); + + public InMemoryMemoizingEvaluator( + Map<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions, Differencer differencer) { + this(skyFunctions, differencer, null); + } + + public InMemoryMemoizingEvaluator( + Map<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions, Differencer differencer, + @Nullable EvaluationProgressReceiver invalidationReceiver) { + this(skyFunctions, differencer, invalidationReceiver, new EmittedEventState(), true); + } + + public InMemoryMemoizingEvaluator( + Map<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions, Differencer differencer, + @Nullable EvaluationProgressReceiver invalidationReceiver, + EmittedEventState emittedEventState, boolean keepEdges) { + this.skyFunctions = ImmutableMap.copyOf(skyFunctions); + this.differencer = Preconditions.checkNotNull(differencer); + this.progressReceiver = invalidationReceiver; + this.graph = new InMemoryGraph(keepEdges); + this.emittedEventState = emittedEventState; + this.keepEdges = keepEdges; + } + + private void invalidate(Iterable<SkyKey> diff) { + Iterables.addAll(valuesToDirty, diff); + } + + @Override + public void delete(final Predicate<SkyKey> deletePredicate) { + valuesToDelete.addAll( + Maps.filterEntries(graph.getAllValues(), new Predicate<Entry<SkyKey, NodeEntry>>() { + @Override + public boolean apply(Entry<SkyKey, NodeEntry> input) { + return input.getValue().isDirty() || deletePredicate.apply(input.getKey()); + } + }).keySet()); + } + + @Override + public void deleteDirty(long versionAgeLimit) { + Preconditions.checkArgument(versionAgeLimit >= 0); + final Version threshold = new IntVersion(lastGraphVersion.getVal() - versionAgeLimit); + valuesToDelete.addAll( + Sets.filter(dirtyKeyTracker.getDirtyKeys(), new Predicate<SkyKey>() { + @Override + public boolean apply(SkyKey skyKey) { + NodeEntry entry = graph.get(skyKey); + Preconditions.checkNotNull(entry, skyKey); + Preconditions.checkState(entry.isDirty(), skyKey); + return entry.getVersion().atMost(threshold); + } + })); + } + + @Override + public <T extends SkyValue> EvaluationResult<T> evaluate(Iterable<SkyKey> roots, Version version, + boolean keepGoing, int numThreads, EventHandler eventHandler) + throws InterruptedException { + // NOTE: Performance critical code. See bug "Null build performance parity". + IntVersion intVersion = (IntVersion) version; + Preconditions.checkState((lastGraphVersion == null && intVersion.getVal() == 0) + || version.equals(lastGraphVersion.next()), + "InMemoryGraph supports only monotonically increasing Integer versions: %s %s", + lastGraphVersion, version); + setAndCheckEvaluateState(true, roots); + try { + // The RecordingDifferencer implementation is not quite working as it should be at this point. + // It clears the internal data structures after getDiff is called and will not return + // diffs for historical versions. This makes the following code sensitive to interrupts. + // Ideally we would simply not update lastGraphVersion if an interrupt occurs. + Diff diff = differencer.getDiff(lastGraphVersion, version); + valuesToInject.putAll(diff.changedKeysWithNewValues()); + invalidate(diff.changedKeysWithoutNewValues()); + pruneInjectedValues(valuesToInject); + invalidate(valuesToInject.keySet()); + + performInvalidation(); + injectValues(intVersion); + + ParallelEvaluator evaluator = new ParallelEvaluator(graph, intVersion, + skyFunctions, eventHandler, emittedEventState, keepGoing, numThreads, progressReceiver, + dirtyKeyTracker); + return evaluator.eval(roots); + } finally { + lastGraphVersion = intVersion; + setAndCheckEvaluateState(false, roots); + } + } + + /** + * Removes entries in {@code valuesToInject} whose values are equal to the present values in the + * graph. + */ + private void pruneInjectedValues(Map<SkyKey, SkyValue> valuesToInject) { + for (Iterator<Entry<SkyKey, SkyValue>> it = valuesToInject.entrySet().iterator(); + it.hasNext();) { + Entry<SkyKey, SkyValue> entry = it.next(); + SkyKey key = entry.getKey(); + SkyValue newValue = entry.getValue(); + NodeEntry prevEntry = graph.get(key); + if (prevEntry != null && prevEntry.isDone()) { + Iterable<SkyKey> directDeps = prevEntry.getDirectDeps(); + Preconditions.checkState(Iterables.isEmpty(directDeps), + "existing entry for %s has deps: %s", key, directDeps); + if (newValue.equals(prevEntry.getValue()) + && !valuesToDirty.contains(key) && !valuesToDelete.contains(key)) { + it.remove(); + } + } + } + } + + /** + * Injects values in {@code valuesToInject} into the graph. + */ + private void injectValues(IntVersion version) { + if (valuesToInject.isEmpty()) { + return; + } + for (Entry<SkyKey, SkyValue> entry : valuesToInject.entrySet()) { + SkyKey key = entry.getKey(); + SkyValue value = entry.getValue(); + Preconditions.checkState(value != null, key); + NodeEntry prevEntry = graph.createIfAbsent(key); + if (prevEntry.isDirty()) { + // There was an existing entry for this key in the graph. + // Get the node in the state where it is able to accept a value. + Preconditions.checkState(prevEntry.getTemporaryDirectDeps().isEmpty(), key); + + DependencyState newState = prevEntry.addReverseDepAndCheckIfDone(null); + Preconditions.checkState(newState == DependencyState.NEEDS_SCHEDULING, key); + + // Check that the previous node has no dependencies. Overwriting a value with deps with an + // injected value (which is by definition deps-free) needs a little additional bookkeeping + // (removing reverse deps from the dependencies), but more importantly it's something that + // we want to avoid, because it indicates confusion of input values and derived values. + Preconditions.checkState(prevEntry.noDepsLastBuild(), + "existing entry for %s has deps: %s", key, prevEntry); + } + prevEntry.setValue(value, version); + // The evaluate method previously invalidated all keys in valuesToInject that survived the + // pruneInjectedValues call. Now that this key's injected value is set, it is no longer dirty. + dirtyKeyTracker.notDirty(key); + } + // Start with a new map to avoid bloat since clear() does not downsize the map. + valuesToInject = new HashMap<>(); + } + + private void performInvalidation() throws InterruptedException { + EagerInvalidator.delete(graph, valuesToDelete, progressReceiver, deleterState, keepEdges, + dirtyKeyTracker); + // Note that clearing the valuesToDelete would not do an internal resizing. Therefore, if any + // build has a large set of dirty values, subsequent operations (even clearing) will be slower. + // Instead, just start afresh with a new LinkedHashSet. + valuesToDelete = new LinkedHashSet<>(); + + EagerInvalidator.invalidate(graph, valuesToDirty, progressReceiver, invalidatorState, + dirtyKeyTracker); + // Ditto. + valuesToDirty = new LinkedHashSet<>(); + } + + private void setAndCheckEvaluateState(boolean newValue, Object requestInfo) { + Preconditions.checkState(evaluating.getAndSet(newValue) != newValue, + "Re-entrant evaluation for request: %s", requestInfo); + } + + @Override + public Map<SkyKey, SkyValue> getValues() { + return graph.getValues(); + } + + @Override + public Map<SkyKey, SkyValue> getDoneValues() { + return graph.getDoneValues(); + } + + @Override + @Nullable public SkyValue getExistingValueForTesting(SkyKey key) { + return graph.getValue(key); + } + + @Override + @Nullable public ErrorInfo getExistingErrorForTesting(SkyKey key) { + NodeEntry entry = graph.get(key); + return (entry == null || !entry.isDone()) ? null : entry.getErrorInfo(); + } + + public void setGraphForTesting(InMemoryGraph graph) { + this.graph = graph; + } + + @Override + public void dump(boolean summarize, PrintStream out) { + if (summarize) { + long nodes = 0; + long edges = 0; + for (NodeEntry entry : graph.getAllValues().values()) { + nodes++; + if (entry.isDone()) { + edges += Iterables.size(entry.getDirectDeps()); + } + } + out.println("Node count: " + nodes); + out.println("Edge count: " + edges); + } else { + Function<SkyKey, String> keyFormatter = + new Function<SkyKey, String>() { + @Override + public String apply(SkyKey key) { + return String.format("%s:%s", + key.functionName(), key.argument().toString().replace('\n', '_')); + } + }; + + for (Entry<SkyKey, NodeEntry> mapPair : graph.getAllValues().entrySet()) { + SkyKey key = mapPair.getKey(); + NodeEntry entry = mapPair.getValue(); + if (entry.isDone()) { + out.print(keyFormatter.apply(key)); + out.print("|"); + out.println(Joiner.on('|').join( + Iterables.transform(entry.getDirectDeps(), keyFormatter))); + } + } + } + } + + public static final EvaluatorSupplier SUPPLIER = new EvaluatorSupplier() { + @Override + public MemoizingEvaluator create( + Map<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions, Differencer differencer, + @Nullable EvaluationProgressReceiver invalidationReceiver, + EmittedEventState emittedEventState, boolean keepEdges) { + return new InMemoryMemoizingEvaluator(skyFunctions, differencer, invalidationReceiver, + emittedEventState, keepEdges); + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/Injectable.java b/src/main/java/com/google/devtools/build/skyframe/Injectable.java new file mode 100644 index 0000000..5325df3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/Injectable.java
@@ -0,0 +1,23 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import java.util.Map; + +/** + * An object that accepts Skyframe key / value mapping. + */ +public interface Injectable { + void inject(Map<SkyKey, ? extends SkyValue> values); +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/IntVersion.java b/src/main/java/com/google/devtools/build/skyframe/IntVersion.java new file mode 100644 index 0000000..3d2a31d --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/IntVersion.java
@@ -0,0 +1,61 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +/** + * Versioning scheme based on integers. + */ +public final class IntVersion implements Version { + + private final long val; + + public IntVersion(long val) { + this.val = val; + } + + public long getVal() { + return val; + } + + public IntVersion next() { + return new IntVersion(val + 1); + } + + @Override + public boolean atMost(Version other) { + if (!(other instanceof IntVersion)) { + return false; + } + return val <= ((IntVersion) other).val; + } + + @Override + public int hashCode() { + return Long.valueOf(val).hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof IntVersion) { + IntVersion other = (IntVersion) obj; + return other.val == val; + } + return false; + } + + @Override + public String toString() { + return "IntVersion: " + val; + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/InvalidatingNodeVisitor.java b/src/main/java/com/google/devtools/build/skyframe/InvalidatingNodeVisitor.java new file mode 100644 index 0000000..7abf6c6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/InvalidatingNodeVisitor.java
@@ -0,0 +1,350 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.util.Pair; + +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import javax.annotation.Nullable; + +/** + * A visitor that is useful for invalidating transitive dependencies of Skyframe nodes. + * + * <p>Interruptibility: It is safe to interrupt the invalidation process at any time. Consider a + * graph and a set of modified nodes. Then the reverse transitive closure of the modified nodes is + * the set of dirty nodes. We provide interruptibility by making sure that the following invariant + * holds at any time: + * + * <p>If a node is dirty, but not removed (or marked as dirty) yet, then either it or any of its + * transitive dependencies must be in the {@link #pendingVisitations} set. Furthermore, reverse dep + * pointers must always point to existing nodes. + * + * <p>Thread-safety: This class should only be instantiated and called on a single thread, but + * internally it spawns many worker threads to process the graph. The thread-safety of the workers + * on the graph can be delicate, and is documented below. Moreover, no other modifications to the + * graph can take place while invalidation occurs. + * + * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations. + */ +public abstract class InvalidatingNodeVisitor extends AbstractQueueVisitor { + + // Default thread count is equal to the number of cores to exploit + // that level of hardware parallelism, since invalidation should be CPU-bound. + // We may consider increasing this in the future. + private static final int DEFAULT_THREAD_COUNT = Runtime.getRuntime().availableProcessors(); + + private static final boolean MUST_EXIST = true; + + protected final DirtiableGraph graph; + @Nullable protected final EvaluationProgressReceiver invalidationReceiver; + protected final DirtyKeyTracker dirtyKeyTracker; + // Aliased to InvalidationState.pendingVisitations. + protected final Set<Pair<SkyKey, InvalidationType>> pendingVisitations; + + protected InvalidatingNodeVisitor( + DirtiableGraph graph, @Nullable EvaluationProgressReceiver invalidationReceiver, + InvalidationState state, DirtyKeyTracker dirtyKeyTracker) { + super(/*concurrent*/true, + /*corePoolSize*/DEFAULT_THREAD_COUNT, + /*maxPoolSize*/DEFAULT_THREAD_COUNT, + 1, TimeUnit.SECONDS, + /*failFastOnException*/true, + /*failFastOnInterrupt*/true, + "skyframe-invalidator"); + this.graph = Preconditions.checkNotNull(graph); + this.invalidationReceiver = invalidationReceiver; + this.dirtyKeyTracker = Preconditions.checkNotNull(dirtyKeyTracker); + this.pendingVisitations = state.pendingValues; + } + + /** + * Initiates visitation and waits for completion. + */ + void run() throws InterruptedException { + // Make a copy to avoid concurrent modification confusing us as to which nodes were passed by + // the caller, and which are added by other threads during the run. Since no tasks have been + // started yet (the queueDirtying calls start them), this is thread-safe. + for (Pair<SkyKey, InvalidationType> visitData : ImmutableList.copyOf(pendingVisitations)) { + // The caller may have specified non-existent SkyKeys, or there may be stale SkyKeys in + // pendingVisitations that have already been deleted. In both these cases, the nodes will not + // exist in the graph, so we must be tolerant of that case. + visit(visitData.first, visitData.second, !MUST_EXIST); + } + work(/*failFastOnInterrupt=*/true); + Preconditions.checkState(pendingVisitations.isEmpty(), + "All dirty nodes should have been processed: %s", pendingVisitations); + } + + protected void informInvalidationReceiver(SkyValue value, + EvaluationProgressReceiver.InvalidationState state) { + if (invalidationReceiver != null && value != null) { + invalidationReceiver.invalidated(value, state); + } + } + + /** + * Enqueues a node for invalidation. + */ + @ThreadSafe + abstract void visit(SkyKey key, InvalidationType second, boolean mustExist); + + @VisibleForTesting + enum InvalidationType { + /** + * The node is dirty and must be recomputed. + */ + CHANGED, + /** + * The node is dirty, but may be marked clean later during change pruning. + */ + DIRTIED, + /** + * The node is deleted. + */ + DELETED; + } + + /** + * Invalidation state object that keeps track of which nodes need to be invalidated, but have not + * been dirtied/deleted yet. This supports interrupts - by only deleting a node from this set + * when all its parents have been invalidated, we ensure that no information is lost when an + * interrupt comes in. + */ + static class InvalidationState { + private final Set<Pair<SkyKey, InvalidationType>> pendingValues = Sets.newConcurrentHashSet(); + private final InvalidationType defaultUpdateType; + + private InvalidationState(InvalidationType defaultUpdateType) { + this.defaultUpdateType = Preconditions.checkNotNull(defaultUpdateType); + } + + void update(Iterable<SkyKey> diff) { + Iterables.addAll(pendingValues, Iterables.transform(diff, + new Function<SkyKey, Pair<SkyKey, InvalidationType>>() { + @Override + public Pair<SkyKey, InvalidationType> apply(SkyKey skyKey) { + return Pair.of(skyKey, defaultUpdateType); + } + })); + } + + @VisibleForTesting + boolean isEmpty() { + return pendingValues.isEmpty(); + } + + @VisibleForTesting + Set<Pair<SkyKey, InvalidationType>> getInvalidationsForTesting() { + return ImmutableSet.copyOf(pendingValues); + } + } + + public static class DirtyingInvalidationState extends InvalidationState { + public DirtyingInvalidationState() { + super(InvalidationType.CHANGED); + } + } + + static class DeletingInvalidationState extends InvalidationState { + public DeletingInvalidationState() { + super(InvalidationType.DELETED); + } + } + + /** + * A node-deleting implementation. + */ + static class DeletingNodeVisitor extends InvalidatingNodeVisitor { + + private final Set<SkyKey> visitedValues = Sets.newConcurrentHashSet(); + private final boolean traverseGraph; + + protected DeletingNodeVisitor(DirtiableGraph graph, + EvaluationProgressReceiver invalidationReceiver, InvalidationState state, + boolean traverseGraph, DirtyKeyTracker dirtyKeyTracker) { + super(graph, invalidationReceiver, state, dirtyKeyTracker); + this.traverseGraph = traverseGraph; + } + + @Override + public void visit(final SkyKey key, InvalidationType invalidationType, boolean mustExist) { + Preconditions.checkState(invalidationType == InvalidationType.DELETED, key); + if (!visitedValues.add(key)) { + return; + } + final Pair<SkyKey, InvalidationType> invalidationPair = Pair.of(key, invalidationType); + pendingVisitations.add(invalidationPair); + enqueue(new Runnable() { + @Override + public void run() { + NodeEntry entry = graph.get(key); + if (entry == null) { + pendingVisitations.remove(invalidationPair); + return; + } + + if (traverseGraph) { + // Propagate deletion upwards. + for (SkyKey reverseDep : entry.getReverseDeps()) { + visit(reverseDep, InvalidationType.DELETED, !MUST_EXIST); + } + } + + if (entry.isDone()) { + // Only process this node's value and children if it is done, since dirty nodes have + // no awareness of either. + + // Unregister this node from direct deps, since reverse dep edges cannot point to + // non-existent nodes. + if (traverseGraph) { + for (SkyKey directDep : entry.getDirectDeps()) { + NodeEntry dep = graph.get(directDep); + if (dep != null) { + dep.removeReverseDep(key); + } + } + } + // Allow custom Value-specific logic to update dirtiness status. + informInvalidationReceiver(entry.getValue(), + EvaluationProgressReceiver.InvalidationState.DELETED); + } + if (traverseGraph) { + // Force reverseDeps consolidation (validates that attempts to remove reverse deps were + // really successful. + entry.getReverseDeps(); + } + // Actually remove the node. + graph.remove(key); + dirtyKeyTracker.notDirty(key); + + // Remove the node from the set as the last operation. + pendingVisitations.remove(invalidationPair); + } + }); + } + } + + /** + * A node-dirtying implementation. + */ + static class DirtyingNodeVisitor extends InvalidatingNodeVisitor { + + private final Set<Pair<SkyKey, InvalidationType>> visited = Sets.newConcurrentHashSet(); + + protected DirtyingNodeVisitor(DirtiableGraph graph, + EvaluationProgressReceiver invalidationReceiver, InvalidationState state, + DirtyKeyTracker dirtyKeyTracker) { + super(graph, invalidationReceiver, state, dirtyKeyTracker); + } + + /** + * Queues a task to dirty the node named by {@code key}. May be called from multiple threads. + * It is possible that the same node is enqueued many times. However, we require that a node + * is only actually marked dirty/changed once, with two exceptions: + * + * (1) If a node is marked dirty, it can subsequently be marked changed. This can occur if, for + * instance, FileValue workspace/foo/foo.cc is marked dirty because FileValue workspace/foo is + * marked changed (and every FileValue depends on its parent). Then FileValue + * workspace/foo/foo.cc is itself changed (this can even happen on the same build). + * + * (2) If a node is going to be marked both dirty and changed, as, for example, in the previous + * case if both workspace/foo/foo.cc and workspace/foo have been changed in the same build, the + * thread marking workspace/foo/foo.cc dirty may race with the one marking it changed, and so + * try to mark it dirty after it has already been marked changed. In that case, the + * {@link NodeEntry} ignores the second marking. + * + * The invariant that we do not process a (SkyKey, InvalidationType) pair twice is enforced by + * the {@link #visited} set. + * + * The "invariant" is also enforced across builds by checking to see if the entry is already + * marked changed, or if it is already marked dirty and we are just going to mark it dirty + * again. + * + * If either of the above tests shows that we have already started a task to mark this entry + * dirty/changed, or that it is already marked dirty/changed, we do not continue this task. + */ + @Override + @ThreadSafe + public void visit(final SkyKey key, final InvalidationType invalidationType, + final boolean mustExist) { + Preconditions.checkState(invalidationType != InvalidationType.DELETED, key); + final boolean isChanged = (invalidationType == InvalidationType.CHANGED); + final Pair<SkyKey, InvalidationType> invalidationPair = Pair.of(key, invalidationType); + if (!visited.add(invalidationPair)) { + return; + } + pendingVisitations.add(invalidationPair); + enqueue(new Runnable() { + @Override + public void run() { + NodeEntry entry = graph.get(key); + + if (entry == null) { + Preconditions.checkState(!mustExist, + "%s does not exist in the graph but was enqueued for dirtying by another node", + key); + pendingVisitations.remove(invalidationPair); + return; + } + + if (entry.isChanged() || (!isChanged && entry.isDirty())) { + // If this node is already marked changed, or we are only marking this node dirty, and + // it already is, move along. + pendingVisitations.remove(invalidationPair); + return; + } + + // This entry remains in the graph in this dirty state until it is re-evaluated. + Pair<? extends Iterable<SkyKey>, ? extends SkyValue> depsAndValue = + entry.markDirty(isChanged); + // It is not safe to interrupt the logic from this point until the end of the method. + // Any exception thrown should be unrecoverable. + if (depsAndValue == null) { + // Another thread has already dirtied this node. Don't do anything in this thread. + pendingVisitations.remove(invalidationPair); + return; + } + // Propagate dirtiness upwards and mark this node dirty/changed. Reverse deps should only + // be marked dirty (because only a dependency of theirs has changed). + for (SkyKey reverseDep : entry.getReverseDeps()) { + visit(reverseDep, InvalidationType.DIRTIED, MUST_EXIST); + } + + // Remove this node as a reverse dep from its children, since we have reset it and it no + // longer lists its children as direct deps. + for (SkyKey dep : depsAndValue.first) { + graph.get(dep).removeReverseDep(key); + } + + SkyValue value = ValueWithMetadata.justValue(depsAndValue.second); + informInvalidationReceiver(value, EvaluationProgressReceiver.InvalidationState.DIRTY); + dirtyKeyTracker.dirty(key); + // Remove the node from the set as the last operation. + pendingVisitations.remove(invalidationPair); + } + }); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java b/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java new file mode 100644 index 0000000..2c7f14e --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java
@@ -0,0 +1,143 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Predicate; +import com.google.devtools.build.lib.collect.nestedset.NestedSetVisitor; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile; +import com.google.devtools.build.lib.events.EventHandler; + +import java.io.PrintStream; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * A graph, defined by a set of functions that can construct values from value keys. + * + * <p>The value constructor functions ({@link SkyFunction}s) can declare dependencies on + * prerequisite {@link SkyValue}s. The {@link MemoizingEvaluator} implementation makes sure that + * those are created beforehand. + * + * <p>The graph caches previously computed value values. Arbitrary values can be invalidated between + * calls to {@link #evaluate}; they will be recreated the next time they are requested. + */ +public interface MemoizingEvaluator { + + /** + * Computes the transitive closure of a given set of values at the given {@link Version}. See + * {@link EagerInvalidator#invalidate}. + * + * <p>The returned EvaluationResult is guaranteed to contain a result for at least one root if + * keepGoing is false. It will contain a result for every root if keepGoing is true, <i>unless</i> + * the evaluation failed with a "catastrophic" error. In that case, some or all results may be + * missing. + */ + <T extends SkyValue> EvaluationResult<T> evaluate( + Iterable<SkyKey> roots, + Version version, + boolean keepGoing, + int numThreads, + EventHandler reporter) + throws InterruptedException; + + /** + * Ensures that after the next completed {@link #evaluate} call the current values of any value + * matching this predicate (and all values that transitively depend on them) will be removed from + * the value cache. All values that were already marked dirty in the graph will also be deleted, + * regardless of whether or not they match the predicate. + * + * <p>If a later call to {@link #evaluate} requests some of the deleted values, those values will + * be recomputed and the new values stored in the cache again. + * + * <p>To delete all dirty values, you can specify a predicate that's always false. + */ + void delete(Predicate<SkyKey> pred); + + /** + * Marks dirty values for deletion if they have been dirty for at least as many graph versions + * as the specified limit. + * + * <p>This ensures that after the next completed {@link #evaluate} call, all such values, along + * with all values that transitively depend on them, will be removed from the value cache. Values + * that were marked dirty after the threshold version will not be affected by this call. + * + * <p>If a later call to {@link #evaluate} requests some of the deleted values, those values will + * be recomputed and the new values stored in the cache again. + * + * <p>To delete all dirty values, you can specify 0 for the limit. + */ + void deleteDirty(long versionAgeLimit); + + /** + * Returns the values in the graph. + * + * <p>The returned map may be a live view of the graph. + */ + Map<SkyKey, SkyValue> getValues(); + + + /** + * Returns the done (without error) values in the graph. + * + * <p>The returned map may be a live view of the graph. + */ + Map<SkyKey, SkyValue> getDoneValues(); + + /** + * Returns a value if and only if an earlier call to {@link #evaluate} created it; null otherwise. + * + * <p>This method should only be used by tests that need to verify the presence of a value in the + * graph after an {@link #evaluate} call. + */ + @VisibleForTesting + @Nullable + SkyValue getExistingValueForTesting(SkyKey key); + + /** + * Returns an error if and only if an earlier call to {@link #evaluate} created it; null + * otherwise. + * + * <p>This method should only be used by tests that need to verify the presence of an error in the + * graph after an {@link #evaluate} call. + */ + @VisibleForTesting + @Nullable + ErrorInfo getExistingErrorForTesting(SkyKey key); + + /** + * Write the graph to the output stream. Not necessarily thread-safe. Use only for debugging + * purposes. + */ + @ThreadHostile + void dump(boolean summarize, PrintStream out); + + /** + * A supplier for creating instances of a particular evaluator implementation. + */ + public static interface EvaluatorSupplier { + MemoizingEvaluator create( + Map<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions, Differencer differencer, + @Nullable EvaluationProgressReceiver invalidationReceiver, + EmittedEventState emittedEventState, boolean keepEdges); + } + + /** + * Keeps track of already-emitted events. Users of the graph should instantiate an + * {@code EmittedEventState} first and pass it to the graph during creation. This allows them to + * determine whether or not to replay events. + */ + public static class EmittedEventState extends NestedSetVisitor.VisitedState<TaggedEvents> {} +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/MinimalVersion.java b/src/main/java/com/google/devtools/build/skyframe/MinimalVersion.java new file mode 100644 index 0000000..6f75c15 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/MinimalVersion.java
@@ -0,0 +1,31 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +/** + * A Version "less than" all other versions, other than itself. + * + * <p>Only use in custom evaluator implementations. + */ +public class MinimalVersion implements Version { + public static final MinimalVersion INSTANCE = new MinimalVersion(); + + private MinimalVersion() { + } + + @Override + public boolean atMost(Version other) { + return true; + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/NodeEntry.java b/src/main/java/com/google/devtools/build/skyframe/NodeEntry.java new file mode 100644 index 0000000..243189d --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/NodeEntry.java
@@ -0,0 +1,581 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.util.GroupedList; +import com.google.devtools.build.lib.util.GroupedList.GroupedListHelper; +import com.google.devtools.build.lib.util.Pair; + +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A node in the graph. All operations on this class are thread-safe. Care was taken to provide + * certain compound operations to avoid certain check-then-act races. That means this class is + * somewhat closely tied to the exact Evaluator implementation. + * + * <p>Consider the example with two threads working on two nodes, where one depends on the other, + * say b depends on a. If a completes first, it's done. If it completes second, it needs to signal + * b, and potentially re-schedule it. If b completes first, it must exit, because it will be + * signaled (and re-scheduled) by a. If it completes second, it must signal (and re-schedule) + * itself. However, if the Evaluator supported re-entrancy for a node, then this wouldn't have to + * be so strict, because duplicate scheduling would be less problematic. + * + * <p>The transient state of a {@code NodeEntry} is kept in a {@link BuildingState} object. Many of + * the methods of {@code NodeEntry} are just wrappers around the corresponding + * {@link BuildingState} methods. + * + * <p>This class is non-final only for testing purposes. + * <p>This class is public only for the benefit of alternative graph implementations outside of the + * package. + */ +public class NodeEntry { + /** + * Return code for {@link #addReverseDepAndCheckIfDone(SkyKey)}. + */ + enum DependencyState { + /** The node is done. */ + DONE, + + /** + * The node was just created and needs to be scheduled for its first evaluation pass. The + * evaluator is responsible for signaling the reverse dependency node. + */ + NEEDS_SCHEDULING, + + /** + * The node was already created, but isn't done yet. The evaluator is responsible for + * signaling the reverse dependency node. + */ + ADDED_DEP; + } + + /** Actual data stored in this entry when it is done. */ + private SkyValue value = null; + + /** + * The last version of the graph at which this node entry was changed. In {@link #setValue} it + * may be determined that the data being written to the graph at a given version is the same as + * the already-stored data. In that case, the version will remain the same. The version can be + * thought of as the latest timestamp at which this entry was changed. + */ + private Version version = MinimalVersion.INSTANCE; + + /** + * This object represents a {@link GroupedList}<SkyKey> in a memory-efficient way. It stores the + * direct dependencies of this node, in groups if the {@code SkyFunction} requested them that way. + */ + private Object directDeps = null; + + /** + * This list stores the reverse dependencies of this node that have been declared so far. + * + * <p>In case of a single object we store the object unwrapped, without the list, for + * memory-efficiency. + */ + @VisibleForTesting + protected Object reverseDeps = ImmutableList.of(); + + /** + * We take advantage of memory alignment to avoid doing a nasty {@code instanceof} for knowing + * if {@code reverseDeps} is a single object or a list. + */ + protected boolean reverseDepIsSingleObject = false; + + /** + * During the invalidation we keep the reverse deps to be removed in this list instead of directly + * removing them from {@code reverseDeps}. That is because removals from reverseDeps are O(N). + * Originally reverseDeps was a HashSet, but because of memory consumption we switched to a list. + * + * <p>This requires that any usage of reverseDeps (contains, add, the list of reverse deps) call + * {@code consolidateReverseDepsRemovals} first. While this operation is not free, it can be done + * more effectively than trying to remove each dirty reverse dependency individually (O(N) each + * time). + */ + private List<SkyKey> reverseDepsToRemove = null; + + private static final ReverseDepsUtil<NodeEntry> REVERSE_DEPS_UTIL = + new ReverseDepsUtil<NodeEntry>() { + @Override + void setReverseDepsObject(NodeEntry container, Object object) { + container.reverseDeps = object; + } + + @Override + void setSingleReverseDep(NodeEntry container, boolean singleObject) { + container.reverseDepIsSingleObject = singleObject; + } + + @Override + void setReverseDepsToRemove(NodeEntry container, List<SkyKey> object) { + container.reverseDepsToRemove = object; + } + + @Override + Object getReverseDepsObject(NodeEntry container) { + return container.reverseDeps; + } + + @Override + boolean isSingleReverseDep(NodeEntry container) { + return container.reverseDepIsSingleObject; + } + + @Override + List<SkyKey> getReverseDepsToRemove(NodeEntry container) { + return container.reverseDepsToRemove; + } + }; + + /** + * The transient state of this entry, after it has been created but before it is done. It allows + * us to keep the current state of the entry across invalidation and successive evaluations. + */ + @VisibleForTesting + protected BuildingState buildingState = new BuildingState(); + + /** + * Construct a NodeEntry. Use ONLY in Skyframe evaluation and graph implementations. + */ + public NodeEntry() { + } + + protected boolean keepEdges() { + return true; + } + + /** Returns whether the entry has been built and is finished evaluating. */ + synchronized boolean isDone() { + return buildingState == null; + } + + /** + * Returns the value stored in this entry. This method may only be called after the evaluation of + * this node is complete, i.e., after {@link #setValue} has been called. + */ + synchronized SkyValue getValue() { + Preconditions.checkState(isDone(), "no value until done. ValueEntry: %s", this); + return ValueWithMetadata.justValue(value); + } + + /** + * Returns the {@link SkyValue} for this entry and the metadata associated with it (Like events + * and errors). This method may only be called after the evaluation of this node is complete, + * i.e., after {@link #setValue} has been called. + */ + synchronized ValueWithMetadata getValueWithMetadata() { + Preconditions.checkState(isDone(), "no value until done: %s", this); + return ValueWithMetadata.wrapWithMetadata(value); + } + + /** + * Returns the value, even if dirty or changed. Returns null otherwise. + */ + public synchronized SkyValue toValue() { + if (isDone()) { + return getErrorInfo() == null ? getValue() : null; + } else if (isChanged() || isDirty()) { + return (buildingState.getLastBuildValue() == null) + ? null + : ValueWithMetadata.justValue(buildingState.getLastBuildValue()); + } + throw new AssertionError("Value in bad state: " + this); + } + + /** + * Returns an immutable iterable of the direct deps of this node. This method may only be called + * after the evaluation of this node is complete, i.e., after {@link #setValue} has been called. + * + * <p>This method is not very efficient, but is only be called in limited circumstances -- + * when the node is about to be deleted, or when the node is expected to have no direct deps (in + * which case the overhead is not so bad). It should not be called repeatedly for the same node, + * since each call takes time proportional to the number of direct deps of the node. + */ + synchronized Iterable<SkyKey> getDirectDeps() { + assertKeepEdges(); + Preconditions.checkState(isDone(), "no deps until done. ValueEntry: %s", this); + return GroupedList.<SkyKey>create(directDeps).toSet(); + } + + /** + * Returns the error, if any, associated to this node. This method may only be called after + * the evaluation of this node is complete, i.e., after {@link #setValue} has been called. + */ + @Nullable + synchronized ErrorInfo getErrorInfo() { + Preconditions.checkState(isDone(), "no errors until done. ValueEntry: %s", this); + return ValueWithMetadata.getMaybeErrorInfo(value); + } + + private synchronized Set<SkyKey> setStateFinishedAndReturnReverseDeps() { + // Get reverse deps that need to be signaled. + ImmutableSet<SkyKey> reverseDepsToSignal = buildingState.getReverseDepsToSignal(); + REVERSE_DEPS_UTIL.consolidateReverseDepsRemovals(this); + REVERSE_DEPS_UTIL.addReverseDeps(this, reverseDepsToSignal); + this.directDeps = buildingState.getFinishedDirectDeps().compress(); + + // Set state of entry to done. + buildingState = null; + + if (!keepEdges()) { + this.directDeps = null; + this.reverseDeps = null; + } + return reverseDepsToSignal; + } + + /** + * Returns the set of reverse deps that have been declared so far this build. Only for use in + * debugging and when bubbling errors up in the --nokeep_going case, where we need to know what + * parents this entry has. + */ + synchronized Set<SkyKey> getInProgressReverseDeps() { + Preconditions.checkState(!isDone(), this); + return buildingState.getReverseDepsToSignal(); + } + + /** + * Transitions the node from the EVALUATING to the DONE state and simultaneously sets it to the + * given value and error state. It then returns the set of reverse dependencies that need to be + * signaled. + * + * <p>This is an atomic operation to avoid a race where two threads work on two nodes, where one + * node depends on another (b depends on a). When a finishes, it signals <b>exactly</b> the set + * of reverse dependencies that are registered at the time of the {@code setValue} call. If b + * comes in before a, it is signaled (and re-scheduled) by a, otherwise it needs to do that + * itself. + * + * <p>{@code version} indicates the graph version at which this node is being written. If the + * entry determines that the new value is equal to the previous value, the entry will keep its + * current version. Callers can query that version to see if the node considers its value to have + * changed. + */ + public synchronized Set<SkyKey> setValue(SkyValue value, Version version) { + Preconditions.checkState(isReady(), "%s %s", this, value); + // This check may need to be removed when we move to a non-linear versioning sequence. + Preconditions.checkState(this.version.atMost(version), + "%s %s %s", this, version, value); + + if (isDirty() && buildingState.unchangedFromLastBuild(value)) { + // If the value is the same as before, just use the old value. Note that we don't use the new + // value, because preserving == equality is even better than .equals() equality. + this.value = buildingState.getLastBuildValue(); + } else { + // If this is a new value, or it has changed since the last build, set the version to the + // current graph version. + this.version = version; + this.value = value; + } + + return setStateFinishedAndReturnReverseDeps(); + } + + /** + * Queries if the node is done and adds the given key as a reverse dependency. The return code + * indicates whether a) the node is done, b) the reverse dependency is the first one, so the + * node needs to be scheduled, or c) the reverse dependency was added, and the node does not + * need to be scheduled. + * + * <p>This method <b>must</b> be called before any processing of the entry. This encourages + * callers to check that the entry is ready to be processed. + * + * <p>Adding the dependency and checking if the node needs to be scheduled is an atomic operation + * to avoid a race where two threads work on two nodes, where one depends on the other (b depends + * on a). In that case, we need to ensure that b is re-scheduled exactly once when a is done. + * However, a may complete first, in which case b has to re-schedule itself. Also see {@link + * #setValue}. + * + * <p>If the parameter is {@code null}, then no reverse dependency is added, but we still check + * if the node needs to be scheduled. + */ + synchronized DependencyState addReverseDepAndCheckIfDone(SkyKey reverseDep) { + if (reverseDep != null) { + if (keepEdges()) { + REVERSE_DEPS_UTIL.consolidateReverseDepsRemovals(this); + REVERSE_DEPS_UTIL.maybeCheckReverseDepNotPresent(this, reverseDep); + } + if (isDone()) { + if (keepEdges()) { + REVERSE_DEPS_UTIL.addReverseDeps(this, ImmutableList.of(reverseDep)); + } + } else { + // Parent should never register itself twice in the same build. + buildingState.addReverseDepToSignal(reverseDep); + } + } + if (isDone()) { + return DependencyState.DONE; + } + return buildingState.startEvaluating() ? DependencyState.NEEDS_SCHEDULING + : DependencyState.ADDED_DEP; + } + + /** + * Removes a reverse dependency. + */ + synchronized void removeReverseDep(SkyKey reverseDep) { + if (!keepEdges()) { + return; + } + REVERSE_DEPS_UTIL.removeReverseDep(this, reverseDep); + if (!isDone()) { + // This is currently unnecessary -- the only time we remove a reverse dep that was added this + // build is during the clean following a build failure. In that case, this node that is not + // done will be deleted soon, so clearing the reverse dep is not required. + buildingState.removeReverseDepToSignal(reverseDep); + } + } + + /** + * Returns a copy of the set of reverse dependencies. Note that this introduces a potential + * check-then-act race; {@link #removeReverseDep} may fail for a key that is returned here. + */ + synchronized Iterable<SkyKey> getReverseDeps() { + assertKeepEdges(); + Preconditions.checkState(isDone() || buildingState.getReverseDepsToSignal().isEmpty(), + "Reverse deps should only be queried before the build has begun " + + "or after the node is done %s", this); + return REVERSE_DEPS_UTIL.getReverseDeps(this); + } + + /** + * Tell this node that one of its dependencies is now done. Callers must check the return value, + * and if true, they must re-schedule this node for evaluation. Equivalent to + * {@code #signalDep(Long.MAX_VALUE)}. Since this entry's version is less than + * {@link Long#MAX_VALUE}, informing this entry that a child of it has version + * {@link Long#MAX_VALUE} will force it to re-evaluate. + */ + synchronized boolean signalDep() { + return signalDep(/*childVersion=*/new IntVersion(Long.MAX_VALUE)); + } + + /** + * Tell this entry that one of its dependencies is now done. Callers must check the return value, + * and if true, they must re-schedule this node for evaluation. + * + * @param childVersion If this entry {@link #isDirty()} and {@code childVersion} is not at most + * {@link #getVersion()}, then this entry records that one of its children has changed since it + * was last evaluated (namely, it was last evaluated at version {@link #getVersion()} and the + * child was last evaluated at {@code childVersion}. Thus, the next call to + * {@link #getDirtyState()} will return {@link BuildingState.DirtyState#REBUILDING}. + */ + synchronized boolean signalDep(Version childVersion) { + Preconditions.checkState(!isDone(), "Value must not be done in signalDep %s", this); + return buildingState.signalDep(/*childChanged=*/!childVersion.atMost(getVersion())); + } + + /** + * Returns true if the entry is marked dirty, meaning that at least one of its transitive + * dependencies is marked changed. + */ + public synchronized boolean isDirty() { + return !isDone() && buildingState.isDirty(); + } + + /** + * Returns true if the entry is marked changed, meaning that it must be re-evaluated even if its + * dependencies' values have not changed. + */ + synchronized boolean isChanged() { + return !isDone() && buildingState.isChanged(); + } + + /** Checks that a caller is not trying to access not-stored graph edges. */ + private void assertKeepEdges() { + Preconditions.checkState(keepEdges(), "Graph edges not stored. %s", this); + } + + /** + * Marks this node dirty, or changed if {@code isChanged} is true. The node is put in the + * just-created state. It will be re-evaluated if necessary during the evaluation phase, + * but if it has not changed, it will not force a re-evaluation of its parents. + * + * @return The direct deps and value of this entry, or null if the entry has already been marked + * dirty. In the latter case, the caller should abort its handling of this node, since another + * thread is already dealing with it. + */ + @Nullable + synchronized Pair<? extends Iterable<SkyKey>, ? extends SkyValue> markDirty(boolean isChanged) { + assertKeepEdges(); + if (isDone()) { + GroupedList<SkyKey> lastDirectDeps = GroupedList.create(directDeps); + buildingState = BuildingState.newDirtyState(isChanged, lastDirectDeps, value); + Pair<? extends Iterable<SkyKey>, ? extends SkyValue> result = + Pair.of(lastDirectDeps.toSet(), value); + value = null; + directDeps = null; + return result; + } + // The caller may be simultaneously trying to mark this node dirty and changed, and the dirty + // thread may have lost the race, but it is the caller's responsibility not to try to mark + // this node changed twice. The end result of racing markers must be a changed node, since one + // of the markers is trying to mark the node changed. + Preconditions.checkState(isChanged != isChanged(), + "Cannot mark node dirty twice or changed twice: %s", this); + Preconditions.checkState(value == null, "Value should have been reset already %s", this); + Preconditions.checkState(directDeps == null, "direct deps not already reset %s", this); + if (isChanged) { + // If the changed marker lost the race, we just need to mark changed in this method -- all + // other work was done by the dirty marker. + buildingState.markChanged(); + } + return null; + } + + /** + * Marks this entry as up-to-date at this version. + * + * @return {@link Set} of reverse dependencies to signal that this node is done. + */ + synchronized Set<SkyKey> markClean() { + this.value = buildingState.getLastBuildValue(); + // This checks both the value and the direct deps, but since we're passing in the same value, + // the value check should be trivial. + Preconditions.checkState(buildingState.unchangedFromLastBuild(this.value), + "Direct deps must be the same as those found last build for node to be marked clean: %s", + this); + Preconditions.checkState(isDirty(), this); + Preconditions.checkState(!buildingState.isChanged(), "shouldn't be changed: %s", this); + return setStateFinishedAndReturnReverseDeps(); + } + + /** + * Forces this node to be reevaluated, even if none of its dependencies are known to have + * changed. + * + * <p>Used when an external caller has reason to believe that re-evaluating may yield a new + * result. This method should not be used if one of the normal deps of this node has changed, + * the usual change-pruning process should work in that case. + */ + synchronized void forceRebuild() { + buildingState.forceChanged(); + } + + /** + * Gets the current version of this entry. + */ + synchronized Version getVersion() { + return version; + } + + /** + * Gets the current state of checking this dirty entry to see if it must be re-evaluated. Must be + * called each time evaluation of a dirty entry starts to find the proper action to perform next, + * as enumerated by {@link BuildingState.DirtyState}. + * + * @see BuildingState#getDirtyState() + */ + synchronized BuildingState.DirtyState getDirtyState() { + return buildingState.getDirtyState(); + } + + /** + * Should only be called if the entry is dirty. During the examination to see if the entry must be + * re-evaluated, this method returns the next group of children to be checked. Callers should + * have already called {@link #getDirtyState} and received a return value of + * {@link BuildingState.DirtyState#CHECK_DEPENDENCIES} before calling this method -- any other + * return value from {@link #getDirtyState} means that this method must not be called, since + * whether or not the node needs to be rebuilt is already known. + * + * <p>Deps are returned in groups. The deps in each group were requested in parallel by the + * {@code SkyFunction} last build, meaning independently of the values of any other deps in this + * group (although possibly depending on deps in earlier groups). Thus the caller may check all + * the deps in this group in parallel, since the deps in all previous groups are verified + * unchanged. See {@link SkyFunction.Environment#getValues} for more on dependency groups. + * + * <p>The caller should register these as deps of this entry using {@link #addTemporaryDirectDeps} + * before checking them. + * + * @see BuildingState#getNextDirtyDirectDeps() + */ + synchronized Collection<SkyKey> getNextDirtyDirectDeps() { + return buildingState.getNextDirtyDirectDeps(); + } + + /** + * Returns the set of direct dependencies. This may only be called while the node is being + * evaluated, that is, before {@link #setValue} and after {@link #markDirty}. + */ + synchronized Set<SkyKey> getTemporaryDirectDeps() { + Preconditions.checkState(!isDone(), "temporary shouldn't be done: %s", this); + return buildingState.getDirectDepsForBuild(); + } + + synchronized boolean noDepsLastBuild() { + return buildingState.noDepsLastBuild(); + } + + /** + * Remove dep from direct deps. This should only be called if this entry is about to be + * committed as a cycle node, but some of its children were not checked for cycles, either + * because the cycle was discovered before some children were checked; some children didn't have a + * chance to finish before the evaluator aborted; or too many cycles were found when it came time + * to check the children. + */ + synchronized void removeUnfinishedDeps(Set<SkyKey> unfinishedDeps) { + buildingState.removeDirectDeps(unfinishedDeps); + } + + synchronized void addTemporaryDirectDeps(GroupedListHelper<SkyKey> helper) { + Preconditions.checkState(!isDone(), "add temp shouldn't be done: %s %s", helper, this); + buildingState.addDirectDeps(helper); + } + + /** + * Returns true if the node is ready to be evaluated, i.e., it has been signaled exactly as many + * times as it has temporary dependencies. This may only be called while the node is being + * evaluated, that is, before {@link #setValue} and after {@link #markDirty}. + */ + synchronized boolean isReady() { + Preconditions.checkState(!isDone(), "can't be ready if done: %s", this); + return buildingState.isReady(); + } + + @Override + @SuppressWarnings("deprecation") + public String toString() { + return Objects.toStringHelper(this) // MoreObjects is not in Guava + .add("value", value) + .add("version", version) + .add("directDeps", directDeps == null ? null : GroupedList.create(directDeps)) + .add("reverseDeps", REVERSE_DEPS_UTIL.toString(this)) + .add("buildingState", buildingState).toString(); + } + + /** + * Do not use except in custom evaluator implementations! Added only temporarily. + * + * <p>Clones a NodeEntry iff it is a done node. Otherwise it fails. + */ + @Deprecated + public synchronized NodeEntry cloneNodeEntry() { + // As this is temporary, for now lets limit to done nodes + Preconditions.checkState(isDone(), "Only done nodes can be copied"); + NodeEntry nodeEntry = new NodeEntry(); + nodeEntry.value = value; + nodeEntry.version = this.version; + REVERSE_DEPS_UTIL.addReverseDeps(nodeEntry, REVERSE_DEPS_UTIL.getReverseDeps(this)); + nodeEntry.directDeps = directDeps; + nodeEntry.buildingState = null; + return nodeEntry; + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/NullDirtyKeyTrackerImpl.java b/src/main/java/com/google/devtools/build/skyframe/NullDirtyKeyTrackerImpl.java new file mode 100644 index 0000000..937f1cb --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/NullDirtyKeyTrackerImpl.java
@@ -0,0 +1,37 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.collect.ImmutableSet; + +import java.util.Set; + +/** + * Tracks nothing. Should be used by evaluators that don't do dirty node garbage collection. + */ +public class NullDirtyKeyTrackerImpl implements DirtyKeyTracker { + + @Override + public void dirty(SkyKey skyKey) { + } + + @Override + public void notDirty(SkyKey skyKey) { + } + + @Override + public Set<SkyKey> getDirtyKeys() { + return ImmutableSet.of(); + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ParallelEvaluator.java b/src/main/java/com/google/devtools/build/skyframe/ParallelEvaluator.java new file mode 100644 index 0000000..39f11d7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/ParallelEvaluator.java
@@ -0,0 +1,1786 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.NestedSetVisitor; +import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor; +import com.google.devtools.build.lib.concurrent.ExecutorShutdownUtil; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.util.GroupedList.GroupedListHelper; +import com.google.devtools.build.skyframe.BuildingState.DirtyState; +import com.google.devtools.build.skyframe.EvaluationProgressReceiver.EvaluationState; +import com.google.devtools.build.skyframe.NodeEntry.DependencyState; +import com.google.devtools.build.skyframe.Scheduler.SchedulerException; +import com.google.devtools.build.skyframe.SkyFunctionException.ReifiedSkyFunctionException; +import com.google.devtools.build.skyframe.ValueOrExceptionUtils.BottomException; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.annotation.Nullable; + +/** + * Evaluates a set of given functions ({@code SkyFunction}s) with arguments ({@code SkyKey}s). + * Cycles are not allowed and are detected during the traversal. + * + * <p>This class implements multi-threaded evaluation. This is a fairly complex process that has + * strong consistency requirements between the {@link ProcessableGraph}, the nodes in the graph of + * type {@link NodeEntry}, the work queue, and the set of in-flight nodes. + * + * <p>The basic invariants are: + * + * <p>A node can be in one of three states: ready, waiting, and done. A node is ready if and only + * if all of its dependencies have been signaled. A node is done if it has a value. It is waiting + * if not all of its dependencies have been signaled. + * + * <p>A node must be in the work queue if and only if it is ready. It is an error for a node to be + * in the work queue twice at the same time. + * + * <p>A node is considered in-flight if it has been created, and is not done yet. In case of an + * interrupt, the work queue is discarded, and the in-flight set is used to remove partially + * computed values. + * + * <p>Each evaluation of the graph takes place at a "version," which is currently given by a + * non-negative {@code long}. The version can also be thought of as an "mtime." Each node in the + * graph has a version, which is the last version at which its value changed. This version data is + * used to avoid unnecessary re-evaluation of values. If a node is re-evaluated and found to have + * the same data as before, its version (mtime) remains the same. If all of a node's children's + * have the same version as before, its re-evaluation can be skipped. + * + * <p>This class is not intended for direct use, and is only exposed as public for use in + * evaluation implementations outside of this package. + */ +public final class ParallelEvaluator implements Evaluator { + private final ProcessableGraph graph; + private final Version graphVersion; + + private final Predicate<SkyKey> nodeEntryIsDone = new Predicate<SkyKey>() { + @Override + public boolean apply(SkyKey skyKey) { + return isDoneForBuild(graph.get(skyKey)); + } + }; + + private final ImmutableMap<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions; + + private final EventHandler reporter; + private final NestedSetVisitor<TaggedEvents> replayingNestedSetEventVisitor; + private final boolean keepGoing; + private final int threadCount; + @Nullable private final EvaluationProgressReceiver progressReceiver; + private final DirtyKeyTracker dirtyKeyTracker; + private final AtomicBoolean errorEncountered = new AtomicBoolean(false); + + private static final Interner<SkyKey> KEY_CANONICALIZER = Interners.newWeakInterner(); + + public ParallelEvaluator(ProcessableGraph graph, Version graphVersion, + ImmutableMap<? extends SkyFunctionName, ? extends SkyFunction> skyFunctions, + final EventHandler reporter, + MemoizingEvaluator.EmittedEventState emittedEventState, + boolean keepGoing, int threadCount, + @Nullable EvaluationProgressReceiver progressReceiver, + DirtyKeyTracker dirtyKeyTracker) { + this.graph = graph; + this.skyFunctions = skyFunctions; + this.graphVersion = graphVersion; + this.reporter = Preconditions.checkNotNull(reporter); + this.keepGoing = keepGoing; + this.threadCount = threadCount; + this.progressReceiver = progressReceiver; + this.dirtyKeyTracker = Preconditions.checkNotNull(dirtyKeyTracker); + this.replayingNestedSetEventVisitor = + new NestedSetVisitor<>(new NestedSetEventReceiver(reporter), emittedEventState); + } + + /** + * Receives the events from the NestedSet and delegates to the reporter. + */ + private static class NestedSetEventReceiver implements NestedSetVisitor.Receiver<TaggedEvents> { + + private final EventHandler reporter; + + public NestedSetEventReceiver(EventHandler reporter) { + this.reporter = reporter; + } + @Override + public void accept(TaggedEvents events) { + String tag = events.getTag(); + for (Event e : events.getEvents()) { + reporter.handle(e.withTag(tag)); + } + } + } + + /** + * A suitable {@link SkyFunction.Environment} implementation. + */ + class SkyFunctionEnvironment implements SkyFunction.Environment { + private boolean building = true; + private boolean valuesMissing = false; + private SkyKey depErrorKey = null; + private final SkyKey skyKey; + private SkyValue value = null; + private ErrorInfo errorInfo = null; + private final Map<SkyKey, ValueWithMetadata> bubbleErrorInfo; + /** The set of values previously declared as dependencies. */ + private final Set<SkyKey> directDeps; + + /** + * The grouped list of values requested during this build as dependencies. On a subsequent + * build, if this value is dirty, all deps in the same dependency group can be checked in + * parallel for changes. In other words, if dep1 and dep2 are in the same group, then dep1 will + * be checked in parallel with dep2. See {@link #getValues} for more. + */ + private final GroupedListHelper<SkyKey> newlyRequestedDeps = new GroupedListHelper<>(); + + /** + * The value visitor managing the thread pool. Used to enqueue parents when this value is + * finished, and, during testing, to block until an exception is thrown if a value builder + * requests that. + */ + private final ValueVisitor visitor; + + /** The set of errors encountered while fetching children. */ + private final Collection<ErrorInfo> childErrorInfos = new LinkedHashSet<>(); + private final StoredEventHandler eventHandler = new StoredEventHandler() { + @Override + public void handle(Event e) { + checkActive(); + switch (e.getKind()) { + case INFO: + throw new UnsupportedOperationException("Values should not display INFO messages: " + + skyKey + " printed " + e.getLocation() + ": " + e.getMessage()); + case PROGRESS: + reporter.handle(e); + break; + default: + super.handle(e); + } + } + }; + + private SkyFunctionEnvironment(SkyKey skyKey, Set<SkyKey> directDeps, ValueVisitor visitor) { + this(skyKey, directDeps, null, visitor); + } + + private SkyFunctionEnvironment(SkyKey skyKey, Set<SkyKey> directDeps, + @Nullable Map<SkyKey, ValueWithMetadata> bubbleErrorInfo, ValueVisitor visitor) { + this.skyKey = skyKey; + this.directDeps = Collections.unmodifiableSet(directDeps); + this.bubbleErrorInfo = bubbleErrorInfo; + this.childErrorInfos.addAll(childErrorInfos); + this.visitor = visitor; + } + + private void checkActive() { + Preconditions.checkState(building, skyKey); + } + + private NestedSet<TaggedEvents> buildEvents(boolean missingChildren) { + // Aggregate the nested set of events from the direct deps, also adding the events from + // building this value. + NestedSetBuilder<TaggedEvents> eventBuilder = NestedSetBuilder.stableOrder(); + ImmutableList<Event> events = eventHandler.getEvents(); + if (!events.isEmpty()) { + eventBuilder.add(new TaggedEvents(getTagFromKey(), events)); + } + for (SkyKey dep : graph.get(skyKey).getTemporaryDirectDeps()) { + ValueWithMetadata value = getValueMaybeFromError(dep, bubbleErrorInfo); + if (value != null) { + eventBuilder.addTransitive(value.getTransitiveEvents()); + } else { + Preconditions.checkState(missingChildren, "", dep, skyKey); + } + } + return eventBuilder.build(); + } + + /** + * If this node has an error, that is, if errorInfo is non-null, do nothing. Otherwise, set + * errorInfo to the union of the child errors that were recorded earlier by getValueOrException, + * if there are any. + */ + private void finalizeErrorInfo() { + if (errorInfo == null && !childErrorInfos.isEmpty()) { + errorInfo = new ErrorInfo(skyKey, childErrorInfos); + } + } + + private void setValue(SkyValue newValue) { + Preconditions.checkState(errorInfo == null && bubbleErrorInfo == null, + "%s %s %s %s", skyKey, newValue, errorInfo, bubbleErrorInfo); + Preconditions.checkState(value == null, "%s %s %s", skyKey, value, newValue); + value = newValue; + } + + /** + * Set this node to be in error. The node's value must not have already been set. However, all + * dependencies of this node <i>must</i> already have been registered, since this method may + * register a dependence on the error transience node, which should always be the last dep. + */ + private void setError(ErrorInfo errorInfo) { + Preconditions.checkState(value == null, "%s %s %s", skyKey, value, errorInfo); + Preconditions.checkState(this.errorInfo == null, + "%s %s %s", skyKey, this.errorInfo, errorInfo); + + if (errorInfo.isTransient()) { + DependencyState triState = + graph.get(ErrorTransienceValue.key()).addReverseDepAndCheckIfDone(skyKey); + Preconditions.checkState(triState == DependencyState.DONE, + "%s %s %s", skyKey, triState, errorInfo); + + final NodeEntry state = graph.get(skyKey); + state.addTemporaryDirectDeps( + GroupedListHelper.create(ImmutableList.of(ErrorTransienceValue.key()))); + state.signalDep(); + } + + this.errorInfo = Preconditions.checkNotNull(errorInfo, skyKey); + } + + /** Get a child of the value being evaluated, for use by the value builder. */ + private ValueOrUntypedException getValueOrUntypedException(SkyKey depKey) { + checkActive(); + depKey = KEY_CANONICALIZER.intern(depKey); // Canonicalize SkyKeys to save memory. + ValueWithMetadata value = getValueMaybeFromError(depKey, bubbleErrorInfo); + if (value == null) { + // If this entry is not yet done then (optionally) record the missing dependency and return + // null. + valuesMissing = true; + if (bubbleErrorInfo != null) { + // Values being built just for their errors don't get to request new children. + return ValueOrExceptionUtils.ofNull(); + } + Preconditions.checkState(!directDeps.contains(depKey), "%s %s %s", skyKey, depKey, value); + addDep(depKey); + valuesMissing = true; + return ValueOrExceptionUtils.ofNull(); + } + + if (!directDeps.contains(depKey)) { + // If this child is done, we will return it, but also record that it was newly requested so + // that the dependency can be properly registered in the graph. + addDep(depKey); + } + + replayingNestedSetEventVisitor.visit(value.getTransitiveEvents()); + ErrorInfo errorInfo = value.getErrorInfo(); + + if (errorInfo != null) { + childErrorInfos.add(errorInfo); + } + + if (value.getValue() != null && (keepGoing || errorInfo == null)) { + // The caller is given the value of the value if there was no error computing the value, or + // if this is a keepGoing build (in which case each value should get child values even if + // there are also errors). + return ValueOrExceptionUtils.ofValueUntyped(value.getValue()); + } + + // There was an error building the value, which we will either report by throwing an exception + // or insulate the caller from by returning null. + Preconditions.checkNotNull(errorInfo, "%s %s %s", skyKey, depKey, value); + + if (!keepGoing && errorInfo.getException() != null && bubbleErrorInfo == null) { + // Child errors should not be propagated in noKeepGoing mode (except during error bubbling). + // Instead we should fail fast. + + // We arbitrarily record the first child error. + if (depErrorKey == null) { + depErrorKey = depKey; + } + valuesMissing = true; + return ValueOrExceptionUtils.ofNull(); + } + + if (bubbleErrorInfo != null) { + // Set interrupted status, so that builder doesn't try anything fancy after this. + Thread.currentThread().interrupt(); + } + if (errorInfo.getException() != null) { + // Give builder a chance to handle this exception. + Exception e = errorInfo.getException(); + return ValueOrExceptionUtils.ofExn(e); + } + // In a cycle. + Preconditions.checkState(!Iterables.isEmpty(errorInfo.getCycleInfo()), "%s %s %s %s", skyKey, + depKey, errorInfo, value); + valuesMissing = true; + return ValueOrExceptionUtils.ofNull(); + } + + private <E extends Exception> ValueOrException<E> getValueOrException(SkyKey depKey, + Class<E> exceptionClass) { + return ValueOrExceptionUtils.downcovert(getValueOrException(depKey, exceptionClass, + BottomException.class), exceptionClass); + } + + private <E1 extends Exception, E2 extends Exception> ValueOrException2<E1, E2> + getValueOrException(SkyKey depKey, Class<E1> exceptionClass1, Class<E2> exceptionClass2) { + return ValueOrExceptionUtils.downconvert(getValueOrException(depKey, exceptionClass1, + exceptionClass2, BottomException.class), exceptionClass1, exceptionClass2); + } + + private <E1 extends Exception, E2 extends Exception, E3 extends Exception> + ValueOrException3<E1, E2, E3> getValueOrException(SkyKey depKey, Class<E1> exceptionClass1, + Class<E2> exceptionClass2, Class<E3> exceptionClass3) { + return ValueOrExceptionUtils.downconvert(getValueOrException(depKey, exceptionClass1, + exceptionClass2, exceptionClass3, BottomException.class), exceptionClass1, + exceptionClass2, exceptionClass3); + } + + private <E1 extends Exception, E2 extends Exception, E3 extends Exception, + E4 extends Exception> ValueOrException4<E1, E2, E3, E4> getValueOrException(SkyKey depKey, + Class<E1> exceptionClass1, Class<E2> exceptionClass2, Class<E3> exceptionClass3, + Class<E4> exceptionClass4) { + SkyFunctionException.validateExceptionType(exceptionClass1); + SkyFunctionException.validateExceptionType(exceptionClass2); + SkyFunctionException.validateExceptionType(exceptionClass3); + SkyFunctionException.validateExceptionType(exceptionClass4); + ValueOrUntypedException voe = getValueOrUntypedException(depKey); + SkyValue value = voe.getValue(); + if (value != null) { + return ValueOrExceptionUtils.ofValue(value); + } + Exception e = voe.getException(); + if (e != null) { + if (exceptionClass1.isInstance(e)) { + return ValueOrExceptionUtils.ofExn1(exceptionClass1.cast(e)); + } + if (exceptionClass2.isInstance(e)) { + return ValueOrExceptionUtils.ofExn2(exceptionClass2.cast(e)); + } + if (exceptionClass3.isInstance(e)) { + return ValueOrExceptionUtils.ofExn3(exceptionClass3.cast(e)); + } + if (exceptionClass4.isInstance(e)) { + return ValueOrExceptionUtils.ofExn4(exceptionClass4.cast(e)); + } + } + valuesMissing = true; + return ValueOrExceptionUtils.ofNullValue(); + } + + @Override + @Nullable + public SkyValue getValue(SkyKey depKey) { + try { + return getValueOrThrow(depKey, BottomException.class); + } catch (BottomException e) { + throw new IllegalStateException("shouldn't reach here"); + } + } + + @Override + @Nullable + public <E extends Exception> SkyValue getValueOrThrow(SkyKey depKey, Class<E> exceptionClass) + throws E { + return getValueOrException(depKey, exceptionClass).get(); + } + + @Override + @Nullable + public <E1 extends Exception, E2 extends Exception> SkyValue getValueOrThrow(SkyKey depKey, + Class<E1> exceptionClass1, Class<E2> exceptionClass2) throws E1, E2 { + return getValueOrException(depKey, exceptionClass1, exceptionClass2).get(); + } + + @Override + @Nullable + public <E1 extends Exception, E2 extends Exception, + E3 extends Exception> SkyValue getValueOrThrow(SkyKey depKey, Class<E1> exceptionClass1, + Class<E2> exceptionClass2, Class<E3> exceptionClass3) throws E1, E2, E3 { + return getValueOrException(depKey, exceptionClass1, exceptionClass2, exceptionClass3).get(); + } + + @Override + public <E1 extends Exception, E2 extends Exception, E3 extends Exception, + E4 extends Exception> SkyValue getValueOrThrow(SkyKey depKey, Class<E1> exceptionClass1, + Class<E2> exceptionClass2, Class<E3> exceptionClass3, Class<E4> exceptionClass4) throws E1, + E2, E3, E4 { + return getValueOrException(depKey, exceptionClass1, exceptionClass2, exceptionClass3, + exceptionClass4).get(); + } + + @Override + public Map<SkyKey, SkyValue> getValues(Iterable<SkyKey> depKeys) { + return Maps.transformValues(getValuesOrThrow(depKeys, BottomException.class), + GET_VALUE_FROM_VOE); + } + + @Override + public <E extends Exception> Map<SkyKey, ValueOrException<E>> getValuesOrThrow( + Iterable<SkyKey> depKeys, Class<E> exceptionClass) { + return Maps.transformValues(getValuesOrThrow(depKeys, exceptionClass, BottomException.class), + makeSafeDowncastToVOEFunction(exceptionClass)); + } + + @Override + public <E1 extends Exception, + E2 extends Exception> Map<SkyKey, ValueOrException2<E1, E2>> getValuesOrThrow( + Iterable<SkyKey> depKeys, Class<E1> exceptionClass1, Class<E2> exceptionClass2) { + return Maps.transformValues(getValuesOrThrow(depKeys, exceptionClass1, exceptionClass2, + BottomException.class), makeSafeDowncastToVOE2Function(exceptionClass1, + exceptionClass2)); + } + + @Override + public <E1 extends Exception, E2 extends Exception, E3 extends Exception> Map<SkyKey, + ValueOrException3<E1, E2, E3>> getValuesOrThrow(Iterable<SkyKey> depKeys, + Class<E1> exceptionClass1, Class<E2> exceptionClass2, Class<E3> exceptionClass3) { + return Maps.transformValues(getValuesOrThrow(depKeys, exceptionClass1, exceptionClass2, + exceptionClass3, BottomException.class), makeSafeDowncastToVOE3Function(exceptionClass1, + exceptionClass2, exceptionClass3)); + } + + @Override + public <E1 extends Exception, E2 extends Exception, E3 extends Exception, + E4 extends Exception> Map<SkyKey, ValueOrException4<E1, E2, E3, E4>> getValuesOrThrow( + Iterable<SkyKey> depKeys, Class<E1> exceptionClass1, Class<E2> exceptionClass2, + Class<E3> exceptionClass3, Class<E4> exceptionClass4) { + Map<SkyKey, ValueOrException4<E1, E2, E3, E4>> result = new HashMap<>(); + newlyRequestedDeps.startGroup(); + for (SkyKey depKey : depKeys) { + if (result.containsKey(depKey)) { + continue; + } + result.put(depKey, getValueOrException(depKey, exceptionClass1, exceptionClass2, + exceptionClass3, exceptionClass4)); + } + newlyRequestedDeps.endGroup(); + return Collections.unmodifiableMap(result); + } + + private void addDep(SkyKey key) { + if (!newlyRequestedDeps.contains(key)) { + // dep may have been requested already this evaluation. If not, add it. + newlyRequestedDeps.add(key); + } + } + + @Override + public boolean valuesMissing() { + return valuesMissing; + } + + /** + * If {@code !keepGoing} and there is at least one dep in error, returns a dep in error. + * Otherwise returns {@code null}. + */ + @Nullable + private SkyKey getDepErrorKey() { + return depErrorKey; + } + + @Override + public EventHandler getListener() { + checkActive(); + return eventHandler; + } + + private void doneBuilding() { + building = false; + } + + /** + * Apply the change to the graph (mostly) atomically and signal all nodes that are waiting for + * this node to complete. Adding nodes and signaling is not atomic, but may need to be changed + * for interruptibility. + * + * <p>Parents are only enqueued if {@code enqueueParents} holds. Parents should be enqueued + * unless (1) this node is being built after the main evaluation has aborted, or (2) this node + * is being built with --nokeep_going, and so we are about to shut down the main evaluation + * anyway. + * + * <p>The node entry is informed if the node's value and error are definitive via the flag + * {@code completeValue}. + */ + void commit(boolean enqueueParents) { + NodeEntry primaryEntry = Preconditions.checkNotNull(graph.get(skyKey), skyKey); + // Construct the definitive error info, if there is one. + finalizeErrorInfo(); + + // We have the following implications: + // errorInfo == null => value != null => enqueueParents. + // All these implications are strict: + // (1) errorInfo != null && value != null happens for values with recoverable errors. + // (2) value == null && enqueueParents happens for values that are found to have errors + // during a --keep_going build. + + NestedSet<TaggedEvents> events = buildEvents(/*missingChildren=*/false); + if (value == null) { + Preconditions.checkNotNull(errorInfo, "%s %s", skyKey, primaryEntry); + // We could consider using max(childVersions) here instead of graphVersion. When full + // versioning is implemented, this would allow evaluation at a version between + // max(childVersions) and graphVersion to re-use this result. + Set<SkyKey> reverseDeps = primaryEntry.setValue( + ValueWithMetadata.error(errorInfo, events), graphVersion); + signalValuesAndEnqueueIfReady(enqueueParents ? visitor : null, reverseDeps, graphVersion); + } else { + // We must be enqueueing parents if we have a value. + Preconditions.checkState(enqueueParents, "%s %s", skyKey, primaryEntry); + Set<SkyKey> reverseDeps; + Version valueVersion; + // If this entry is dirty, setValue may not actually change it, if it determines that + // the data being written now is the same as the data already present in the entry. + // We could consider using max(childVersions) here instead of graphVersion. When full + // versioning is implemented, this would allow evaluation at a version between + // max(childVersions) and graphVersion to re-use this result. + reverseDeps = primaryEntry.setValue( + ValueWithMetadata.normal(value, errorInfo, events), graphVersion); + // Note that if this update didn't actually change the value entry, this version may not + // be the graph version. + valueVersion = primaryEntry.getVersion(); + Preconditions.checkState(valueVersion.atMost(graphVersion), + "%s should be at most %s in the version partial ordering", + valueVersion, graphVersion); + if (progressReceiver != null) { + // Tell the receiver that this value was built. If valueVersion.equals(graphVersion), it + // was evaluated this run, and so was changed. Otherwise, it is less than graphVersion, + // by the Preconditions check above, and was not actually changed this run -- when it was + // written above, its version stayed below this update's version, so its value remains the + // same as before. + progressReceiver.evaluated(skyKey, value, + valueVersion.equals(graphVersion) ? EvaluationState.BUILT : EvaluationState.CLEAN); + } + signalValuesAndEnqueueIfReady(visitor, reverseDeps, valueVersion); + } + + visitor.notifyDone(skyKey); + replayingNestedSetEventVisitor.visit(events); + } + + @Nullable + private String getTagFromKey() { + return skyFunctions.get(skyKey.functionName()).extractTag(skyKey); + } + + /** + * Gets the latch that is counted down when an exception is thrown in {@code + * AbstractQueueVisitor}. For use in tests to check if an exception actually was thrown. Calling + * {@code AbstractQueueVisitor#awaitExceptionForTestingOnly} can throw a spurious {@link + * InterruptedException} because {@link CountDownLatch#await} checks the interrupted bit before + * returning, even if the latch is already at 0. See bug "testTwoErrors is flaky". + */ + CountDownLatch getExceptionLatchForTesting() { + return visitor.getExceptionLatchForTestingOnly(); + } + + @Override + public boolean inErrorBubblingForTesting() { + return bubbleErrorInfo != null; + } + } + + private static final Function<ValueOrException<BottomException>, SkyValue> GET_VALUE_FROM_VOE = + new Function<ValueOrException<BottomException>, SkyValue>() { + @Override + public SkyValue apply(ValueOrException<BottomException> voe) { + return ValueOrExceptionUtils.downcovert(voe); + } + }; + + private static <E extends Exception> + Function<ValueOrException2<E, BottomException>, ValueOrException<E>> + makeSafeDowncastToVOEFunction(final Class<E> exceptionClass) { + return new Function<ValueOrException2<E, BottomException>, ValueOrException<E>>() { + @Override + public ValueOrException<E> apply(ValueOrException2<E, BottomException> voe) { + return ValueOrExceptionUtils.downcovert(voe, exceptionClass); + } + }; + } + + private static <E1 extends Exception, E2 extends Exception> + Function<ValueOrException3<E1, E2, BottomException>, ValueOrException2<E1, E2>> + makeSafeDowncastToVOE2Function(final Class<E1> exceptionClass1, + final Class<E2> exceptionClass2) { + return new Function<ValueOrException3<E1, E2, BottomException>, + ValueOrException2<E1, E2>>() { + @Override + public ValueOrException2<E1, E2> apply(ValueOrException3<E1, E2, BottomException> voe) { + return ValueOrExceptionUtils.downconvert(voe, exceptionClass1, exceptionClass2); + } + }; + } + + private static <E1 extends Exception, E2 extends Exception, E3 extends Exception> + Function<ValueOrException4<E1, E2, E3, BottomException>, ValueOrException3<E1, E2, E3>> + makeSafeDowncastToVOE3Function(final Class<E1> exceptionClass1, + final Class<E2> exceptionClass2, final Class<E3> exceptionClass3) { + return new Function<ValueOrException4<E1, E2, E3, BottomException>, + ValueOrException3<E1, E2, E3>>() { + @Override + public ValueOrException3<E1, E2, E3> apply(ValueOrException4<E1, E2, E3, + BottomException> voe) { + return ValueOrExceptionUtils.downconvert(voe, exceptionClass1, exceptionClass2, + exceptionClass3); + } + }; + } + + private class ValueVisitor extends AbstractQueueVisitor { + private AtomicBoolean preventNewEvaluations = new AtomicBoolean(false); + private final Set<SkyKey> inflightNodes = Sets.newConcurrentHashSet(); + + private ValueVisitor(int threadCount) { + super(/*concurrent*/true, + threadCount, + threadCount, + 1, TimeUnit.SECONDS, + /*failFastOnException*/true, + /*failFastOnInterrupt*/true, + "skyframe-evaluator"); + } + + @Override + protected boolean isCriticalError(Throwable e) { + return e instanceof RuntimeException; + } + + protected void waitForCompletion() throws InterruptedException { + work(/*failFastOnInterrupt=*/true); + } + + public void enqueueEvaluation(final SkyKey key) { + // We unconditionally add the key to the set of in-flight nodes because even if evaluation is + // never scheduled we still want to remove the previously created NodeEntry from the graph. + // Otherwise we would leave the graph in a weird state (wasteful garbage in the best case and + // inconsistent in the worst case). + boolean newlyEnqueued = inflightNodes.add(key); + // All nodes enqueued for evaluation will be either verified clean, re-evaluated, or cleaned + // up after being in-flight when an error happens in nokeep_going mode or in the event of an + // interrupt. In any of these cases, they won't be dirty anymore. + if (newlyEnqueued) { + dirtyKeyTracker.notDirty(key); + } + if (preventNewEvaluations.get()) { + return; + } + if (newlyEnqueued && progressReceiver != null) { + progressReceiver.enqueueing(key); + } + enqueue(new Evaluate(this, key)); + } + + public void preventNewEvaluations() { + preventNewEvaluations.set(true); + } + + public void notifyDone(SkyKey key) { + inflightNodes.remove(key); + } + + private boolean isInflight(SkyKey key) { + return inflightNodes.contains(key); + } + } + + /** + * An action that evaluates a value. + */ + private class Evaluate implements Runnable { + private final ValueVisitor visitor; + /** The name of the value to be evaluated. */ + private final SkyKey skyKey; + + private Evaluate(ValueVisitor visitor, SkyKey skyKey) { + this.visitor = visitor; + this.skyKey = skyKey; + } + + private void enqueueChild(SkyKey skyKey, NodeEntry entry, SkyKey child) { + Preconditions.checkState(!entry.isDone(), "%s %s", skyKey, entry); + Preconditions.checkState(!ErrorTransienceValue.key().equals(child), + "%s cannot request ErrorTransienceValue as a dep: %s", skyKey, entry); + + NodeEntry depEntry = graph.createIfAbsent(child); + switch (depEntry.addReverseDepAndCheckIfDone(skyKey)) { + case DONE : + if (entry.signalDep(depEntry.getVersion())) { + // This can only happen if there are no more children to be added. + visitor.enqueueEvaluation(skyKey); + } + break; + case ADDED_DEP : + break; + case NEEDS_SCHEDULING : + visitor.enqueueEvaluation(child); + break; + } + } + + /** + * Returns true if this depGroup consists of the error transience value and the error transience + * value is newer than the entry, meaning that the entry must be re-evaluated. + */ + private boolean invalidatedByErrorTransience(Collection<SkyKey> depGroup, NodeEntry entry) { + return depGroup.size() == 1 + && depGroup.contains(ErrorTransienceValue.key()) + && !graph.get(ErrorTransienceValue.key()).getVersion().atMost(entry.getVersion()); + } + + @Override + public void run() { + NodeEntry state = graph.get(skyKey); + Preconditions.checkNotNull(state, "%s %s", skyKey, state); + Preconditions.checkState(state.isReady(), "%s %s", skyKey, state); + + if (state.isDirty()) { + switch (state.getDirtyState()) { + case CHECK_DEPENDENCIES: + // Evaluating a dirty node for the first time, and checking its children to see if any + // of them have changed. Note that there must be dirty children for this to happen. + + // Check the children group by group -- we don't want to evaluate a value that is no + // longer needed because an earlier dependency changed. For example, //foo:foo depends + // on target //bar:bar and is built. Then foo/BUILD is modified to remove the dependence + // on bar, and bar/BUILD is deleted. Reloading //bar:bar would incorrectly throw an + // exception. To avoid this, we must reload foo/BUILD first, at which point we will + // discover that it has changed, and re-evaluate target //foo:foo from scratch. + // On the other hand, when an action requests all of its inputs, we can safely check all + // of them in parallel on a subsequent build. So we allow checking an entire group in + // parallel here, if the node builder requested a group last build. + Collection<SkyKey> directDepsToCheck = state.getNextDirtyDirectDeps(); + + if (invalidatedByErrorTransience(directDepsToCheck, state)) { + // If this dep is the ErrorTransienceValue and the ErrorTransienceValue has been + // updated then we need to force a rebuild. We would like to just signal the entry as + // usual, but we can't, because then the ErrorTransienceValue would remain as a dep, + // which would be incorrect if, for instance, the value re-evaluated to a non-error. + state.forceRebuild(); + break; // Fall through to re-evaluation. + } else { + // If this isn't the error transience value, it is safe to add these deps back to the + // node -- even if one of them has changed, the contract of pruning is that the node + // will request these deps again when it rebuilds. We must add these deps before + // enqueuing them, so that the node knows that it depends on them. + state.addTemporaryDirectDeps(GroupedListHelper.create(directDepsToCheck)); + } + + for (SkyKey directDep : directDepsToCheck) { + enqueueChild(skyKey, state, directDep); + } + return; + case VERIFIED_CLEAN: + // No child has a changed value. This node can be marked done and its parents signaled + // without any re-evaluation. + visitor.notifyDone(skyKey); + Set<SkyKey> reverseDeps = state.markClean(); + SkyValue value = state.getValue(); + if (progressReceiver != null && value != null) { + // Tell the receiver that the value was not actually changed this run. + progressReceiver.evaluated(skyKey, value, EvaluationState.CLEAN); + } + signalValuesAndEnqueueIfReady(visitor, reverseDeps, state.getVersion()); + return; + case REBUILDING: + // Nothing to be done if we are already rebuilding. + } + } + + // TODO(bazel-team): Once deps are requested in a deterministic order within a group, or the + // framework is resilient to rearranging group order, change this so that + // SkyFunctionEnvironment "follows along" as the node builder runs, iterating through the + // direct deps that were requested on a previous run. This would allow us to avoid the + // conversion of the direct deps into a set. + Set<SkyKey> directDeps = state.getTemporaryDirectDeps(); + Preconditions.checkState(!directDeps.contains(ErrorTransienceValue.key()), + "%s cannot have a dep on ErrorTransienceValue during building: %s", skyKey, state); + // Get the corresponding SkyFunction and call it on this value. + SkyFunctionEnvironment env = new SkyFunctionEnvironment(skyKey, directDeps, visitor); + SkyFunctionName functionName = skyKey.functionName(); + SkyFunction factory = skyFunctions.get(functionName); + Preconditions.checkState(factory != null, "%s %s", functionName, state); + + SkyValue value = null; + Profiler.instance().startTask(ProfilerTask.SKYFUNCTION, skyKey); + try { + // TODO(bazel-team): count how many of these calls returns null vs. non-null + value = factory.compute(skyKey, env); + } catch (final SkyFunctionException builderException) { + ReifiedSkyFunctionException reifiedBuilderException = + new ReifiedSkyFunctionException(builderException, skyKey); + // Propagated transitive errors are treated the same as missing deps. + if (reifiedBuilderException.getRootCauseSkyKey().equals(skyKey)) { + boolean shouldFailFast = !keepGoing || builderException.isCatastrophic(); + if (shouldFailFast) { + // After we commit this error to the graph but before the eval call completes with the + // error there is a race-like opportunity for the error to be used, either by an + // in-flight computation or by a future computation. + if (errorEncountered.compareAndSet(false, true)) { + // This is the first error encountered. + visitor.preventNewEvaluations(); + } else { + // This is not the first error encountered, so we ignore it so that we can terminate + // with the first error. + return; + } + } + + registerNewlyDiscoveredDepsForDoneEntry(skyKey, state, env); + ErrorInfo errorInfo = new ErrorInfo(reifiedBuilderException); + env.setError(errorInfo); + env.commit(/*enqueueParents=*/keepGoing); + if (!shouldFailFast) { + return; + } + throw SchedulerException.ofError(errorInfo, skyKey); + } + } catch (InterruptedException ie) { + // InterruptedException cannot be thrown by Runnable.run, so we must wrap it. + // Interrupts can be caught by both the Evaluator and the AbstractQueueVisitor. + // The former will unwrap the IE and propagate it as is; the latter will throw a new IE. + throw SchedulerException.ofInterruption(ie, skyKey); + } catch (RuntimeException re) { + // Programmer error (most likely NPE or a failed precondition in a SkyFunction). Output + // some context together with the exception. + String msg = prepareCrashMessage(skyKey, state.getInProgressReverseDeps()); + throw new RuntimeException(msg, re); + } finally { + env.doneBuilding(); + Profiler.instance().completeTask(ProfilerTask.SKYFUNCTION); + } + + GroupedListHelper<SkyKey> newDirectDeps = env.newlyRequestedDeps; + + if (value != null) { + Preconditions.checkState(!env.valuesMissing, + "%s -> %s, ValueEntry: %s", skyKey, newDirectDeps, state); + env.setValue(value); + registerNewlyDiscoveredDepsForDoneEntry(skyKey, state, env); + env.commit(/*enqueueParents=*/true); + return; + } + + if (!newDirectDeps.isEmpty() && env.getDepErrorKey() != null) { + Preconditions.checkState(!keepGoing); + // We encountered a child error in noKeepGoing mode, so we want to fail fast. But we first + // need to add the edge between the current node and the child error it requested so that + // error bubbling can occur. Note that this edge will subsequently be removed during graph + // cleaning (since the current node will never be committed to the graph). + SkyKey childErrorKey = env.getDepErrorKey(); + NodeEntry childErrorEntry = Preconditions.checkNotNull(graph.get(childErrorKey), + "skyKey: %s, state: %s childErrorKey: %s", skyKey, state, childErrorKey); + if (!state.getTemporaryDirectDeps().contains(childErrorKey)) { + // This means the cached error was freshly requested (e.g. the parent has never been + // built before). + Preconditions.checkState(newDirectDeps.contains(childErrorKey), "%s %s %s", state, + childErrorKey, newDirectDeps); + state.addTemporaryDirectDeps(GroupedListHelper.create(ImmutableList.of(childErrorKey))); + DependencyState childErrorState = childErrorEntry.addReverseDepAndCheckIfDone(skyKey); + Preconditions.checkState(childErrorState == DependencyState.DONE, + "skyKey: %s, state: %s childErrorKey: %s", skyKey, state, childErrorKey, + childErrorEntry); + } else { + // This means the cached error was previously requested, and was then subsequently (after + // a restart) requested along with another sibling dep. This can happen on an incremental + // eval call when the parent is dirty and the child error is in a separate dependency + // group from the sibling dep. + Preconditions.checkState(!newDirectDeps.contains(childErrorKey), "%s %s %s", state, + childErrorKey, newDirectDeps); + Preconditions.checkState(childErrorEntry.isDone(), + "skyKey: %s, state: %s childErrorKey: %s", skyKey, state, childErrorKey, + childErrorEntry); + } + ErrorInfo childErrorInfo = Preconditions.checkNotNull(childErrorEntry.getErrorInfo()); + throw SchedulerException.ofError(childErrorInfo, childErrorKey); + } + + // TODO(bazel-team): This code is not safe to interrupt, because we would lose the state in + // newDirectDeps. + + // TODO(bazel-team): An ill-behaved SkyFunction can throw us into an infinite loop where we + // add more dependencies on every run. [skyframe-core] + + // Add all new keys to the set of known deps. + state.addTemporaryDirectDeps(newDirectDeps); + + // If there were no newly requested dependencies, at least one of them was in error or there + // is a bug in the SkyFunction implementation. The environment has collected its errors, so we + // just order it to be built. + if (newDirectDeps.isEmpty()) { + // TODO(bazel-team): This means a bug in the SkyFunction. What to do? + Preconditions.checkState(!env.childErrorInfos.isEmpty(), "%s %s", skyKey, state); + env.commit(/*enqueueParents=*/keepGoing); + if (!keepGoing) { + throw SchedulerException.ofError(state.getErrorInfo(), skyKey); + } + return; + } + + for (SkyKey newDirectDep : newDirectDeps) { + enqueueChild(skyKey, state, newDirectDep); + } + // It is critical that there is no code below this point. + } + + private String prepareCrashMessage(SkyKey skyKey, Iterable<SkyKey> reverseDeps) { + StringBuilder reverseDepDump = new StringBuilder(); + for (SkyKey key : reverseDeps) { + if (reverseDepDump.length() > MAX_REVERSEDEP_DUMP_LENGTH) { + reverseDepDump.append(", ..."); + break; + } + if (reverseDepDump.length() > 0) { + reverseDepDump.append(", "); + } + reverseDepDump.append("'"); + reverseDepDump.append(key.toString()); + reverseDepDump.append("'"); + } + + return String.format( + "Unrecoverable error while evaluating node '%s' (requested by nodes %s)", + skyKey, reverseDepDump); + } + + private static final int MAX_REVERSEDEP_DUMP_LENGTH = 1000; + } + + /** + * Signals all parents that this node is finished. If visitor is not null, also enqueues any + * parents that are ready. If visitor is null, indicating that we are building this node after + * the main build aborted, then skip any parents that are already done (that can happen with + * cycles). + */ + private void signalValuesAndEnqueueIfReady(@Nullable ValueVisitor visitor, Iterable<SkyKey> keys, + Version version) { + if (visitor != null) { + for (SkyKey key : keys) { + if (graph.get(key).signalDep(version)) { + visitor.enqueueEvaluation(key); + } + } + } else { + for (SkyKey key : keys) { + NodeEntry entry = Preconditions.checkNotNull(graph.get(key), key); + if (!entry.isDone()) { + // In cycles, we can have parents that are already done. + entry.signalDep(version); + } + } + } + } + + /** + * If child is not done, removes key from child's reverse deps. Returns whether child should be + * removed from key's entry's direct deps. + */ + private boolean removeIncompleteChild(SkyKey key, SkyKey child) { + NodeEntry childEntry = graph.get(child); + if (!isDoneForBuild(childEntry)) { + childEntry.removeReverseDep(key); + return true; + } + return false; + } + + /** + * Add any additional deps that were registered during the run of a builder that finished by + * creating a node or throwing an error. Builders may throw errors even if all their deps were + * not provided -- we trust that a SkyFunction may be know it should throw an error even if not + * all of its requested deps are done. However, that means we're assuming the SkyFunction would + * throw that same error if all of its requested deps were done. Unfortunately, there is no way to + * enforce that condition. + */ + private void registerNewlyDiscoveredDepsForDoneEntry(SkyKey skyKey, NodeEntry entry, + SkyFunctionEnvironment env) { + Set<SkyKey> unfinishedDeps = new HashSet<>(); + Iterables.addAll(unfinishedDeps, + Iterables.filter(env.newlyRequestedDeps, Predicates.not(nodeEntryIsDone))); + env.newlyRequestedDeps.remove(unfinishedDeps); + entry.addTemporaryDirectDeps(env.newlyRequestedDeps); + for (SkyKey newDep : env.newlyRequestedDeps) { + NodeEntry depEntry = graph.get(newDep); + DependencyState triState = depEntry.addReverseDepAndCheckIfDone(skyKey); + Preconditions.checkState(DependencyState.DONE == triState, + "new dep %s was not already done for %s. ValueEntry: %s. DepValueEntry: %s", + newDep, skyKey, entry, depEntry); + entry.signalDep(); + } + Preconditions.checkState(entry.isReady(), "%s %s %s", skyKey, entry, env.newlyRequestedDeps); + } + + private void informProgressReceiverThatValueIsDone(SkyKey key) { + if (progressReceiver != null) { + NodeEntry entry = graph.get(key); + Preconditions.checkState(entry.isDone(), entry); + SkyValue value = entry.getValue(); + if (value != null) { + Version valueVersion = entry.getVersion(); + Preconditions.checkState(valueVersion.atMost(graphVersion), + "%s should be at most %s in the version partial ordering", valueVersion, graphVersion); + // Nodes with errors will have no value. Don't inform the receiver in that case. + // For most nodes we do not inform the progress receiver if they were already done + // when we retrieve them, but top-level nodes are presumably of more interest. + // If valueVersion is not equal to graphVersion, it must be less than it (by the + // Preconditions check above), and so the node is clean. + progressReceiver.evaluated(key, value, valueVersion.equals(graphVersion) + ? EvaluationState.BUILT + : EvaluationState.CLEAN); + } + } + } + + @Override + @ThreadCompatible + public <T extends SkyValue> EvaluationResult<T> eval(Iterable<SkyKey> skyKeys) + throws InterruptedException { + ImmutableSet<SkyKey> skyKeySet = ImmutableSet.copyOf(skyKeys); + + // Optimization: if all required node values are already present in the cache, return them + // directly without launching the heavy machinery, spawning threads, etc. + // Inform progressReceiver that these nodes are done to be consistent with the main code path. + if (Iterables.all(skyKeySet, nodeEntryIsDone)) { + for (SkyKey skyKey : skyKeySet) { + informProgressReceiverThatValueIsDone(skyKey); + } + // Note that the 'catastrophe' parameter doesn't really matter here (it's only used for + // sanity checking). + return constructResult(null, skyKeySet, null, /*catastrophe=*/false); + } + + if (!keepGoing) { + Set<SkyKey> cachedErrorKeys = new HashSet<>(); + for (SkyKey skyKey : skyKeySet) { + NodeEntry entry = graph.get(skyKey); + if (entry == null) { + continue; + } + if (entry.isDone() && entry.getErrorInfo() != null) { + informProgressReceiverThatValueIsDone(skyKey); + cachedErrorKeys.add(skyKey); + } + } + + // Errors, even cached ones, should halt evaluations not in keepGoing mode. + if (!cachedErrorKeys.isEmpty()) { + // Note that the 'catastrophe' parameter doesn't really matter here (it's only used for + // sanity checking). + return constructResult(null, cachedErrorKeys, null, /*catastrophe=*/false); + } + } + + // We delay this check until we know that some kind of evaluation is necessary, since !keepGoing + // and !keepsEdges are incompatible only in the case of a failed evaluation -- there is no + // need to be overly harsh to callers who are just trying to retrieve a cached result. + Preconditions.checkState(keepGoing || !(graph instanceof InMemoryGraph) + || ((InMemoryGraph) graph).keepsEdges(), + "nokeep_going evaluations are not allowed if graph edges are not kept: %s", skyKeys); + + Profiler.instance().startTask(ProfilerTask.SKYFRAME_EVAL, skyKeySet); + try { + return eval(skyKeySet, new ValueVisitor(threadCount)); + } finally { + Profiler.instance().completeTask(ProfilerTask.SKYFRAME_EVAL); + } + } + + @ThreadCompatible + private <T extends SkyValue> EvaluationResult<T> eval(ImmutableSet<SkyKey> skyKeys, + ValueVisitor visitor) throws InterruptedException { + // We unconditionally add the ErrorTransienceValue here, to ensure that it will be created, and + // in the graph, by the time that it is needed. Creating it on demand in a parallel context sets + // up a race condition, because there is no way to atomically create a node and set its value. + NodeEntry errorTransienceEntry = graph.createIfAbsent(ErrorTransienceValue.key()); + DependencyState triState = errorTransienceEntry.addReverseDepAndCheckIfDone(null); + Preconditions.checkState(triState != DependencyState.ADDED_DEP, + "%s %s", errorTransienceEntry, triState); + if (triState != DependencyState.DONE) { + errorTransienceEntry.setValue(new ErrorTransienceValue(), graphVersion); + // The error transience entry is always invalidated by the RecordingDifferencer. + // Now that the entry's value is set, it is no longer dirty. + dirtyKeyTracker.notDirty(ErrorTransienceValue.key()); + + Preconditions.checkState( + errorTransienceEntry.addReverseDepAndCheckIfDone(null) != DependencyState.ADDED_DEP, + errorTransienceEntry); + } + for (SkyKey skyKey : skyKeys) { + NodeEntry entry = graph.createIfAbsent(skyKey); + // This must be equivalent to the code in enqueueChild above, in order to be thread-safe. + switch (entry.addReverseDepAndCheckIfDone(null)) { + case NEEDS_SCHEDULING: + visitor.enqueueEvaluation(skyKey); + break; + case DONE: + informProgressReceiverThatValueIsDone(skyKey); + break; + case ADDED_DEP: + break; + default: + throw new IllegalStateException(entry + " for " + skyKey + " in unknown state"); + } + } + try { + return waitForCompletionAndConstructResult(visitor, skyKeys); + } finally { + // TODO(bazel-team): In nokeep_going mode or in case of an interrupt, we need to remove + // partial values from the graph. Find a better way to handle those cases. + clean(visitor.inflightNodes); + } + } + + private void clean(Set<SkyKey> inflightNodes) throws InterruptedException { + boolean alreadyInterrupted = Thread.interrupted(); + // This parallel computation is fully cpu-bound, so we use a thread for each processor. + ExecutorService executor = Executors.newFixedThreadPool( + Runtime.getRuntime().availableProcessors(), + new ThreadFactoryBuilder().setNameFormat("ParallelEvaluator#clean %d").build()); + ThrowableRecordingRunnableWrapper wrapper = + new ThrowableRecordingRunnableWrapper("ParallelEvaluator#clean"); + for (final SkyKey key : inflightNodes) { + final NodeEntry entry = graph.get(key); + if (entry.isDone()) { + // Entry may be done in case of a RuntimeException or other programming bug. Do nothing, + // since (a) we're about to crash anyway, and (b) getTemporaryDirectDeps cannot be called + // on a done node, so the call below would crash, which would mask the actual exception + // that caused this state. + continue; + } + executor.execute(wrapper.wrap(new Runnable() { + @Override + public void run() { + cleanInflightNode(key, entry); + } + })); + } + // We uninterruptibly wait for all nodes to be cleaned because we want to make sure the graph + // is left in a good state. + // + // TODO(bazel-team): Come up with a better design for graph cleaning such that we can respond + // to interrupts in constant time. + boolean newlyInterrupted = ExecutorShutdownUtil.uninterruptibleShutdown(executor); + Throwables.propagateIfPossible(wrapper.getFirstThrownError()); + if (newlyInterrupted || alreadyInterrupted) { + throw new InterruptedException(); + } + } + + private void cleanInflightNode(SkyKey key, NodeEntry entry) { + Set<SkyKey> temporaryDeps = entry.getTemporaryDirectDeps(); + graph.remove(key); + for (SkyKey dep : temporaryDeps) { + NodeEntry nodeEntry = graph.get(dep); + // The direct dep might have already been cleaned from the graph. + if (nodeEntry != null) { + // Only bother removing the reverse dep on done nodes since other in-flight nodes will be + // cleaned too. + if (nodeEntry.isDone()) { + nodeEntry.removeReverseDep(key); + } + } + } + } + + private <T extends SkyValue> EvaluationResult<T> waitForCompletionAndConstructResult( + ValueVisitor visitor, Iterable<SkyKey> skyKeys) throws InterruptedException { + Map<SkyKey, ValueWithMetadata> bubbleErrorInfo = null; + boolean catastrophe = false; + try { + visitor.waitForCompletion(); + } catch (final SchedulerException e) { + Throwables.propagateIfPossible(e.getCause(), InterruptedException.class); + if (Thread.interrupted()) { + // As per the contract of AbstractQueueVisitor#work, if an unchecked exception is thrown and + // the build is interrupted, the thrown exception is what will be rethrown. Since the user + // presumably wanted to interrupt the build, we ignore the thrown SchedulerException (which + // doesn't indicate a programming bug) and throw an InterruptedException. + throw new InterruptedException(); + } + + SkyKey errorKey = Preconditions.checkNotNull(e.getFailedValue(), e); + // ErrorInfo could only be null if SchedulerException wrapped an InterruptedException, but + // that should have been propagated. + ErrorInfo errorInfo = Preconditions.checkNotNull(e.getErrorInfo(), errorKey); + catastrophe = errorInfo.isCatastrophic(); + if (!catastrophe || !keepGoing) { + bubbleErrorInfo = bubbleErrorUp(errorInfo, errorKey, skyKeys, visitor); + } else { + // Bubbling the error up requires that graph edges are present for done nodes. This is not + // always the case in a keepGoing evaluation, since it is assumed that done nodes do not + // need to be traversed. In this case, we hope the caller is tolerant of a possibly empty + // result, and return prematurely. + bubbleErrorInfo = ImmutableMap.of(errorKey, graph.get(errorKey).getValueWithMetadata()); + } + } + + // Successful evaluation, either because keepGoing or because we actually did succeed. + // TODO(bazel-team): Maybe report root causes during the build for lower latency. + return constructResult(visitor, skyKeys, bubbleErrorInfo, catastrophe); + } + + /** + * Walk up graph to find a top-level node (without parents) that wanted this failure. Store + * the failed nodes along the way in a map, with ErrorInfos that are appropriate for that layer. + * Example: + * foo bar + * \ / + * unrequested baz + * \ | + * failed-node + * User requests foo, bar. When failed-node fails, we look at its parents. unrequested is not + * in-flight, so we replace failed-node by baz and repeat. We look at baz's parents. foo is + * in-flight, so we replace baz by foo. Since foo is a top-level node and doesn't have parents, + * we then break, since we know a top-level node, foo, that depended on the failed node. + * + * There's the potential for a weird "track jump" here in the case: + * foo + * / \ + * fail1 fail2 + * If fail1 and fail2 fail simultaneously, fail2 may start propagating up in the loop below. + * However, foo requests fail1 first, and then throws an exception based on that. This is not + * incorrect, but may be unexpected. + * + * <p>Returns a map of errors that have been constructed during the bubbling up, so that the + * appropriate error can be returned to the caller, even though that error was not written to the + * graph. If a cycle is detected during the bubbling, this method aborts and returns null so that + * the normal cycle detection can handle the cycle. + * + * <p>Note that we are not propagating error to the first top-level node but to the highest one, + * because during this process we can add useful information about error from other nodes. + */ + private Map<SkyKey, ValueWithMetadata> bubbleErrorUp(final ErrorInfo leafFailure, + SkyKey errorKey, Iterable<SkyKey> skyKeys, ValueVisitor visitor) { + Set<SkyKey> rootValues = ImmutableSet.copyOf(skyKeys); + ErrorInfo error = leafFailure; + Map<SkyKey, ValueWithMetadata> bubbleErrorInfo = new HashMap<>(); + boolean externalInterrupt = false; + while (true) { + NodeEntry errorEntry = graph.get(errorKey); + Iterable<SkyKey> reverseDeps = errorEntry.isDone() + ? errorEntry.getReverseDeps() + : errorEntry.getInProgressReverseDeps(); + // We should break from loop only when node doesn't have any parents. + if (Iterables.isEmpty(reverseDeps)) { + Preconditions.checkState(rootValues.contains(errorKey), + "Current key %s has to be a top-level key: %s", errorKey, rootValues); + break; + } + SkyKey parent = null; + NodeEntry parentEntry = null; + for (SkyKey bubbleParent : reverseDeps) { + if (bubbleErrorInfo.containsKey(bubbleParent)) { + // We are in a cycle. Don't try to bubble anything up -- cycle detection will kick in. + return null; + } + NodeEntry bubbleParentEntry = Preconditions.checkNotNull(graph.get(bubbleParent), + "parent %s of %s not in graph", bubbleParent, errorKey); + // Might be the parent that requested the error. + if (bubbleParentEntry.isDone()) { + // This parent is cached from a previous evaluate call. We shouldn't bubble up to it + // since any error message produced won't be meaningful to this evaluate call. + // The child error must also be cached from a previous build. + Preconditions.checkState(errorEntry.isDone(), "%s %s", errorEntry, bubbleParentEntry); + Version parentVersion = bubbleParentEntry.getVersion(); + Version childVersion = errorEntry.getVersion(); + Preconditions.checkState(childVersion.atMost(graphVersion) + && !childVersion.equals(graphVersion), + "child entry is not older than the current graph version, but had a done parent. " + + "child: %s childEntry: %s, childVersion: %s" + + "bubbleParent: %s bubbleParentEntry: %s, parentVersion: %s, graphVersion: %s", + errorKey, errorEntry, childVersion, + bubbleParent, bubbleParentEntry, parentVersion, graphVersion); + Preconditions.checkState(parentVersion.atMost(graphVersion) + && !parentVersion.equals(graphVersion), + "parent entry is not older than the current graph version. " + + "child: %s childEntry: %s, childVersion: %s" + + "bubbleParent: %s bubbleParentEntry: %s, parentVersion: %s, graphVersion: %s", + errorKey, errorEntry, childVersion, + bubbleParent, bubbleParentEntry, parentVersion, graphVersion); + continue; + } + // Arbitrarily pick the first in-flight parent. + Preconditions.checkState(visitor.isInflight(bubbleParent), + "errorKey: %s, errorEntry: %s, bubbleParent: %s, bubbleParentEntry: %s", errorKey, + errorEntry, bubbleParent, bubbleParentEntry); + parent = bubbleParent; + parentEntry = bubbleParentEntry; + break; + } + Preconditions.checkNotNull(parent, "", errorKey, bubbleErrorInfo); + errorKey = parent; + SkyFunction factory = skyFunctions.get(parent.functionName()); + if (parentEntry.isDirty()) { + switch (parentEntry.getDirtyState()) { + case CHECK_DEPENDENCIES: + // If this value's child was bubbled up to, it did not signal this value, and so we must + // manually make it ready to build. + parentEntry.signalDep(); + // Fall through to REBUILDING, since state is now REBUILDING. + case REBUILDING: + // Nothing to be done. + break; + default: + throw new AssertionError(parent + " not in valid dirty state: " + parentEntry); + } + } + SkyFunctionEnvironment env = + new SkyFunctionEnvironment(parent, parentEntry.getTemporaryDirectDeps(), + bubbleErrorInfo, visitor); + externalInterrupt = externalInterrupt || Thread.currentThread().isInterrupted(); + try { + // This build is only to check if the parent node can give us a better error. We don't + // care about a return value. + factory.compute(parent, env); + } catch (SkyFunctionException builderException) { + ReifiedSkyFunctionException reifiedBuilderException = + new ReifiedSkyFunctionException(builderException, parent); + if (reifiedBuilderException.getRootCauseSkyKey().equals(parent)) { + error = new ErrorInfo(reifiedBuilderException); + bubbleErrorInfo.put(errorKey, + ValueWithMetadata.error(new ErrorInfo(errorKey, ImmutableSet.of(error)), + env.buildEvents(/*missingChildren=*/true))); + continue; + } + } catch (InterruptedException interruptedException) { + // Do nothing. + // This throw happens if the builder requested the failed node, and then checked the + // interrupted state later -- getValueOrThrow sets the interrupted bit after the failed + // value is requested, to prevent the builder from doing too much work. + } finally { + // Clear interrupted status. We're not listening to interrupts here. + Thread.interrupted(); + } + // Builder didn't throw an exception, so just propagate this one up. + bubbleErrorInfo.put(errorKey, + ValueWithMetadata.error(new ErrorInfo(errorKey, ImmutableSet.of(error)), + env.buildEvents(/*missingChildren=*/true))); + } + + // Reset the interrupt bit if there was an interrupt from outside this evaluator interrupt. + // Note that there are internal interrupts set in the node builder environment if an error + // bubbling node calls getValueOrThrow() on a node in error. + if (externalInterrupt) { + Thread.currentThread().interrupt(); + } + return bubbleErrorInfo; + } + + /** + * Constructs an {@link EvaluationResult} from the {@link #graph}. Looks for cycles if there + * are unfinished nodes but no error was already found through bubbling up + * (as indicated by {@code bubbleErrorInfo} being null). + * + * <p>{@code visitor} may be null, but only in the case where all graph entries corresponding to + * {@code skyKeys} are known to be in the DONE state ({@code entry.isDone()} returns true). + */ + private <T extends SkyValue> EvaluationResult<T> constructResult( + @Nullable ValueVisitor visitor, Iterable<SkyKey> skyKeys, + Map<SkyKey, ValueWithMetadata> bubbleErrorInfo, boolean catastrophe) { + Preconditions.checkState(!keepGoing || catastrophe || bubbleErrorInfo == null, + "", skyKeys, bubbleErrorInfo); + EvaluationResult.Builder<T> result = EvaluationResult.builder(); + List<SkyKey> cycleRoots = new ArrayList<>(); + boolean hasError = false; + for (SkyKey skyKey : skyKeys) { + ValueWithMetadata valueWithMetadata = getValueMaybeFromError(skyKey, bubbleErrorInfo); + // Cycle checking: if there is a cycle, evaluation cannot progress, therefore, + // the final values will not be in DONE state when the work runs out. + if (valueWithMetadata == null) { + // Don't look for cycles if the build failed for a known reason. + if (bubbleErrorInfo == null) { + cycleRoots.add(skyKey); + } + hasError = true; + continue; + } + SkyValue value = valueWithMetadata.getValue(); + // TODO(bazel-team): Verify that message replay is fast and works in failure + // modes [skyframe-core] + // Note that replaying events here is only necessary on null builds, because otherwise we + // would have already printed the transitive messages after building these values. + replayingNestedSetEventVisitor.visit(valueWithMetadata.getTransitiveEvents()); + ErrorInfo errorInfo = valueWithMetadata.getErrorInfo(); + Preconditions.checkState(value != null || errorInfo != null, skyKey); + hasError = hasError || (errorInfo != null); + if (!keepGoing && errorInfo != null) { + // value will be null here unless the value was already built on a prior keepGoing build. + result.addError(skyKey, errorInfo); + continue; + } + if (value == null) { + // Note that we must be in the keepGoing case. Only make this value an error if it doesn't + // have a value. The error shouldn't matter to the caller since the value succeeded after a + // fashion. + result.addError(skyKey, errorInfo); + } else { + result.addResult(skyKey, value); + } + } + if (!cycleRoots.isEmpty()) { + Preconditions.checkState(visitor != null, skyKeys); + checkForCycles(cycleRoots, result, visitor, keepGoing); + } + Preconditions.checkState(bubbleErrorInfo == null || hasError, + "If an error bubbled up, some top-level node must be in error", bubbleErrorInfo, skyKeys); + result.setHasError(hasError); + return result.build(); + } + + private <T extends SkyValue> void checkForCycles( + Iterable<SkyKey> badRoots, EvaluationResult.Builder<T> result, final ValueVisitor visitor, + boolean keepGoing) { + for (SkyKey root : badRoots) { + ErrorInfo errorInfo = checkForCycles(root, visitor, keepGoing); + if (errorInfo == null) { + // This node just wasn't finished when evaluation aborted -- there were no cycles below it. + Preconditions.checkState(!keepGoing, "", root, badRoots); + continue; + } + Preconditions.checkState(!Iterables.isEmpty(errorInfo.getCycleInfo()), + "%s was not evaluated, but was not part of a cycle", root); + result.addError(root, errorInfo); + if (!keepGoing) { + return; + } + } + } + + /** + * Marker value that we push onto a stack before we push a node's children on. When the marker + * value is popped, we know that all the children are finished. We would use null instead, but + * ArrayDeque does not permit null elements. + */ + private static final SkyKey CHILDREN_FINISHED = + new SkyKey(new SkyFunctionName("MARKER", false), "MARKER"); + + /** The max number of cycles we will report to the user for a given root, to avoid OOMing. */ + private static final int MAX_CYCLES = 20; + + /** + * The algorithm for this cycle detector is as follows. We visit the graph depth-first, keeping + * track of the path we are currently on. We skip any DONE nodes (they are transitively + * error-free). If we come to a node already on the path, we immediately construct a cycle. If + * we are in the noKeepGoing case, we return ErrorInfo with that cycle to the caller. Otherwise, + * we continue. Once all of a node's children are done, we construct an error value for it, based + * on those children. Finally, when the original root's node is constructed, we return its + * ErrorInfo. + */ + private ErrorInfo checkForCycles(SkyKey root, ValueVisitor visitor, boolean keepGoing) { + // The number of cycles found. Do not keep on searching for more cycles after this many were + // found. + int cyclesFound = 0; + // The path through the graph currently being visited. + List<SkyKey> graphPath = new ArrayList<>(); + // Set of nodes on the path, to avoid expensive searches through the path for cycles. + Set<SkyKey> pathSet = new HashSet<>(); + + // Maintain a stack explicitly instead of recursion to avoid stack overflows + // on extreme graphs (with long dependency chains). + Deque<SkyKey> toVisit = new ArrayDeque<>(); + + toVisit.push(root); + + // The procedure for this check is as follows: we visit a node, push it onto the graph stack, + // push a marker value onto the toVisit stack, and then push all of its children onto the + // toVisit stack. Thus, when the marker node comes to the top of the toVisit stack, we have + // visited the downward transitive closure of the value. At that point, all of its children must + // be finished, and so we can build the definitive error info for the node, popping it off the + // graph stack. + while (!toVisit.isEmpty()) { + SkyKey key = toVisit.pop(); + NodeEntry entry = graph.get(key); + + if (key == CHILDREN_FINISHED) { + // A marker node means we are done with all children of a node. Since all nodes have + // errors, we must have found errors in the children when that happens. + key = graphPath.remove(graphPath.size() - 1); + entry = graph.get(key); + pathSet.remove(key); + // Skip this node if it was first/last node of a cycle, and so has already been processed. + if (entry.isDone()) { + continue; + } + if (!keepGoing) { + // in the --nokeep_going mode, we would have already returned if we'd found a cycle below + // this node. The fact that we haven't means that there were no cycles below this node + // -- it just hadn't finished evaluating. So skip it. + continue; + } + if (cyclesFound < MAX_CYCLES) { + // Value must be ready, because all of its children have finished, so we can build its + // error. + Preconditions.checkState(entry.isReady(), "%s not ready. ValueEntry: %s", key, entry); + } else if (!entry.isReady()) { + removeIncompleteChildrenForCycle(key, entry, entry.getTemporaryDirectDeps()); + } + Set<SkyKey> directDeps = entry.getTemporaryDirectDeps(); + // Find out which children have errors. Similar logic to that in Evaluate#run(). + List<ErrorInfo> errorDeps = getChildrenErrorsForCycle(directDeps); + Preconditions.checkState(!errorDeps.isEmpty(), + "Value %s was not successfully evaluated, but had no child errors. ValueEntry: %s", key, + entry); + SkyFunctionEnvironment env = new SkyFunctionEnvironment(key, directDeps, visitor); + env.setError(new ErrorInfo(key, errorDeps)); + env.commit(/*enqueueParents=*/false); + } + + // Nothing to be done for this node if it already has an entry. + if (entry.isDone()) { + continue; + } + if (cyclesFound == MAX_CYCLES) { + // Do not keep on searching for cycles indefinitely, to avoid excessive runtime/OOMs. + continue; + } + + if (pathSet.contains(key)) { + int cycleStart = graphPath.indexOf(key); + // Found a cycle! + cyclesFound++; + Iterable<SkyKey> cycle = graphPath.subList(cycleStart, graphPath.size()); + // Put this node into a consistent state for building if it is dirty. + if (entry.isDirty() && entry.getDirtyState() == DirtyState.CHECK_DEPENDENCIES) { + // In the check deps state, entry has exactly one child not done yet. Note that this node + // must be part of the path to the cycle we have found (since done nodes cannot be in + // cycles, and this is the only missing one). Thus, it will not be removed below in + // removeDescendantsOfCycleValue, so it is safe here to signal that it is done. + entry.signalDep(); + } + if (keepGoing) { + // Any children of this node that we haven't already visited are not worth visiting, + // since this node is about to be done. Thus, the only child worth visiting is the one in + // this cycle, the cycleChild (which may == key if this cycle is a self-edge). + SkyKey cycleChild = selectCycleChild(key, graphPath, cycleStart); + removeDescendantsOfCycleValue(key, entry, cycleChild, toVisit, + graphPath.size() - cycleStart); + ValueWithMetadata dummyValue = ValueWithMetadata.wrapWithMetadata(new SkyValue() {}); + + + SkyFunctionEnvironment env = + new SkyFunctionEnvironment(key, entry.getTemporaryDirectDeps(), + ImmutableMap.of(cycleChild, dummyValue), visitor); + + // Construct error info for this node. Get errors from children, which are all done + // except possibly for the cycleChild. + List<ErrorInfo> allErrors = + getChildrenErrors(entry.getTemporaryDirectDeps(), /*unfinishedChild=*/cycleChild); + CycleInfo cycleInfo = new CycleInfo(cycle); + // Add in this cycle. + allErrors.add(new ErrorInfo(cycleInfo)); + env.setError(new ErrorInfo(key, allErrors)); + env.commit(/*enqueueParents=*/false); + continue; + } else { + // We need to return right away in the noKeepGoing case, so construct the cycle (with the + // path) and return. + Preconditions.checkState(graphPath.get(0).equals(root), + "%s not reached from %s. ValueEntry: %s", key, root, entry); + return new ErrorInfo(new CycleInfo(graphPath.subList(0, cycleStart), cycle)); + } + } + + // This node is not yet known to be in a cycle. So process its children. + Iterable<? extends SkyKey> children = graph.get(key).getTemporaryDirectDeps(); + if (Iterables.isEmpty(children)) { + continue; + } + + // This marker flag will tell us when all this node's children have been processed. + toVisit.push(CHILDREN_FINISHED); + // This node is now part of the path through the graph. + graphPath.add(key); + pathSet.add(key); + for (SkyKey nextValue : children) { + toVisit.push(nextValue); + } + } + return keepGoing ? getAndCheckDone(root).getErrorInfo() : null; + } + + /** + * Returns the child of this node that is in the cycle that was just found. If the cycle is a + * self-edge, returns the node itself. + */ + private static SkyKey selectCycleChild(SkyKey key, List<SkyKey> graphPath, int cycleStart) { + return cycleStart + 1 == graphPath.size() ? key : graphPath.get(cycleStart + 1); + } + + /** + * Get all the errors of child nodes. There must be at least one cycle amongst them. + * + * @param children child nodes to query for errors. + * @return List of ErrorInfos from all children that had errors. + */ + private List<ErrorInfo> getChildrenErrorsForCycle(Iterable<SkyKey> children) { + List<ErrorInfo> allErrors = new ArrayList<>(); + boolean foundCycle = false; + for (SkyKey child : children) { + ErrorInfo errorInfo = getAndCheckDone(child).getErrorInfo(); + if (errorInfo != null) { + foundCycle |= !Iterables.isEmpty(errorInfo.getCycleInfo()); + allErrors.add(errorInfo); + } + } + Preconditions.checkState(foundCycle, "", children, allErrors); + return allErrors; + } + + /** + * Get all the errors of child nodes. + * + * @param children child nodes to query for errors. + * @param unfinishedChild child which is allowed to not be done. + * @return List of ErrorInfos from all children that had errors. + */ + private List<ErrorInfo> getChildrenErrors(Iterable<SkyKey> children, SkyKey unfinishedChild) { + List<ErrorInfo> allErrors = new ArrayList<>(); + for (SkyKey child : children) { + ErrorInfo errorInfo = getErrorMaybe(child, /*allowUnfinished=*/child.equals(unfinishedChild)); + if (errorInfo != null) { + allErrors.add(errorInfo); + } + } + return allErrors; + } + + @Nullable + private ErrorInfo getErrorMaybe(SkyKey key, boolean allowUnfinished) { + if (!allowUnfinished) { + return getAndCheckDone(key).getErrorInfo(); + } + NodeEntry entry = Preconditions.checkNotNull(graph.get(key), key); + return entry.isDone() ? entry.getErrorInfo() : null; + } + + /** + * Removes direct children of key from toVisit and from the entry itself, and makes the entry + * ready if necessary. We must do this because it would not make sense to try to build the + * children after building the entry. It would violate the invariant that a parent can only be + * built after its children are built; See bug "Precondition error while evaluating a Skyframe + * graph with a cycle". + * + * @param key SkyKey of node in a cycle. + * @param entry NodeEntry of node in a cycle. + * @param cycleChild direct child of key in the cycle, or key itself if the cycle is a self-edge. + * @param toVisit list of remaining nodes to visit by the cycle-checker. + * @param cycleLength the length of the cycle found. + */ + private void removeDescendantsOfCycleValue(SkyKey key, NodeEntry entry, + @Nullable SkyKey cycleChild, Iterable<SkyKey> toVisit, int cycleLength) { + Set<SkyKey> unvisitedDeps = new HashSet<>(entry.getTemporaryDirectDeps()); + unvisitedDeps.remove(cycleChild); + // Remove any children from this node that are not part of the cycle we just found. They are + // irrelevant to the node as it stands, and if they are deleted from the graph because they are + // not built by the end of cycle-checking, we would have dangling references. + removeIncompleteChildrenForCycle(key, entry, unvisitedDeps); + if (!entry.isReady()) { + // The entry has at most one undone dep now, its cycleChild. Signal to make entry ready. Note + // that the entry can conceivably be ready if its cycleChild already found a different cycle + // and was built. + entry.signalDep(); + } + Preconditions.checkState(entry.isReady(), "%s %s %s", key, cycleChild, entry); + Iterator<SkyKey> it = toVisit.iterator(); + while (it.hasNext()) { + SkyKey descendant = it.next(); + if (descendant == CHILDREN_FINISHED) { + // Marker value, delineating the end of a group of children that were enqueued. + cycleLength--; + if (cycleLength == 0) { + // We have seen #cycleLength-1 marker values, and have arrived at the one for this value, + // so we are done. + return; + } + continue; // Don't remove marker values. + } + if (cycleLength == 1) { + // Remove the direct children remaining to visit of the cycle node. + Preconditions.checkState(unvisitedDeps.contains(descendant), + "%s %s %s %s %s", key, descendant, cycleChild, unvisitedDeps, entry); + it.remove(); + } + } + throw new IllegalStateException("There were not " + cycleLength + " groups of children in " + + toVisit + " when trying to remove children of " + key + " other than " + cycleChild); + } + + private void removeIncompleteChildrenForCycle(SkyKey key, NodeEntry entry, + Iterable<SkyKey> children) { + Set<SkyKey> unfinishedDeps = new HashSet<>(); + for (SkyKey child : children) { + if (removeIncompleteChild(key, child)) { + unfinishedDeps.add(child); + } + } + entry.removeUnfinishedDeps(unfinishedDeps); + } + + private NodeEntry getAndCheckDone(SkyKey key) { + NodeEntry entry = graph.get(key); + Preconditions.checkNotNull(entry, key); + Preconditions.checkState(entry.isDone(), "%s %s", key, entry); + return entry; + } + + private ValueWithMetadata getValueMaybeFromError(SkyKey key, + @Nullable Map<SkyKey, ValueWithMetadata> bubbleErrorInfo) { + SkyValue value = bubbleErrorInfo == null ? null : bubbleErrorInfo.get(key); + NodeEntry entry = graph.get(key); + if (value != null) { + Preconditions.checkNotNull(entry, + "Value cannot have error before evaluation started", key, value); + return ValueWithMetadata.wrapWithMetadata(value); + } + return isDoneForBuild(entry) ? entry.getValueWithMetadata() : null; + } + + /** + * Return true if the entry does not need to be re-evaluated this build. The entry will need to + * be re-evaluated if it is not done, but also if it was not completely evaluated last build and + * this build is keepGoing. + */ + private boolean isDoneForBuild(@Nullable NodeEntry entry) { + return entry != null && entry.isDone(); + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ProcessableGraph.java b/src/main/java/com/google/devtools/build/skyframe/ProcessableGraph.java new file mode 100644 index 0000000..8bf8a38 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/ProcessableGraph.java
@@ -0,0 +1,24 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +/** + * A graph that is both Dirtiable (values can be deleted) and Evaluable (values can be added). All + * methods in this interface (as inherited from super-interfaces) should be thread-safe. + * + * <p>This class is not intended for direct use, and is only exposed as public for use in + * evaluation implementations outside of this package. + */ +public interface ProcessableGraph extends DirtiableGraph, EvaluableGraph { +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/QueryableGraph.java b/src/main/java/com/google/devtools/build/skyframe/QueryableGraph.java new file mode 100644 index 0000000..e1cfc0a --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/QueryableGraph.java
@@ -0,0 +1,24 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +/** + * A graph that exposes its entries and structure, for use by classes that must traverse it. + */ +public interface QueryableGraph { + /** + * Returns the node with the given name, or {@code null} if the node does not exist. + */ + NodeEntry get(SkyKey key); +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java b/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java new file mode 100644 index 0000000..3ebbf33 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java
@@ -0,0 +1,76 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.concurrent.ThreadSafety; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * A simple Differencer which just records the invalidated values it's been given. + */ +@ThreadSafety.ThreadCompatible +public class RecordingDifferencer implements Differencer, Injectable { + + private List<SkyKey> valuesToInvalidate; + private Map<SkyKey, SkyValue> valuesToInject; + + public RecordingDifferencer() { + clear(); + } + + private void clear() { + valuesToInvalidate = new ArrayList<>(); + valuesToInject = new HashMap<>(); + } + + @Override + public Diff getDiff(Version fromVersion, Version toVersion) { + Diff diff = new ImmutableDiff(valuesToInvalidate, valuesToInject); + clear(); + return diff; + } + + /** + * Store the given values for invalidation. + */ + public void invalidate(Iterable<SkyKey> values) { + Iterables.addAll(valuesToInvalidate, values); + } + + /** + * Invalidates the cached values of any values in error transiently. + * + * <p>If a future call to {@link MemoizingEvaluator#evaluate} requests a value that transitively + * depends on any value that was in an error state (or is one of these), they will be re-computed. + */ + public void invalidateTransientErrors() { + // All transient error values have a dependency on the single global ERROR_TRANSIENCE value, + // so we only have to invalidate that one value to catch everything. + invalidate(ImmutableList.of(ErrorTransienceValue.key())); + } + + /** + * Store the given values for injection. + */ + @Override + public void inject(Map<SkyKey, ? extends SkyValue> values) { + valuesToInject.putAll(values); + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ReverseDepsUtil.java b/src/main/java/com/google/devtools/build/skyframe/ReverseDepsUtil.java new file mode 100644 index 0000000..13d8c4b --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/ReverseDepsUtil.java
@@ -0,0 +1,211 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +import java.util.Collection; +import java.util.List; +import java.util.Set; + +/** + * A utility class that allows us to keep the reverse dependencies as an array list instead of a + * set. This is more memory-efficient. At the same time it allows us to group the removals and + * uniqueness checks so that it also performs well. + * + * <p>The reason of this class it to share non-trivial code between BuildingState and NodeEntry. We + * could simply make those two classes extend this class instead, but we would be less + * memory-efficient since object memory alignment does not cross classes ( you would have two memory + * alignments, one for the base class and one for the extended one). + */ +abstract class ReverseDepsUtil<T> { + + static final int MAYBE_CHECK_THRESHOLD = 10; + + abstract void setReverseDepsObject(T container, Object object); + + abstract void setSingleReverseDep(T container, boolean singleObject); + + abstract void setReverseDepsToRemove(T container, List<SkyKey> object); + + abstract Object getReverseDepsObject(T container); + + abstract boolean isSingleReverseDep(T container); + + abstract List<SkyKey> getReverseDepsToRemove(T container); + + /** + * We check that the reverse dependency is not already present. We only do that if reverseDeps is + * small, so that it does not impact performance. + */ + void maybeCheckReverseDepNotPresent(T container, SkyKey reverseDep) { + if (isSingleReverseDep(container)) { + Preconditions.checkState(!getReverseDepsObject(container).equals(reverseDep), + "Reverse dep %s already present", reverseDep); + return; + } + @SuppressWarnings("unchecked") + List<SkyKey> asList = (List<SkyKey>) getReverseDepsObject(container); + if (asList.size() < MAYBE_CHECK_THRESHOLD) { + Preconditions.checkState(!asList.contains(reverseDep), "Reverse dep %s already present" + + " in %s", reverseDep, asList); + } + } + + /** + * We use a memory-efficient trick to keep reverseDeps memory usage low. Edges in Bazel are + * dominant over the number of nodes. + * + * <p>Most of the nodes have zero or one reverse dep. That is why we use immutable versions of the + * lists for those cases. In case of the size being > 1 we switch to an ArrayList. That is because + * we also have a decent number of nodes for which the reverseDeps are huge (for example almost + * everything depends on BuildInfo node). + * + * <p>We also optimize for the case where we have only one dependency. In that case we keep the + * object directly instead of a wrapper list. + */ + @SuppressWarnings("unchecked") + void addReverseDeps(T container, Collection<SkyKey> newReverseDeps) { + if (newReverseDeps.isEmpty()) { + return; + } + Object reverseDeps = getReverseDepsObject(container); + int reverseDepsSize = isSingleReverseDep(container) ? 1 : ((List<SkyKey>) reverseDeps).size(); + int newSize = reverseDepsSize + newReverseDeps.size(); + if (newSize == 1) { + overwriteReverseDepsWithObject(container, Iterables.getOnlyElement(newReverseDeps)); + } else if (reverseDepsSize == 0) { + overwriteReverseDepsList(container, Lists.newArrayList(newReverseDeps)); + } else if (reverseDepsSize == 1) { + List<SkyKey> newList = Lists.newArrayListWithExpectedSize(newSize); + newList.add((SkyKey) reverseDeps); + newList.addAll(newReverseDeps); + overwriteReverseDepsList(container, newList); + } else { + ((List<SkyKey>) reverseDeps).addAll(newReverseDeps); + } + } + + /** + * See {@code addReverseDeps} method. + */ + void removeReverseDep(T container, SkyKey reverseDep) { + if (isSingleReverseDep(container)) { + // This removal is cheap so let's do it and not keep it in reverseDepsToRemove. + // !equals should only happen in case of catastrophe. + if (getReverseDepsObject(container).equals(reverseDep)) { + overwriteReverseDepsList(container, ImmutableList.<SkyKey>of()); + } + return; + } + @SuppressWarnings("unchecked") + List<SkyKey> reverseDepsAsList = (List<SkyKey>) getReverseDepsObject(container); + if (reverseDepsAsList.isEmpty()) { + return; + } + List<SkyKey> reverseDepsToRemove = getReverseDepsToRemove(container); + if (reverseDepsToRemove == null) { + reverseDepsToRemove = Lists.newArrayListWithExpectedSize(1); + setReverseDepsToRemove(container, reverseDepsToRemove); + } + reverseDepsToRemove.add(reverseDep); + } + + ImmutableSet<SkyKey> getReverseDeps(T container) { + consolidateReverseDepsRemovals(container); + + // TODO(bazel-team): Unfortunately, we need to make a copy here right now to be on the safe side + // wrt. thread-safety. The parents of a node get modified when any of the parents is deleted, + // and we can't handle that right now. + if (isSingleReverseDep(container)) { + return ImmutableSet.of((SkyKey) getReverseDepsObject(container)); + } else { + @SuppressWarnings("unchecked") + List<SkyKey> reverseDeps = (List<SkyKey>) getReverseDepsObject(container); + ImmutableSet<SkyKey> set = ImmutableSet.copyOf(reverseDeps); + Preconditions.checkState(set.size() == reverseDeps.size(), + "Duplicate reverse deps present in %s: %s. %s", this, reverseDeps, container); + return set; + } + } + + void consolidateReverseDepsRemovals(T container) { + List<SkyKey> reverseDepsToRemove = getReverseDepsToRemove(container); + Object reverseDeps = getReverseDepsObject(container); + if (reverseDepsToRemove == null) { + return; + } + Preconditions.checkState(!isSingleReverseDep(container), + "We do not use reverseDepsToRemove for single lists: %s", container); + // Should not happen, as we only create reverseDepsToRemove in case we have at least one + // reverse dep to remove. + Preconditions.checkState((!((List<?>) reverseDeps).isEmpty()), + "Could not remove %s elements from %s.\nReverse deps to remove: %s. %s", + reverseDepsToRemove.size(), reverseDeps, reverseDepsToRemove, container); + + Set<SkyKey> toRemove = Sets.newHashSet(reverseDepsToRemove); + int expectedRemovals = toRemove.size(); + Preconditions.checkState(expectedRemovals == reverseDepsToRemove.size(), + "A reverse dependency tried to remove itself twice: %s. %s", reverseDepsToRemove, + container); + + @SuppressWarnings("unchecked") + List<SkyKey> reverseDepsAsList = (List<SkyKey>) reverseDeps; + List<SkyKey> newReverseDeps = Lists + .newArrayListWithExpectedSize(Math.max(0, reverseDepsAsList.size() - expectedRemovals)); + + for (SkyKey reverseDep : reverseDepsAsList) { + if (!toRemove.contains(reverseDep)) { + newReverseDeps.add(reverseDep); + } + } + Preconditions.checkState(newReverseDeps.size() == reverseDepsAsList.size() - expectedRemovals, + "Could not remove some elements from %s.\nReverse deps to remove: %s. %s", reverseDeps, + toRemove, container); + + if (newReverseDeps.isEmpty()) { + overwriteReverseDepsList(container, ImmutableList.<SkyKey>of()); + } else if (newReverseDeps.size() == 1) { + overwriteReverseDepsWithObject(container, newReverseDeps.get(0)); + } else { + overwriteReverseDepsList(container, newReverseDeps); + } + setReverseDepsToRemove(container, null); + } + + @SuppressWarnings("deprecation") + String toString(T container) { + return Objects.toStringHelper("ReverseDeps") // MoreObjects is not in Guava + .add("reverseDeps", getReverseDepsObject(container)) + .add("singleReverseDep", isSingleReverseDep(container)) + .add("reverseDepsToRemove", getReverseDepsToRemove(container)) + .toString(); + } + + private void overwriteReverseDepsWithObject(T container, SkyKey newObject) { + setReverseDepsObject(container, newObject); + setSingleReverseDep(container, true); + } + + private void overwriteReverseDepsList(T container, List<SkyKey> list) { + setReverseDepsObject(container, list); + setSingleReverseDep(container, false); + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/Scheduler.java b/src/main/java/com/google/devtools/build/skyframe/Scheduler.java new file mode 100644 index 0000000..f05860f --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/Scheduler.java
@@ -0,0 +1,78 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Preconditions; + +import javax.annotation.Nullable; + +/** + * A work queue -- takes {@link Runnable}s and runs them when requested. + */ +interface Scheduler { + /** + * Schedules a new action to be eventually done. + */ + void schedule(Runnable action); + + /** + * Runs the actions that have been scheduled. These actions can in turn schedule new actions, + * which will be run as well. + * + * @throw SchedulerException wrapping a scheduled action's exception. + */ + void run() throws SchedulerException; + + /** + * Wrapper exception that {@link Runnable}s can throw, to be caught and handled + * by callers of {@link #run}. + */ + static class SchedulerException extends RuntimeException { + private final SkyKey failedValue; + private final ErrorInfo errorInfo; + + private SchedulerException(@Nullable Throwable cause, @Nullable ErrorInfo errorInfo, + SkyKey failedValue) { + super(errorInfo != null ? errorInfo.getException() : cause); + this.errorInfo = errorInfo; + this.failedValue = Preconditions.checkNotNull(failedValue, errorInfo); + } + + /** + * Returns a SchedulerException wrapping an expected error, e.g. an error describing an expected + * build failure when trying to evaluate the given value, that should cause Skyframe to produce + * useful error information to the user. + */ + static SchedulerException ofError(ErrorInfo errorInfo, SkyKey failedValue) { + Preconditions.checkNotNull(errorInfo); + return new SchedulerException(errorInfo.getException(), errorInfo, failedValue); + } + + /** + * Returns a SchedulerException wrapping an InterruptedException, e.g. if the user interrupts + * the build, that should cause Skyframe to exit as soon as possible. + */ + static SchedulerException ofInterruption(InterruptedException cause, SkyKey failedValue) { + return new SchedulerException(cause, null, failedValue); + } + + SkyKey getFailedValue() { + return failedValue; + } + + @Nullable ErrorInfo getErrorInfo() { + return errorInfo; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/SequentialBuildDriver.java b/src/main/java/com/google/devtools/build/skyframe/SequentialBuildDriver.java new file mode 100644 index 0000000..9b7f036 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/SequentialBuildDriver.java
@@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.events.EventHandler; + +/** + * A driver for auto-updating graphs which operate over monotonically increasing integer versions. + */ +public class SequentialBuildDriver implements BuildDriver { + private final MemoizingEvaluator memoizingEvaluator; + private IntVersion curVersion; + + public SequentialBuildDriver(MemoizingEvaluator evaluator) { + this.memoizingEvaluator = Preconditions.checkNotNull(evaluator); + this.curVersion = new IntVersion(0); + } + + @Override + public <T extends SkyValue> EvaluationResult<T> evaluate( + Iterable<SkyKey> roots, boolean keepGoing, int numThreads, EventHandler reporter) + throws InterruptedException { + try { + return memoizingEvaluator.evaluate(roots, curVersion, keepGoing, numThreads, reporter); + } finally { + curVersion = curVersion.next(); + } + } + + @Override + public MemoizingEvaluator getGraphForTesting() { + return memoizingEvaluator; + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/SkyFunction.java b/src/main/java/com/google/devtools/build/skyframe/SkyFunction.java new file mode 100644 index 0000000..324c03d --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/SkyFunction.java
@@ -0,0 +1,187 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.lib.events.EventHandler; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Machinery to evaluate a single value. + * + * <p>The builder is supposed to access only direct dependencies of the value. However, the direct + * dependencies need not be known in advance. The builder can request arbitrary values using + * {@link Environment#getValue}. If the values are not ready, the call will return null; in that + * case the builder can either try to proceed (and potentially indicate more dependencies by + * additional {@code getValue} calls), or just return null, in which case the missing dependencies + * will be computed and the builder will be started again. + */ +public interface SkyFunction { + + /** + * When a value is requested, this method is called with the name of the value and a value + * building environment. + * + * <p>This method should return a constructed value, or null if any dependencies were missing + * ({@link Environment#valuesMissing} was true before returning). In that case the missing + * dependencies will be computed and the value builder restarted. + * + * <p>Implementations must be threadsafe and reentrant. + * + * @throws SkyFunctionException on failure + * @throws InterruptedException when the user interrupts the build + */ + @Nullable SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, + InterruptedException; + + /** + * Extracts a tag (target label) from a SkyKey if it has one. Otherwise return null. + * + * <p>The tag is used for filtering out non-error event messages that do not match --output_filter + * flag. If a SkyFunction returns null in this method it means that all the info/warning messages + * associated with this value will be shown, no matter what --output_filter says. + */ + @Nullable + String extractTag(SkyKey skyKey); + + /** + * The services provided to the value builder by the graph implementation. + */ + interface Environment { + /** + * Returns a direct dependency. If the specified value is not in the set of already evaluated + * direct dependencies, returns null. Also returns null if the specified value has already been + * evaluated and found to be in error. + * + * <p>On a subsequent build, if any of this value's dependencies have changed they will be + * re-evaluated in the same order as originally requested by the {@code SkyFunction} using + * this {@code getValue} call (see {@link #getValues} for when preserving the order is not + * important). + */ + @Nullable + SkyValue getValue(SkyKey valueName); + + /** + * Returns a direct dependency. If the specified value is not in the set of already evaluated + * direct dependencies, returns null. If the specified value has already been evaluated and + * found to be in error, throws the exception coming from the error. Value builders may + * use this method to continue evaluation even if one of their children is in error by catching + * the thrown exception and proceeding. The caller must specify the exception that might be + * thrown using the {@code exceptionClass} argument. If the child's exception is not an instance + * of {@code exceptionClass}, returns null without throwing. + * + * <p>The exception class given cannot be a supertype or a subtype of {@link RuntimeException}, + * or a subtype of {@link InterruptedException}. See + * {@link SkyFunctionException#validateExceptionType} for details. + */ + @Nullable + <E extends Exception> SkyValue getValueOrThrow(SkyKey depKey, Class<E> exceptionClass) throws E; + @Nullable + <E1 extends Exception, E2 extends Exception> SkyValue getValueOrThrow(SkyKey depKey, + Class<E1> exceptionClass1, Class<E2> exceptionClass2) throws E1, E2; + @Nullable + <E1 extends Exception, E2 extends Exception, E3 extends Exception> SkyValue getValueOrThrow( + SkyKey depKey, Class<E1> exceptionClass1, Class<E2> exceptionClass2, + Class<E3> exceptionClass3) throws E1, E2, E3; + @Nullable + <E1 extends Exception, E2 extends Exception, E3 extends Exception, E4 extends Exception> + SkyValue getValueOrThrow(SkyKey depKey, Class<E1> exceptionClass1, + Class<E2> exceptionClass2, Class<E3> exceptionClass3, Class<E4> exceptionClass4) + throws E1, E2, E3, E4; + + /** + * Returns true iff any of the past {@link #getValue}(s) or {@link #getValueOrThrow} method + * calls for this instance returned null (because the value was not yet present and done in the + * graph). + * + * <p>If this returns true, the {@link SkyFunction} must return {@code null}. + */ + boolean valuesMissing(); + + /** + * Requests {@code depKeys} "in parallel", independent of each others' values. These keys may be + * thought of as a "dependency group" -- they are requested together by this value. + * + * <p>In general, if the result of one getValue call can affect the argument of a later getValue + * call, the two calls cannot be merged into a single getValues call, since the result of the + * first call might change on a later build. Inversely, if the result of one getValue call + * cannot affect the parameters of the next getValue call, the two keys can form a dependency + * group and the two getValue calls merged into one getValues call. + * + * <p>This means that on subsequent builds, when checking to see if a value requires rebuilding, + * all the values in this group may be simultaneously checked. A SkyFunction should request a + * dependency group if checking the deps serially on a subsequent build would take too long, and + * if the builder would request all deps anyway as long as no earlier deps had changed. + * SkyFunction.Environment implementations may also choose to request these deps in + * parallel on the first build, potentially speeding up the build. + * + * <p>While re-evaluating every value in the group may take longer than re-evaluating just the + * first one and finding that it has changed, no extra work is done: the contract of the + * dependency group means that the builder, when called to rebuild this value, will request all + * values in the group again anyway, so they would have to have been built in any case. + * + * <p>Example of when to use getValues: A ListProcessor value is built with key inputListRef. + * The builder first calls getValue(InputList.key(inputListRef)), and retrieves inputList. It + * then iterates through inputList, calling getValue on each input. Finally, it processes the + * whole list and returns. Say inputList is (a, b, c). Since the builder will unconditionally + * call getValue(a), getValue(b), and getValue(c), the builder can instead just call + * getValues({a, b, c}). If the value is later dirtied the evaluator will build a, b, and c in + * parallel (assuming the inputList value was unchanged), and re-evaluate the ListProcessor + * value only if at least one of them was changed. On the other hand, if the InputList changes + * to be (a, b, d), then the evaluator will see that the first dep has changed, and call the + * builder to rebuild from scratch, without considering the dep group of {a, b, c}. + * + * <p>Example of when not to use getValues: A BestMatch value is built with key + * <potentialMatchesRef, matchCriterion>. The builder first calls + * getValue(PotentialMatches.key(potentialMatchesRef) and retrieves potentialMatches. It then + * iterates through potentialMatches, calling getValue on each potential match until it finds + * one that satisfies matchCriterion. In this case, if potentialMatches is (a, b, c), it would + * be <i>incorrect</i> to call getValues({a, b, c}), because it is not known yet whether + * requesting b or c will be necessary -- if a matches, then we will never call b or c. + */ + Map<SkyKey, SkyValue> getValues(Iterable<SkyKey> depKeys); + + /** + * The same as {@link #getValues} but the returned objects may throw when attempting to retrieve + * their value. Note that even if the requested values can throw different kinds of exceptions, + * only exceptions of type {@code E} will be preserved in the returned objects. All others will + * be null. + */ + <E extends Exception> Map<SkyKey, ValueOrException<E>> getValuesOrThrow( + Iterable<SkyKey> depKeys, Class<E> exceptionClass); + <E1 extends Exception, E2 extends Exception> Map<SkyKey, ValueOrException2<E1, E2>> + getValuesOrThrow(Iterable<SkyKey> depKeys, Class<E1> exceptionClass1, + Class<E2> exceptionClass2); + <E1 extends Exception, E2 extends Exception, E3 extends Exception> + Map<SkyKey, ValueOrException3<E1, E2, E3>> getValuesOrThrow(Iterable<SkyKey> depKeys, + Class<E1> exceptionClass1, Class<E2> exceptionClass2, Class<E3> exceptionClass3); + <E1 extends Exception, E2 extends Exception, E3 extends Exception, E4 extends Exception> + Map<SkyKey, ValueOrException4<E1, E2, E3, E4>> getValuesOrThrow(Iterable<SkyKey> depKeys, + Class<E1> exceptionClass1, Class<E2> exceptionClass2, Class<E3> exceptionClass3, + Class<E4> exceptionClass4); + + /** + * Returns the {@link EventHandler} that a SkyFunction should use to print any errors, + * warnings, or progress messages while building. + */ + EventHandler getListener(); + + /** Returns whether we are currently in error bubbling. */ + @VisibleForTesting + boolean inErrorBubblingForTesting(); + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/SkyFunctionException.java b/src/main/java/com/google/devtools/build/skyframe/SkyFunctionException.java new file mode 100644 index 0000000..71b4710 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/SkyFunctionException.java
@@ -0,0 +1,133 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + + +import com.google.common.base.Preconditions; + +import javax.annotation.Nullable; + +/** + * Base class of exceptions thrown by {@link SkyFunction#compute} on failure. + * + * SkyFunctions should declare a subclass {@code C} of {@link SkyFunctionException} whose + * constructors forward fine-grained exception types (e.g. {@link IOException}) to + * {@link SkyFunctionException}'s constructor, and they should also declare + * {@link SkyFunction#compute} to throw {@code C}. This way the type system checks that no + * unexpected exceptions are thrown by the {@link SkyFunction}. + * + * <p>We took this approach over using a generic exception class since Java disallows it because of + * type erasure + * (see http://docs.oracle.com/javase/tutorial/java/generics/restrictions.html#cannotCatch). + * + * <p> Note that there are restrictions on what Exception types are allowed to be wrapped in this + * manner. See {@link SkyFunctionException#validateExceptionType}. + * + * <p>Failures are explicitly either transient or persistent. The transience of the failure from + * {@link SkyFunction#compute} should be influenced only by the computations done, and not by the + * transience of the failures from computations requested via + * {@link SkyFunction.Environment#getValueOrThrow}. + */ +public abstract class SkyFunctionException extends Exception { + + /** The transience of the error. */ + public enum Transience { + // An error that may or may not occur again if the computation were re-run. If a computation + // results in a transient error and is needed on a subsequent MemoizingEvaluator#evaluate call, + // it will be re-executed. + TRANSIENT, + + // An error that is completely deterministic and persistent in terms of the computation's + // inputs. Persistent errors may be cached. + PERSISTENT; + } + + private final Transience transience; + @Nullable + private final SkyKey rootCause; + + public SkyFunctionException(Exception cause, Transience transience) { + this(cause, transience, null); + } + + /** Used to rethrow a child error that the parent cannot handle. */ + public SkyFunctionException(Exception cause, SkyKey childKey) { + this(cause, Transience.PERSISTENT, childKey); + } + + private SkyFunctionException(Exception cause, Transience transience, SkyKey rootCause) { + super(Preconditions.checkNotNull(cause)); + SkyFunctionException.validateExceptionType(cause.getClass()); + this.transience = transience; + this.rootCause = rootCause; + } + + @Nullable + final SkyKey getRootCauseSkyKey() { + return rootCause; + } + + final boolean isTransient() { + return transience == Transience.TRANSIENT; + } + + /** + * Catastrophic failures halt the build even when in keepGoing mode. + */ + public boolean isCatastrophic() { + return false; + } + + @Override + public Exception getCause() { + return (Exception) super.getCause(); + } + + static <E extends Throwable> void validateExceptionType(Class<E> exceptionClass) { + if (exceptionClass.equals(ValueOrExceptionUtils.BottomException.class)) { + return; + } + + if (exceptionClass.isAssignableFrom(RuntimeException.class)) { + throw new IllegalStateException(exceptionClass.getSimpleName() + " is a supertype of " + + "RuntimeException. Don't do this since then you would potentially swallow all " + + "RuntimeExceptions, even those from Skyframe"); + } + if (RuntimeException.class.isAssignableFrom(exceptionClass)) { + throw new IllegalStateException(exceptionClass.getSimpleName() + " is a subtype of " + + "RuntimeException. You should rewrite your code to use checked exceptions."); + } + if (InterruptedException.class.isAssignableFrom(exceptionClass)) { + throw new IllegalStateException(exceptionClass.getSimpleName() + " is a subtype of " + + "InterruptedException. Don't do this; Skyframe handles interrupts separately from the " + + "general SkyFunctionException mechanism."); + } + } + + /** A {@link SkyFunctionException} with a definite root cause. */ + static class ReifiedSkyFunctionException extends SkyFunctionException { + private final boolean isCatastrophic; + + ReifiedSkyFunctionException(SkyFunctionException e, SkyKey key) { + super(e.getCause(), e.transience, Preconditions.checkNotNull(e.getRootCauseSkyKey() == null + ? key : e.getRootCauseSkyKey())); + this.isCatastrophic = e.isCatastrophic(); + } + + @Override + public boolean isCatastrophic() { + return isCatastrophic; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/SkyFunctionName.java b/src/main/java/com/google/devtools/build/skyframe/SkyFunctionName.java new file mode 100644 index 0000000..389d4d8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/SkyFunctionName.java
@@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Predicate; + +import java.io.Serializable; +import java.util.Set; + +/** + * An identifier for a {@code SkyFunction}. + */ +public final class SkyFunctionName implements Serializable { + public static SkyFunctionName computed(String name) { + return new SkyFunctionName(name, true); + } + + private final String name; + private final boolean isComputed; + + public SkyFunctionName(String name, boolean isComputed) { + this.name = name; + this.isComputed = isComputed; + } + + @Override + public String toString() { + return name; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof SkyFunctionName)) { + return false; + } + SkyFunctionName other = (SkyFunctionName) obj; + return name.equals(other.name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + /** + * Returns whether the values of this type are computed. The computation of a computed value must + * be deterministic and may only access requested dependencies. + */ + public boolean isComputed() { + return isComputed; + } + + /** + * A predicate that returns true for {@link SkyKey}s that have the given {@link SkyFunctionName}. + */ + public static Predicate<SkyKey> functionIs(final SkyFunctionName functionName) { + return new Predicate<SkyKey>() { + @Override + public boolean apply(SkyKey skyKey) { + return functionName.equals(skyKey.functionName()); + } + }; + } + + /** + * A predicate that returns true for {@link SkyKey}s that have the given {@link SkyFunctionName}. + */ + public static Predicate<SkyKey> functionIsIn(final Set<SkyFunctionName> functionNames) { + return new Predicate<SkyKey>() { + @Override + public boolean apply(SkyKey skyKey) { + return functionNames.contains(skyKey.functionName()); + } + }; + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/SkyKey.java b/src/main/java/com/google/devtools/build/skyframe/SkyKey.java new file mode 100644 index 0000000..cc1dd1f --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/SkyKey.java
@@ -0,0 +1,86 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; + +import java.io.Serializable; + +/** + * A {@link SkyKey} is effectively a pair (type, name) that identifies a Skyframe value. + */ +public final class SkyKey implements Serializable { + private final SkyFunctionName functionName; + + /** + * The name of the value. + * + * <p>This is deliberately an untyped Object so that we can use arbitrary value types (e.g., + * Labels, PathFragments, BuildConfigurations, etc.) as value names without incurring + * serialization costs in the in-memory implementation of the graph. + */ + private final Object argument; + + /** + * Cache the hash code for this object. It might be expensive to compute. + */ + private final int hashCode; + + public SkyKey(SkyFunctionName functionName, Object valueName) { + this.functionName = Preconditions.checkNotNull(functionName); + this.argument = Preconditions.checkNotNull(valueName); + this.hashCode = 31 * functionName.hashCode() + argument.hashCode(); + } + + public SkyFunctionName functionName() { + return functionName; + } + + public Object argument() { + return argument; + } + + @Override + public String toString() { + return functionName + ":" + argument; + } + + @Override + public int hashCode() { + return hashCode; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + SkyKey other = (SkyKey) obj; + return argument.equals(other.argument) && functionName.equals(other.functionName); + } + + public static final Function<SkyKey, Object> NODE_NAME = new Function<SkyKey, Object>() { + @Override + public Object apply(SkyKey input) { + return input.argument(); + } + }; +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/SkyValue.java b/src/main/java/com/google/devtools/build/skyframe/SkyValue.java new file mode 100644 index 0000000..7cfaa78 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/SkyValue.java
@@ -0,0 +1,22 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import java.io.Serializable; + +/** + * A return value of a {@code SkyFunction}. + */ +public interface SkyValue extends Serializable { +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/TaggedEvents.java b/src/main/java/com/google/devtools/build/skyframe/TaggedEvents.java new file mode 100644 index 0000000..056175e --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/TaggedEvents.java
@@ -0,0 +1,62 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.events.Event; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * A wrapper of {@link Event} that contains a tag of the label where the event was generated. This + * class allows us to tell where the events are coming from when we group all the tags in a + * NestedSet. + * + * <p>The only usage of this code for now is to be able to use --output_filter in Skyframe + * + * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations. + */ +@Immutable +public final class TaggedEvents { + + @Nullable + private final String tag; + private final ImmutableCollection<Event> events; + + TaggedEvents(@Nullable String tag, ImmutableCollection<Event> events) { + + this.tag = tag; + this.events = events; + } + + @Nullable + String getTag() { + return tag; + } + + ImmutableCollection<Event> getEvents() { + return events; + } + + /** + * Returns <i>some</i> moderately sane representation of the events. Should never be used in + * user-visible places, only for debugging and testing. + */ + @Override + public String toString() { + return tag == null ? "<unknown>" : tag + ": " + Iterables.toString(events); + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueOrException.java b/src/main/java/com/google/devtools/build/skyframe/ValueOrException.java new file mode 100644 index 0000000..d682095 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/ValueOrException.java
@@ -0,0 +1,24 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import javax.annotation.Nullable; + +/** Wrapper for a value or the typed exception thrown when trying to compute it. */ +public abstract class ValueOrException<E extends Exception> extends ValueOrUntypedException { + + /** Gets the stored value. Throws an exception if one was thrown when computing this value. */ + @Nullable + public abstract SkyValue get() throws E; +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueOrException2.java b/src/main/java/com/google/devtools/build/skyframe/ValueOrException2.java new file mode 100644 index 0000000..deedbb1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/ValueOrException2.java
@@ -0,0 +1,25 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import javax.annotation.Nullable; + +/** Wrapper for a value or the typed exception thrown when trying to compute it. */ +public abstract class ValueOrException2<E1 extends Exception, E2 extends Exception> + extends ValueOrUntypedException { + + /** Gets the stored value. Throws an exception if one was thrown when computing this value. */ + @Nullable + public abstract SkyValue get() throws E1, E2; +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueOrException3.java b/src/main/java/com/google/devtools/build/skyframe/ValueOrException3.java new file mode 100644 index 0000000..e737c55 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/ValueOrException3.java
@@ -0,0 +1,25 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import javax.annotation.Nullable; + +/** Wrapper for a value or the typed exception thrown when trying to compute it. */ +public abstract class ValueOrException3<E1 extends Exception, E2 extends Exception, + E3 extends Exception> extends ValueOrUntypedException { + + /** Gets the stored value. Throws an exception if one was thrown when computing this value. */ + @Nullable + public abstract SkyValue get() throws E1, E2, E3; +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueOrException4.java b/src/main/java/com/google/devtools/build/skyframe/ValueOrException4.java new file mode 100644 index 0000000..176f405 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/ValueOrException4.java
@@ -0,0 +1,25 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import javax.annotation.Nullable; + +/** Wrapper for a value or the typed exception thrown when trying to compute it. */ +public abstract class ValueOrException4<E1 extends Exception, E2 extends Exception, + E3 extends Exception, E4 extends Exception> extends ValueOrUntypedException { + + /** Gets the stored value. Throws an exception if one was thrown when computing this value. */ + @Nullable + public abstract SkyValue get() throws E1, E2, E3, E4; +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueOrExceptionUtils.java b/src/main/java/com/google/devtools/build/skyframe/ValueOrExceptionUtils.java new file mode 100644 index 0000000..e66f4fa --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/ValueOrExceptionUtils.java
@@ -0,0 +1,520 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import javax.annotation.Nullable; + +/** Utilities for producing and consuming ValueOrException(2|3|4)? instances. */ +class ValueOrExceptionUtils { + + /** The bottom exception type. */ + class BottomException extends Exception { + } + + @Nullable + public static SkyValue downcovert(ValueOrException<BottomException> voe) { + return voe.getValue(); + } + + public static <E1 extends Exception> ValueOrException<E1> downcovert( + ValueOrException2<E1, BottomException> voe, Class<E1> exceptionClass1) { + Exception e = voe.getException(); + if (e == null) { + return new ValueOrExceptionValueImpl<>(voe.getValue()); + } + // Here and below, we use type-safe casts for performance reasons. Another approach would be + // cascading try-catch-rethrow blocks, but that has a higher performance penalty. + if (exceptionClass1.isInstance(e)) { + return new ValueOrExceptionExnImpl<>(exceptionClass1.cast(e)); + } + throw new IllegalStateException("shouldn't reach here " + e.getClass() + " " + exceptionClass1, + e); + } + + public static <E1 extends Exception, E2 extends Exception> ValueOrException2<E1, E2> downconvert( + ValueOrException3<E1, E2, BottomException> voe, Class<E1> exceptionClass1, + Class<E2> exceptionClass2) { + Exception e = voe.getException(); + if (e == null) { + return new ValueOrException2ValueImpl<>(voe.getValue()); + } + if (exceptionClass1.isInstance(e)) { + return new ValueOrException2Exn1Impl<>(exceptionClass1.cast(e)); + } + if (exceptionClass2.isInstance(e)) { + return new ValueOrException2Exn2Impl<>(exceptionClass2.cast(e)); + } + throw new IllegalStateException("shouldn't reach here " + e.getClass() + " " + exceptionClass1 + + " " + exceptionClass2, e); + } + + public static <E1 extends Exception, E2 extends Exception, E3 extends Exception> + ValueOrException3<E1, E2, E3> downconvert(ValueOrException4<E1, E2, E3, BottomException> voe, + Class<E1> exceptionClass1, Class<E2> exceptionClass2, Class<E3> exceptionClass3) { + Exception e = voe.getException(); + if (e == null) { + return new ValueOrException3ValueImpl<>(voe.getValue()); + } + if (exceptionClass1.isInstance(e)) { + return new ValueOrException3Exn1Impl<>(exceptionClass1.cast(e)); + } + if (exceptionClass2.isInstance(e)) { + return new ValueOrException3Exn2Impl<>(exceptionClass2.cast(e)); + } + if (exceptionClass3.isInstance(e)) { + return new ValueOrException3Exn3Impl<>(exceptionClass3.cast(e)); + } + throw new IllegalStateException("shouldn't reach here " + e.getClass() + " " + exceptionClass1 + + " " + exceptionClass2 + " " + exceptionClass3, e); + } + + public static <E extends Exception> ValueOrException<E> ofNull() { + return ValueOrExceptionValueImpl.ofNull(); + } + + public static ValueOrUntypedException ofValueUntyped(SkyValue value) { + return new ValueOrUntypedExceptionImpl(value); + } + + public static <E extends Exception> ValueOrException<E> ofExn(E e) { + return new ValueOrExceptionExnImpl<>(e); + } + + public static <E1 extends Exception, E2 extends Exception, E3 extends Exception, + E4 extends Exception> ValueOrException4<E1, E2, E3, E4> ofNullValue() { + return ValueOrException4ValueImpl.ofNullValue(); + } + + public static <E1 extends Exception, E2 extends Exception, E3 extends Exception, + E4 extends Exception> ValueOrException4<E1, E2, E3, E4> ofValue(SkyValue value) { + return new ValueOrException4ValueImpl<>(value); + } + + public static <E1 extends Exception, E2 extends Exception, E3 extends Exception, + E4 extends Exception> ValueOrException4<E1, E2, E3, E4> ofExn1(E1 e) { + return new ValueOrException4Exn1Impl<>(e); + } + + public static <E1 extends Exception, E2 extends Exception, E3 extends Exception, + E4 extends Exception> ValueOrException4<E1, E2, E3, E4> ofExn2(E2 e) { + return new ValueOrException4Exn2Impl<>(e); + } + + public static <E1 extends Exception, E2 extends Exception, E3 extends Exception, + E4 extends Exception> ValueOrException4<E1, E2, E3, E4> ofExn3(E3 e) { + return new ValueOrException4Exn3Impl<>(e); + } + + public static <E1 extends Exception, E2 extends Exception, E3 extends Exception, + E4 extends Exception> ValueOrException4<E1, E2, E3, E4> ofExn4(E4 e) { + return new ValueOrException4Exn4Impl<>(e); + } + + private static class ValueOrUntypedExceptionImpl extends ValueOrUntypedException { + @Nullable + private final SkyValue value; + + ValueOrUntypedExceptionImpl(@Nullable SkyValue value) { + this.value = value; + } + + @Override + @Nullable + public SkyValue getValue() { + return value; + } + + @Override + public Exception getException() { + return null; + } + } + + private static class ValueOrExceptionValueImpl<E extends Exception> extends ValueOrException<E> { + private static final ValueOrExceptionValueImpl<Exception> NULL = + new ValueOrExceptionValueImpl<Exception>((SkyValue) null); + + @Nullable + private final SkyValue value; + + private ValueOrExceptionValueImpl(@Nullable SkyValue value) { + this.value = value; + } + + @Override + @Nullable + public SkyValue get() { + return value; + } + + @Override + @Nullable + public SkyValue getValue() { + return value; + } + + @Override + @Nullable + public Exception getException() { + return null; + } + + @SuppressWarnings("unchecked") + public static <E extends Exception> ValueOrExceptionValueImpl<E> ofNull() { + return (ValueOrExceptionValueImpl<E>) NULL; + } + } + + private static class ValueOrExceptionExnImpl<E extends Exception> extends ValueOrException<E> { + private final E e; + + private ValueOrExceptionExnImpl(E e) { + this.e = e; + } + + @Override + public SkyValue get() throws E { + throw e; + } + + @Override + @Nullable + public SkyValue getValue() { + return null; + } + + @Override + public Exception getException() { + return e; + } + } + + private static class ValueOrException2ValueImpl<E1 extends Exception, E2 extends Exception> + extends ValueOrException2<E1, E2> { + @Nullable + private final SkyValue value; + + ValueOrException2ValueImpl(@Nullable SkyValue value) { + this.value = value; + } + + @Override + @Nullable + public SkyValue get() throws E1, E2 { + return value; + } + + @Override + @Nullable + public Exception getException() { + return null; + } + + @Override + @Nullable + public SkyValue getValue() { + return value; + } + } + + private static class ValueOrException2Exn1Impl<E1 extends Exception, E2 extends Exception> + extends ValueOrException2<E1, E2> { + private final E1 e; + + private ValueOrException2Exn1Impl(E1 e) { + this.e = e; + } + + @Override + public SkyValue get() throws E1 { + throw e; + } + + @Override + public Exception getException() { + return e; + } + + @Override + @Nullable + public SkyValue getValue() { + return null; + } + } + + private static class ValueOrException2Exn2Impl<E1 extends Exception, E2 extends Exception> + extends ValueOrException2<E1, E2> { + private final E2 e; + + private ValueOrException2Exn2Impl(E2 e) { + this.e = e; + } + + @Override + public SkyValue get() throws E2 { + throw e; + } + + @Override + public Exception getException() { + return e; + } + + @Override + @Nullable + public SkyValue getValue() { + return null; + } + } + + private static class ValueOrException3ValueImpl<E1 extends Exception, E2 extends Exception, + E3 extends Exception> extends ValueOrException3<E1, E2, E3> { + @Nullable + private final SkyValue value; + + ValueOrException3ValueImpl(@Nullable SkyValue value) { + this.value = value; + } + + @Override + @Nullable + public SkyValue get() throws E1, E2 { + return value; + } + + @Override + @Nullable + public Exception getException() { + return null; + } + + @Override + @Nullable + public SkyValue getValue() { + return value; + } + } + + private static class ValueOrException3Exn1Impl<E1 extends Exception, E2 extends Exception, + E3 extends Exception> extends ValueOrException3<E1, E2, E3> { + private final E1 e; + + private ValueOrException3Exn1Impl(E1 e) { + this.e = e; + } + + @Override + public SkyValue get() throws E1 { + throw e; + } + + @Override + public Exception getException() { + return e; + } + + @Override + @Nullable + public SkyValue getValue() { + return null; + } + } + + private static class ValueOrException3Exn2Impl<E1 extends Exception, E2 extends Exception, + E3 extends Exception> extends ValueOrException3<E1, E2, E3> { + private final E2 e; + + private ValueOrException3Exn2Impl(E2 e) { + this.e = e; + } + + @Override + public SkyValue get() throws E2 { + throw e; + } + + @Override + public Exception getException() { + return e; + } + + @Override + @Nullable + public SkyValue getValue() { + return null; + } + } + + private static class ValueOrException3Exn3Impl<E1 extends Exception, E2 extends Exception, + E3 extends Exception> extends ValueOrException3<E1, E2, E3> { + private final E3 e; + + private ValueOrException3Exn3Impl(E3 e) { + this.e = e; + } + + @Override + public SkyValue get() throws E3 { + throw e; + } + + @Override + public Exception getException() { + return e; + } + + @Override + @Nullable + public SkyValue getValue() { + return null; + } + } + + private static class ValueOrException4ValueImpl<E1 extends Exception, E2 extends Exception, + E3 extends Exception, E4 extends Exception> extends ValueOrException4<E1, E2, E3, E4> { + private static final ValueOrException4ValueImpl<Exception, Exception, Exception, + Exception> NULL = new ValueOrException4ValueImpl<>((SkyValue) null); + + @Nullable + private final SkyValue value; + + ValueOrException4ValueImpl(@Nullable SkyValue value) { + this.value = value; + } + + @Override + @Nullable + public SkyValue get() throws E1, E2 { + return value; + } + + @Override + @Nullable + public Exception getException() { + return null; + } + + @Override + @Nullable + public SkyValue getValue() { + return value; + } + + @SuppressWarnings("unchecked") + private static <E1 extends Exception, E2 extends Exception, E3 extends Exception, + E4 extends Exception>ValueOrException4ValueImpl<E1, E2, E3, E4> ofNullValue() { + return (ValueOrException4ValueImpl<E1, E2, E3, E4>) NULL; + } + } + + private static class ValueOrException4Exn1Impl<E1 extends Exception, E2 extends Exception, + E3 extends Exception, E4 extends Exception> extends ValueOrException4<E1, E2, E3, E4> { + private final E1 e; + + private ValueOrException4Exn1Impl(E1 e) { + this.e = e; + } + + @Override + public SkyValue get() throws E1 { + throw e; + } + + @Override + public Exception getException() { + return e; + } + + @Override + @Nullable + public SkyValue getValue() { + return null; + } + } + + private static class ValueOrException4Exn2Impl<E1 extends Exception, E2 extends Exception, + E3 extends Exception, E4 extends Exception> extends ValueOrException4<E1, E2, E3, E4> { + private final E2 e; + + private ValueOrException4Exn2Impl(E2 e) { + this.e = e; + } + + @Override + public SkyValue get() throws E2 { + throw e; + } + + @Override + public Exception getException() { + return e; + } + + @Override + @Nullable + public SkyValue getValue() { + return null; + } + } + + private static class ValueOrException4Exn3Impl<E1 extends Exception, E2 extends Exception, + E3 extends Exception, E4 extends Exception> extends ValueOrException4<E1, E2, E3, E4> { + private final E3 e; + + private ValueOrException4Exn3Impl(E3 e) { + this.e = e; + } + + @Override + public SkyValue get() throws E3 { + throw e; + } + + @Override + public Exception getException() { + return e; + } + + @Override + @Nullable + public SkyValue getValue() { + return null; + } + } + + private static class ValueOrException4Exn4Impl<E1 extends Exception, E2 extends Exception, + E3 extends Exception, E4 extends Exception> extends ValueOrException4<E1, E2, E3, E4> { + private final E4 e; + + private ValueOrException4Exn4Impl(E4 e) { + this.e = e; + } + + @Override + public SkyValue get() throws E4 { + throw e; + } + + @Override + public Exception getException() { + return e; + } + + @Override + @Nullable + public SkyValue getValue() { + return null; + } + } +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueOrUntypedException.java b/src/main/java/com/google/devtools/build/skyframe/ValueOrUntypedException.java new file mode 100644 index 0000000..c7ea7d4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/ValueOrUntypedException.java
@@ -0,0 +1,34 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import javax.annotation.Nullable; + +/** + * Wrapper for a value or the untyped exception thrown when trying to compute it. + * + * <p>This is an implementation detail of {@link ParallelEvaluator} and + * {@link ValueOrExceptionUtils}. It's an abstract class (as opposed to an interface) to avoid + * exposing the methods outside the package. + */ +abstract class ValueOrUntypedException { + + /** Returns the stored value, if there was one. */ + @Nullable + abstract SkyValue getValue(); + + /** Returns the stored exception, if there was one. */ + @Nullable + abstract Exception getException(); +}
diff --git a/src/main/java/com/google/devtools/build/skyframe/ValueWithMetadata.java b/src/main/java/com/google/devtools/build/skyframe/ValueWithMetadata.java new file mode 100644 index 0000000..956e404 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/ValueWithMetadata.java
@@ -0,0 +1,209 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +package com.google.devtools.build.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; + +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * Encapsulation of data stored by {@link NodeEntry} when the value has finished building. + * + * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations. + */ +public abstract class ValueWithMetadata implements SkyValue { + protected final SkyValue value; + + private static final NestedSet<TaggedEvents> NO_EVENTS = + NestedSetBuilder.<TaggedEvents>emptySet(Order.STABLE_ORDER); + + public ValueWithMetadata(SkyValue value) { + this.value = value; + } + + /** Builds a value entry value that has an error (and no value value). + * + * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations. + */ + public static ValueWithMetadata error(ErrorInfo errorInfo, + NestedSet<TaggedEvents> transitiveEvents) { + return new ErrorInfoValue(errorInfo, null, transitiveEvents); + } + + /** + * Builds a value entry value that has a value value, and possibly an error (constructed from its + * children's errors). + * + * <p>This is intended only for use in alternative {@code MemoizingEvaluator} implementations. + */ + static SkyValue normal(@Nullable SkyValue value, @Nullable ErrorInfo errorInfo, + NestedSet<TaggedEvents> transitiveEvents) { + Preconditions.checkState(value != null || errorInfo != null, + "Value and error cannot both be null"); + if (errorInfo == null) { + return transitiveEvents.isEmpty() + ? value + : new ValueWithEvents(value, transitiveEvents); + } + return new ErrorInfoValue(errorInfo, value, transitiveEvents); + } + + + @Nullable SkyValue getValue() { + return value; + } + + @Nullable + abstract ErrorInfo getErrorInfo(); + + abstract NestedSet<TaggedEvents> getTransitiveEvents(); + + static final class ValueWithEvents extends ValueWithMetadata { + + private final NestedSet<TaggedEvents> transitiveEvents; + + ValueWithEvents(SkyValue value, NestedSet<TaggedEvents> transitiveEvents) { + super(Preconditions.checkNotNull(value)); + this.transitiveEvents = Preconditions.checkNotNull(transitiveEvents); + } + + @Nullable + @Override + ErrorInfo getErrorInfo() { return null; } + + @Override + NestedSet<TaggedEvents> getTransitiveEvents() { return transitiveEvents; } + + /** + * We override equals so that if the same value is written to a {@link NodeEntry} twice, it can + * verify that the two values are equal, and avoid incrementing its version. + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ValueWithEvents that = (ValueWithEvents) o; + + // Shallow equals is a middle ground between using default equals, which might miss + // nested sets with the same elements, and deep equality checking, which would be expensive. + // All three choices are sound, since shallow equals and default equals are more + // conservative than deep equals. Using shallow equals means that we may unnecessarily + // consider some values unequal that are actually equal, but this is still a net win over + // deep equals. + return value.equals(that.value) && transitiveEvents.shallowEquals(that.transitiveEvents); + } + + @Override + public int hashCode() { + return 31 * value.hashCode() + transitiveEvents.hashCode(); + } + + @Override + public String toString() { return value.toString(); } + } + + static final class ErrorInfoValue extends ValueWithMetadata { + + private final ErrorInfo errorInfo; + private final NestedSet<TaggedEvents> transitiveEvents; + + ErrorInfoValue(ErrorInfo errorInfo, @Nullable SkyValue value, + NestedSet<TaggedEvents> transitiveEvents) { + super(value); + this.errorInfo = Preconditions.checkNotNull(errorInfo); + this.transitiveEvents = Preconditions.checkNotNull(transitiveEvents); + } + + @Nullable + @Override + ErrorInfo getErrorInfo() { return errorInfo; } + + @Override + NestedSet<TaggedEvents> getTransitiveEvents() { return transitiveEvents; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ErrorInfoValue that = (ErrorInfoValue) o; + + // Shallow equals is a middle ground between using default equals, which might miss + // nested sets with the same elements, and deep equality checking, which would be expensive. + // All three choices are sound, since shallow equals and default equals are more + // conservative than deep equals. Using shallow equals means that we may unnecessarily + // consider some values unequal that are actually equal, but this is still a net win over + // deep equals. + return Objects.equals(this.value, that.value) + && Objects.equals(this.errorInfo, that.errorInfo) + && transitiveEvents.shallowEquals(that.transitiveEvents); + } + + @Override + public int hashCode() { + return 31 * Objects.hash(value, errorInfo) + transitiveEvents.shallowHashCode(); + } + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + if (value != null) { + result.append("Value: ").append(value); + } + if (errorInfo != null) { + if (result.length() > 0) { + result.append("; "); + } + result.append("Error: ").append(errorInfo); + } + return result.toString(); + } + } + + static SkyValue justValue(SkyValue value) { + if (value instanceof ValueWithMetadata) { + return ((ValueWithMetadata) value).getValue(); + } + return value; + } + + static ValueWithMetadata wrapWithMetadata(SkyValue value) { + if (value instanceof ValueWithMetadata) { + return (ValueWithMetadata) value; + } + return new ValueWithEvents(value, NO_EVENTS); + } + + @Nullable + public static ErrorInfo getMaybeErrorInfo(SkyValue value) { + if (value.getClass() == ErrorInfoValue.class) { + return ((ValueWithMetadata) value).getErrorInfo(); + } + return null; + + } +} \ No newline at end of file
diff --git a/src/main/java/com/google/devtools/build/skyframe/Version.java b/src/main/java/com/google/devtools/build/skyframe/Version.java new file mode 100644 index 0000000..90a6020 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/Version.java
@@ -0,0 +1,32 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +/** + * A Version defines a value in a version tree used in persistent data structures. + * See http://en.wikipedia.org/wiki/Persistent_data_structure. + */ +public interface Version { + /** + * Defines a partial order relation on versions. Returns true if this object is at most + * {@code other} in that partial order. If x.equals(y), then x.atMost(y). + * + * <p>If x.atMost(y) returns false, then there are two possibilities: y < x in the partial order, + * so y.atMost(x) returns true and !x.equals(y), or x and y are incomparable in this partial + * order. This may be because x and y are instances of different Version implementations (although + * it is legal for different Version implementations to be comparable as well). + * See http://en.wikipedia.org/wiki/Partially_ordered_set. + */ + boolean atMost(Version other); +}
diff --git a/src/main/java/com/google/devtools/common/options/Converter.java b/src/main/java/com/google/devtools/common/options/Converter.java new file mode 100644 index 0000000..867ef82 --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/Converter.java
@@ -0,0 +1,33 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.common.options; + +/** + * A converter is a little helper object that can take a String and + * turn it into an instance of type T (the type parameter to the converter). + */ +public interface Converter<T> { + + /** + * Convert a string into type T. + */ + T convert(String input) throws OptionsParsingException; + + /** + * The type description appears in usage messages. E.g.: "a string", + * "a path", etc. + */ + String getTypeDescription(); + +}
diff --git a/src/main/java/com/google/devtools/common/options/Converters.java b/src/main/java/com/google/devtools/common/options/Converters.java new file mode 100644 index 0000000..e8c69ec --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/Converters.java
@@ -0,0 +1,326 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.common.options; + +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * Some convenient converters used by blaze. Note: These are specific to + * blaze. + */ +public final class Converters { + + /** + * Join a list of words as in English. Examples: + * "nothing" + * "one" + * "one or two" + * "one and two" + * "one, two or three". + * "one, two and three". + * The toString method of each element is used. + */ + static String joinEnglishList(Iterable<?> choices) { + StringBuilder buf = new StringBuilder(); + for (Iterator<?> ii = choices.iterator(); ii.hasNext(); ) { + Object choice = ii.next(); + if (buf.length() > 0) { + buf.append(ii.hasNext() ? ", " : " or "); + } + buf.append(choice); + } + return buf.length() == 0 ? "nothing" : buf.toString(); + } + + public static class SeparatedOptionListConverter + implements Converter<List<String>> { + + private final String separatorDescription; + private final Splitter splitter; + + protected SeparatedOptionListConverter(char separator, + String separatorDescription) { + this.separatorDescription = separatorDescription; + this.splitter = Splitter.on(separator); + } + + @Override + public List<String> convert(String input) { + return input.equals("") + ? ImmutableList.<String>of() + : ImmutableList.copyOf(splitter.split(input)); + } + + @Override + public String getTypeDescription() { + return separatorDescription + "-separated list of options"; + } + } + + public static class CommaSeparatedOptionListConverter + extends SeparatedOptionListConverter { + public CommaSeparatedOptionListConverter() { + super(',', "comma"); + } + } + + public static class ColonSeparatedOptionListConverter extends SeparatedOptionListConverter { + public ColonSeparatedOptionListConverter() { + super(':', "colon"); + } + } + + public static class LogLevelConverter implements Converter<Level> { + + public static Level[] LEVELS = new Level[] { + Level.OFF, Level.SEVERE, Level.WARNING, Level.INFO, Level.FINE, + Level.FINER, Level.FINEST + }; + + @Override + public Level convert(String input) throws OptionsParsingException { + try { + int level = Integer.parseInt(input); + return LEVELS[level]; + } catch (NumberFormatException e) { + throw new OptionsParsingException("Not a log level: " + input); + } catch (ArrayIndexOutOfBoundsException e) { + throw new OptionsParsingException("Not a log level: " + input); + } + } + + @Override + public String getTypeDescription() { + return "0 <= an integer <= " + (LEVELS.length - 1); + } + + } + + /** + * Checks whether a string is part of a set of strings. + */ + public static class StringSetConverter implements Converter<String> { + + // TODO(bazel-team): if this class never actually contains duplicates, we could s/List/Set/ + // here. + private final List<String> values; + + public StringSetConverter(String... values) { + this.values = ImmutableList.copyOf(values); + } + + @Override + public String convert(String input) throws OptionsParsingException { + if (values.contains(input)) { + return input; + } + + throw new OptionsParsingException("Not one of " + values); + } + + @Override + public String getTypeDescription() { + return joinEnglishList(values); + } + } + + /** + * Checks whether a string is a valid regex pattern and compiles it. + */ + public static class RegexPatternConverter implements Converter<Pattern> { + + @Override + public Pattern convert(String input) throws OptionsParsingException { + try { + return Pattern.compile(input); + } catch (PatternSyntaxException e) { + throw new OptionsParsingException("Not a valid regular expression: " + e.getMessage()); + } + } + + @Override + public String getTypeDescription() { + return "a valid Java regular expression"; + } + } + + /** + * Limits the length of a string argument. + */ + public static class LengthLimitingConverter implements Converter<String> { + private final int maxSize; + + public LengthLimitingConverter(int maxSize) { + this.maxSize = maxSize; + } + + @Override + public String convert(String input) throws OptionsParsingException { + if (input.length() > maxSize) { + throw new OptionsParsingException("Input must be " + getTypeDescription()); + } + return input; + } + + @Override + public String getTypeDescription() { + return "a string <= " + maxSize + " characters"; + } + } + + /** + * Checks whether an integer is in the given range. + */ + public static class RangeConverter implements Converter<Integer> { + final int minValue; + final int maxValue; + + public RangeConverter(int minValue, int maxValue) { + this.minValue = minValue; + this.maxValue = maxValue; + } + + @Override + public Integer convert(String input) throws OptionsParsingException { + try { + Integer value = Integer.parseInt(input); + if (value < minValue) { + throw new OptionsParsingException("'" + input + "' should be >= " + minValue); + } else if (value < minValue || value > maxValue) { + throw new OptionsParsingException("'" + input + "' should be <= " + maxValue); + } + return value; + } catch (NumberFormatException e) { + throw new OptionsParsingException("'" + input + "' is not an int"); + } + } + + @Override + public String getTypeDescription() { + if (minValue == Integer.MIN_VALUE) { + if (maxValue == Integer.MAX_VALUE) { + return "an integer"; + } else { + return "an integer, <= " + maxValue; + } + } else if (maxValue == Integer.MAX_VALUE) { + return "an integer, >= " + minValue; + } else { + return "an integer in " + + (minValue < 0 ? "(" + minValue + ")" : minValue) + "-" + maxValue + " range"; + } + } + } + + /** + * A converter for variable assignments from the parameter list of a blaze + * command invocation. Assignments are expected to have the form "name=value", + * where names and values are defined to be as permissive as possible. + */ + public static class AssignmentConverter implements Converter<Map.Entry<String, String>> { + + @Override + public Map.Entry<String, String> convert(String input) + throws OptionsParsingException { + int pos = input.indexOf("="); + if (pos <= 0) { + throw new OptionsParsingException("Variable definitions must be in the form of a " + + "'name=value' assignment"); + } + String name = input.substring(0, pos); + String value = input.substring(pos + 1); + return Maps.immutableEntry(name, value); + } + + @Override + public String getTypeDescription() { + return "a 'name=value' assignment"; + } + + } + + /** + * A converter for variable assignments from the parameter list of a blaze + * command invocation. Assignments are expected to have the form "name[=value]", + * where names and values are defined to be as permissive as possible and value + * part can be optional (in which case it is considered to be null). + */ + public static class OptionalAssignmentConverter implements Converter<Map.Entry<String, String>> { + + @Override + public Map.Entry<String, String> convert(String input) + throws OptionsParsingException { + int pos = input.indexOf("="); + if (pos == 0 || input.length() == 0) { + throw new OptionsParsingException("Variable definitions must be in the form of a " + + "'name=value' or 'name' assignment"); + } else if (pos < 0) { + return Maps.immutableEntry(input, null); + } + String name = input.substring(0, pos); + String value = input.substring(pos + 1); + return Maps.immutableEntry(name, value); + } + + @Override + public String getTypeDescription() { + return "a 'name=value' assignment with an optional value part"; + } + + } + + public static class HelpVerbosityConverter extends EnumConverter<OptionsParser.HelpVerbosity> { + public HelpVerbosityConverter() { + super(OptionsParser.HelpVerbosity.class, "--help_verbosity setting"); + } + } + + /** + * A converter for boolean values. This is already one of the defaults, so clients + * should not typically need to add this. + */ + public static class BooleanConverter implements Converter<Boolean> { + @Override + public Boolean convert(String input) throws OptionsParsingException { + if (input == null) { + return false; + } + input = input.toLowerCase(); + if (input.equals("true") || input.equals("1") || input.equals("yes") || + input.equals("t") || input.equals("y")) { + return true; + } + if (input.equals("false") || input.equals("0") || input.equals("no") || + input.equals("f") || input.equals("n")) { + return false; + } + throw new OptionsParsingException("'" + input + "' is not a boolean"); + } + + @Override + public String getTypeDescription() { + return "a boolean"; + } + } + +}
diff --git a/src/main/java/com/google/devtools/common/options/DuplicateOptionDeclarationException.java b/src/main/java/com/google/devtools/common/options/DuplicateOptionDeclarationException.java new file mode 100644 index 0000000..b4e572e --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/DuplicateOptionDeclarationException.java
@@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.common.options; + +/** + * Indicates that an option is declared in more than one class. + */ +public class DuplicateOptionDeclarationException extends RuntimeException { + + DuplicateOptionDeclarationException(String message) { + super(message); + } + +}
diff --git a/src/main/java/com/google/devtools/common/options/EnumConverter.java b/src/main/java/com/google/devtools/common/options/EnumConverter.java new file mode 100644 index 0000000..f65241a --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/EnumConverter.java
@@ -0,0 +1,74 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.common.options; + +import java.util.Arrays; + +/** + * A converter superclass for converters that parse enums. + * + * Just subclass this class, creating a zero aro argument constructor that + * calls {@link #EnumConverter(Class, String)}. + * + * This class compares the input string to the string returned by the toString() + * method of each enum member in a case-insensitive way. Usually, this is the + * name of the symbol, but beware if you override toString()! + */ +public abstract class EnumConverter<T extends Enum<T>> + implements Converter<T> { + + private final Class<T> enumType; + private final String typeName; + + /** + * Creates a new enum converter. You *must* implement a zero-argument + * constructor that delegates to this constructor, passing in the appropriate + * parameters. + * + * @param enumType The type of your enumeration; usually a class literal + * like MyEnum.class + * @param typeName The intuitive name of your enumeration, for example, the + * type name for CompilationMode might be "compilation mode". + */ + protected EnumConverter(Class<T> enumType, String typeName) { + this.enumType = enumType; + this.typeName = typeName; + } + + /** + * Implements {@link #convert(String)}. + */ + @Override + public final T convert(String input) throws OptionsParsingException { + for (T value : enumType.getEnumConstants()) { + if (value.toString().equalsIgnoreCase(input)) { + return value; + } + } + throw new OptionsParsingException("Not a valid " + typeName + ": '" + + input + "' (should be " + + getTypeDescription() + ")"); + } + + /** + * Implements {@link #getTypeDescription()}. + */ + @Override + public final String getTypeDescription() { + return Converters.joinEnglishList( + Arrays.asList(enumType.getEnumConstants())).toLowerCase(); + } + +}
diff --git a/src/main/java/com/google/devtools/common/options/GenericTypeHelper.java b/src/main/java/com/google/devtools/common/options/GenericTypeHelper.java new file mode 100644 index 0000000..2240860 --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/GenericTypeHelper.java
@@ -0,0 +1,133 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.common.options; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.primitives.Primitives; +import com.google.common.reflect.TypeToken; + +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.lang.reflect.TypeVariable; + +/** + * A helper class for {@link OptionsParserImpl} to help checking the return type + * of a {@link Converter} against the type of a field or the element type of a + * list. + * + * <p>This class has to go through considerable contortion to get the correct result + * from the Java reflection system, unfortunately. If the generic reflection part + * had been better designed, some of this would not be necessary. + */ +class GenericTypeHelper { + + /** + * Returns the raw type of t, if t is either a raw or parameterized type. + * Otherwise, this method throws an {@link AssertionError}. + */ + @VisibleForTesting + static Class<?> getRawType(Type t) { + if (t instanceof Class<?>) { + return (Class<?>) t; + } else if (t instanceof ParameterizedType) { + return (Class<?>) ((ParameterizedType) t).getRawType(); + } else { + throw new AssertionError("A known concrete type is not concrete"); + } + } + + /** + * If type is a parameterized type, searches the given type variable in the list + * of declared type variables, and then returns the corresponding actual type. + * Returns null if the type variable is not defined by type. + */ + private static Type matchTypeVariable(Type type, TypeVariable<?> variable) { + if (type instanceof ParameterizedType) { + Class<?> rawInterfaceType = getRawType(type); + TypeVariable<?>[] typeParameters = rawInterfaceType.getTypeParameters(); + for (int i = 0; i < typeParameters.length; i++) { + if (variable.equals(typeParameters[i])) { + return ((ParameterizedType) type).getActualTypeArguments()[i]; + } + } + } + return null; + } + + /** + * Resolves the return type of a method, in particular if the generic return + * type ({@link Method#getGenericReturnType()}) is a type variable + * ({@link TypeVariable}), by checking all super-classes and directly + * implemented interfaces. + * + * <p>The method m must be defined by the given type or by its raw class type. + * + * @throws AssertionError if the generic return type could not be resolved + */ + // TODO(bazel-team): also check enclosing classes and indirectly implemented + // interfaces, which can also contribute type variables. This doesn't happen + // in the existing use cases. + public static Type getActualReturnType(Type type, Method method) { + Type returnType = method.getGenericReturnType(); + if (returnType instanceof Class<?>) { + return returnType; + } else if (returnType instanceof ParameterizedType) { + return returnType; + } else if (returnType instanceof TypeVariable<?>) { + TypeVariable<?> variable = (TypeVariable<?>) returnType; + while (type != null) { + Type candidate = matchTypeVariable(type, variable); + if (candidate != null) { + return candidate; + } + + Class<?> rawType = getRawType(type); + for (Type interfaceType : rawType.getGenericInterfaces()) { + candidate = matchTypeVariable(interfaceType, variable); + if (candidate != null) { + return candidate; + } + } + + type = rawType.getGenericSuperclass(); + } + } + throw new AssertionError("The type " + returnType + + " is not a Class, ParameterizedType, or TypeVariable"); + } + + /** + * Determines if a value of a particular type (from) is assignable to a field of + * a particular type (to). Also allows assigning wrapper types to primitive + * types. + * + * <p>The checks done here should be identical to the checks done by + * {@link java.lang.reflect.Field#set}. I.e., if this method returns true, a + * subsequent call to {@link java.lang.reflect.Field#set} should succeed. + */ + public static boolean isAssignableFrom(Type to, Type from) { + if (to instanceof Class<?>) { + Class<?> toClass = (Class<?>) to; + if (toClass.isPrimitive()) { + return Primitives.wrap(toClass).equals(from); + } + } + return TypeToken.of(to).isAssignableFrom(from); + } + + private GenericTypeHelper() { + // Prevents Java from creating a public constructor. + } +}
diff --git a/src/main/java/com/google/devtools/common/options/Option.java b/src/main/java/com/google/devtools/common/options/Option.java new file mode 100644 index 0000000..e244736 --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/Option.java
@@ -0,0 +1,127 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.common.options; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * An interface for annotating fields in classes (derived from OptionsBase) + * that are options. + */ +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.RUNTIME) +public @interface Option { + + /** + * The name of the option ("--name"). + */ + String name(); + + /** + * The single-character abbreviation of the option ("-abbrev"). + */ + char abbrev() default '\0'; + + /** + * A help string for the usage information. + */ + String help() default ""; + + /** + * The default value for the option. This method should only be invoked + * directly by the parser implementation. Any access to default values + * should go via the parser to allow for application specific defaults. + * + * <p>There are two reasons this is a string. Firstly, it ensures that + * explicitly specifying this option at its default value (as printed in the + * usage message) has the same behavior as not specifying the option at all; + * this would be very hard to achieve if the default value was an instance of + * type T, since we'd need to ensure that {@link #toString()} and {@link + * #converter} were dual to each other. The second reason is more mundane + * but also more restrictive: annotation values must be compile-time + * constants. + * + * <p>If an option's defaultValue() is the string "null", the option's + * converter will not be invoked to interpret it; a null reference will be + * used instead. (It would be nice if defaultValue could simply return null, + * but bizarrely, the Java Language Specification does not consider null to + * be a compile-time constant.) This special interpretation of the string + * "null" is only applicable when computing the default value; if specified + * on the command-line, this string will have its usual literal meaning. + */ + String defaultValue(); + + /** + * A string describing the category of options that this belongs to. {@link + * OptionsParser#describeOptions} prints options of the same category grouped + * together. + */ + String category() default "misc"; + + /** + * The converter that we'll use to convert this option into an object or + * a simple type. The default is to use the builtin converters. + * Custom converters must implement the {@link Converter} interface. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + // Can't figure out how to coerce Converter.class into Class<? extends Converter<?>> + Class<? extends Converter> converter() default Converter.class; + + /** + * A flag indicating whether the option type should be allowed to occur + * multiple times in a single option list. + * + * <p>If the command can occur multiple times, then the attribute value + * <em>must</em> be a list type {@code List<T>}, and the result type of the + * converter for this option must either match the parameter {@code T} or + * {@code List<T>}. In the latter case the individual lists are concatenated + * to form the full options value. + */ + boolean allowMultiple() default false; + + /** + * If the option is actually an abbreviation for other options, this field will + * contain the strings to expand this option into. The original option is dropped + * and the replacement used in its stead. It is recommended that such an option be + * of type {@link Void}. + * + * An expanded option overrides previously specified options of the same name, + * even if it is explicitly specified. This is the original behavior and can + * be surprising if the user is not aware of it, which has led to several + * requests to change this behavior. This was discussed in the blaze team and + * it was decided that it is not a strong enough case to change the behavior. + */ + String[] expansion() default {}; + + /** + * If the option requires that additional options be implicitly appended, this field + * will contain the additional options. Implicit dependencies are parsed at the end + * of each {@link OptionsParser#parse} invocation, and override options specified in + * the same call. However, they can be overridden by options specified in a later + * call or by options with a higher priority. + * + * @see OptionPriority + */ + String[] implicitRequirements() default {}; + + /** + * If this field is a non-empty string, the option is deprecated, and a + * deprecation warning is added to the list of warnings when such an option + * is used. + */ + String deprecationWarning() default ""; +}
diff --git a/src/main/java/com/google/devtools/common/options/OptionPriority.java b/src/main/java/com/google/devtools/common/options/OptionPriority.java new file mode 100644 index 0000000..6e90008 --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/OptionPriority.java
@@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.common.options; + +/** + * The priority of option values, in order of increasing priority. + * + * <p>In general, new values for options can only override values with a lower or + * equal priority. Option values provided in annotations in an options class are + * implicitly at the priority {@code DEFAULT}. + * + * <p>The ordering of the priorities is the source-code order. This is consistent + * with the automatically generated {@code compareTo} method as specified by the + * Java Language Specification. DO NOT change the source-code order of these + * values, or you will break code that relies on the ordering. + */ +public enum OptionPriority { + + /** + * The priority of values specified in the {@link Option} annotation. This + * should never be specified in calls to {@link OptionsParser#parse}. + */ + DEFAULT, + + /** + * Overrides default options at runtime, while still allowing the values to be + * overridden manually. + */ + COMPUTED_DEFAULT, + + /** + * For options coming from a configuration file or rc file. + */ + RC_FILE, + + /** + * For options coming from the command line. + */ + COMMAND_LINE, + + /** + * This priority can be used to unconditionally override any user-provided options. + * This should be used rarely and with caution! + */ + SOFTWARE_REQUIREMENT; + +}
diff --git a/src/main/java/com/google/devtools/common/options/Options.java b/src/main/java/com/google/devtools/common/options/Options.java new file mode 100644 index 0000000..171be2e --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/Options.java
@@ -0,0 +1,104 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.common.options; + +import java.util.Arrays; +import java.util.List; + +/** + * Interface for parsing options from a single options specification class. + * + * The {@link Options#parse(Class, String...)} method in this class has no clear + * use case. Instead, use the {@link OptionsParser} class directly, as in this + * code snippet: + * + * <pre> + * OptionsParser parser = OptionsParser.newOptionsParser(FooOptions.class); + * try { + * parser.parse(FooOptions.class, args); + * } catch (OptionsParsingException e) { + * System.err.print("Error parsing options: " + e.getMessage()); + * System.err.print(options.getUsage()); + * System.exit(1); + * } + * FooOptions foo = parser.getOptions(FooOptions.class); + * List<String> otherArguments = parser.getResidue(); + * </pre> + * + * Using this class in this case actually results in more code. + * + * @see OptionsParser for parsing options from multiple options specification classes. + */ +public class Options<O extends OptionsBase> { + + /** + * Parse the options provided in args, given the specification in + * optionsClass. + */ + public static <O extends OptionsBase> Options<O> parse(Class<O> optionsClass, String... args) + throws OptionsParsingException { + OptionsParser parser = OptionsParser.newOptionsParser(optionsClass); + parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList(args)); + List<String> remainingArgs = parser.getResidue(); + return new Options<O>(parser.getOptions(optionsClass), + remainingArgs.toArray(new String[0])); + } + + /** + * Returns an options object at its default values. The returned object may + * be freely modified by the caller, by assigning its fields. + */ + public static <O extends OptionsBase> O getDefaults(Class<O> optionsClass) { + try { + return parse(optionsClass, new String[0]).getOptions(); + } catch (OptionsParsingException e) { + String message = "Error while parsing defaults: " + e.getMessage(); + throw new AssertionError(message); + } + } + + /** + * Returns a usage string (renders the help information, the defaults, and + * of course the option names). + */ + public static String getUsage(Class<? extends OptionsBase> optionsClass) { + StringBuilder usage = new StringBuilder(); + OptionsUsage.getUsage(optionsClass, usage); + return usage.toString(); + } + + private O options; + private String[] remainingArgs; + + private Options(O options, String[] remainingArgs) { + this.options = options; + this.remainingArgs = remainingArgs; + } + + /** + * Returns an instance of options class O. + */ + public O getOptions() { + return options; + } + + /** + * Returns the arguments that we didn't parse. + */ + public String[] getRemainingArgs() { + return remainingArgs; + } + +}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsBase.java b/src/main/java/com/google/devtools/common/options/OptionsBase.java new file mode 100644 index 0000000..ed9f215 --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/OptionsBase.java
@@ -0,0 +1,118 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.common.options; + +import com.google.common.escape.CharEscaperBuilder; +import com.google.common.escape.Escaper; + +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +/** + * Base class for all options classes. Extend this class, adding public + * instance fields annotated with @Option. Then you can create instances + * either programmatically: + * + * <pre> + * X x = Options.getDefaults(X.class); + * x.host = "localhost"; + * x.port = 80; + * </pre> + * + * or from an array of command-line arguments: + * + * <pre> + * OptionsParser parser = OptionsParser.newOptionsParser(X.class); + * parser.parse("--host", "localhost", "--port", "80"); + * X x = parser.getOptions(X.class); + * </pre> + * + * <p>Subclasses of OptionsBase <b>must</b> be constructed reflectively, + * i.e. using not {@code new MyOptions}, but one of the two methods above + * instead. (Direct construction creates an empty instance, not containing + * default values. This leads to surprising behavior and often + * NullPointerExceptions, etc.) + */ +public abstract class OptionsBase { + + private static final Escaper ESCAPER = new CharEscaperBuilder() + .addEscape('\\', "\\\\").addEscape('"', "\\\"").toEscaper(); + + /** + * Subclasses must provide a default (no argument) constructor. + */ + protected OptionsBase() { + // There used to be a sanity check here that checks the stack trace of this constructor + // invocation; unfortunately, that makes the options construction about 10x slower. So be + // careful with how you construct options classes. + } + + /** + * Returns this options object in the form of a (new) mapping from option + * names, including inherited ones, to option values. If the public fields + * are mutated, this will be reflected in subsequent calls to {@code asMap}. + * Mutation of this map by the caller does not affect this options object. + */ + public final Map<String, Object> asMap() { + return OptionsParserImpl.optionsAsMap(this); + } + + @Override + public final String toString() { + return getClass().getName() + asMap(); + } + + /** + * Returns a string that uniquely identifies the options. This value is + * intended for analysis caching. + */ + public final String cacheKey() { + StringBuilder result = new StringBuilder(getClass().getName()).append("{"); + + for (Entry<String, Object> entry : asMap().entrySet()) { + result.append(entry.getKey()).append("="); + + Object value = entry.getValue(); + // This special case is needed because List.toString() prints the same + // ("[]") for an empty list and for a list with a single empty string. + if (value instanceof List<?> && ((List<?>) value).isEmpty()) { + result.append("EMPTY"); + } else if (value == null) { + result.append("NULL"); + } else { + result + .append('"') + .append(ESCAPER.escape(value.toString())) + .append('"'); + } + result.append(", "); + } + + return result.append("}").toString(); + } + + @Override + public final boolean equals(Object that) { + return that != null && + this.getClass() == that.getClass() && + this.asMap().equals(((OptionsBase) that).asMap()); + } + + @Override + public final int hashCode() { + return this.getClass().hashCode() + asMap().hashCode(); + } +}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsClassProvider.java b/src/main/java/com/google/devtools/common/options/OptionsClassProvider.java new file mode 100644 index 0000000..1868e23 --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/OptionsClassProvider.java
@@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.common.options; + +/** + * A read-only interface for options parser results, which only allows to query the options of + * a specific class, but not e.g. the residue any other information pertaining to the command line. + */ +public interface OptionsClassProvider { + /** + * Returns the options instance for the given {@code optionsClass}, that is, + * the parsed options, or null if it is not among those available. + * + * <p>The returned options should be treated by library code as immutable and + * a provider is permitted to return the same options instance multiple times. + */ + <O extends OptionsBase> O getOptions(Class<O> optionsClass); +}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsData.java b/src/main/java/com/google/devtools/common/options/OptionsData.java new file mode 100644 index 0000000..e9b6574 --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/OptionsData.java
@@ -0,0 +1,264 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.common.options; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import javax.annotation.concurrent.Immutable; + +/** + * An immutable selection of options data corresponding to a set of options + * classes. The data is collected using reflection, which can be expensive. + * Therefore this class can be used internally to cache the results. + */ +@Immutable +final class OptionsData { + + /** + * These are the options-declaring classes which are annotated with + * {@link Option} annotations. + */ + private final Map<Class<? extends OptionsBase>, Constructor<?>> optionsClasses; + + /** Maps option name to Option-annotated Field. */ + private final Map<String, Field> nameToField; + + /** Maps option abbreviation to Option-annotated Field. */ + private final Map<Character, Field> abbrevToField; + + /** + * For each options class, contains a list of all Option-annotated fields in + * that class. + */ + private final Map<Class<? extends OptionsBase>, List<Field>> allOptionsFields; + + /** + * Mapping from each Option-annotated field to the default value for that + * field. + */ + private final Map<Field, Object> optionDefaults; + + /** + * Mapping from each Option-annotated field to the proper converter. + * + * @see OptionsParserImpl#findConverter + */ + private final Map<Field, Converter<?>> converters; + + private OptionsData(Map<Class<? extends OptionsBase>, Constructor<?>> optionsClasses, + Map<String, Field> nameToField, + Map<Character, Field> abbrevToField, + Map<Class<? extends OptionsBase>, List<Field>> allOptionsFields, + Map<Field, Object> optionDefaults, + Map<Field, Converter<?>> converters) { + this.optionsClasses = ImmutableMap.copyOf(optionsClasses); + this.allOptionsFields = ImmutableMap.copyOf(allOptionsFields); + this.nameToField = ImmutableMap.copyOf(nameToField); + this.abbrevToField = ImmutableMap.copyOf(abbrevToField); + // Can't use an ImmutableMap here because of null values. + this.optionDefaults = Collections.unmodifiableMap(optionDefaults); + this.converters = ImmutableMap.copyOf(converters); + } + + public Collection<Class<? extends OptionsBase>> getOptionsClasses() { + return optionsClasses.keySet(); + } + + @SuppressWarnings("unchecked") // The construction ensures that the case is always valid. + public <T extends OptionsBase> Constructor<T> getConstructor(Class<T> clazz) { + return (Constructor<T>) optionsClasses.get(clazz); + } + + public Field getFieldFromName(String name) { + return nameToField.get(name); + } + + public Iterable<Map.Entry<String, Field>> getAllNamedFields() { + return nameToField.entrySet(); + } + + public Field getFieldForAbbrev(char abbrev) { + return abbrevToField.get(abbrev); + } + + public List<Field> getFieldsForClass(Class<? extends OptionsBase> optionsClass) { + return allOptionsFields.get(optionsClass); + } + + public Object getDefaultValue(Field field) { + return optionDefaults.get(field); + } + + public Converter<?> getConverter(Field field) { + return converters.get(field); + } + + private static List<Field> getAllAnnotatedFields(Class<? extends OptionsBase> optionsClass) { + List<Field> allFields = Lists.newArrayList(); + for (Field field : optionsClass.getFields()) { + if (field.isAnnotationPresent(Option.class)) { + allFields.add(field); + } + } + if (allFields.isEmpty()) { + throw new IllegalStateException(optionsClass + " has no public @Option-annotated fields"); + } + return ImmutableList.copyOf(allFields); + } + + private static Object retrieveDefaultFromAnnotation(Field optionField) { + Option annotation = optionField.getAnnotation(Option.class); + // If an option can be specified multiple times, its default value is a new empty list. + if (annotation.allowMultiple()) { + return Collections.emptyList(); + } + String defaultValueString = OptionsParserImpl.getDefaultOptionString(optionField); + try { + return OptionsParserImpl.isSpecialNullDefault(defaultValueString, optionField) + ? null + : OptionsParserImpl.findConverter(optionField).convert(defaultValueString); + } catch (OptionsParsingException e) { + throw new IllegalStateException("OptionsParsingException while " + + "retrieving default for " + optionField.getName() + ": " + + e.getMessage()); + } + } + + static OptionsData of(Collection<Class<? extends OptionsBase>> classes) { + Map<Class<? extends OptionsBase>, Constructor<?>> constructorBuilder = Maps.newHashMap(); + Map<Class<? extends OptionsBase>, List<Field>> allOptionsFieldsBuilder = Maps.newHashMap(); + Map<String, Field> nameToFieldBuilder = Maps.newHashMap(); + Map<Character, Field> abbrevToFieldBuilder = Maps.newHashMap(); + Map<Field, Object> optionDefaultsBuilder = Maps.newHashMap(); + Map<Field, Converter<?>> convertersBuilder = Maps.newHashMap(); + + // Read all Option annotations: + for (Class<? extends OptionsBase> parsedOptionsClass : classes) { + try { + Constructor<? extends OptionsBase> constructor = + parsedOptionsClass.getConstructor(new Class[0]); + constructorBuilder.put(parsedOptionsClass, constructor); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException(parsedOptionsClass + + " lacks an accessible default constructor"); + } + List<Field> fields = getAllAnnotatedFields(parsedOptionsClass); + allOptionsFieldsBuilder.put(parsedOptionsClass, fields); + + for (Field field : fields) { + Option annotation = field.getAnnotation(Option.class); + + // Check that the field type is a List, and that the converter + // type matches the element type of the list. + Type fieldType = field.getGenericType(); + if (annotation.allowMultiple()) { + if (!(fieldType instanceof ParameterizedType)) { + throw new AssertionError("Type of multiple occurrence option must be a List<...>"); + } + ParameterizedType pfieldType = (ParameterizedType) fieldType; + if (pfieldType.getRawType() != List.class) { + // Throw an assertion, because this indicates an undetected type + // error in the code. + throw new AssertionError("Type of multiple occurrence option must be a List<...>"); + } + fieldType = pfieldType.getActualTypeArguments()[0]; + } + + // Get the converter return type. + @SuppressWarnings("rawtypes") + Class<? extends Converter> converter = annotation.converter(); + if (converter == Converter.class) { + Converter<?> actualConverter = OptionsParserImpl.DEFAULT_CONVERTERS.get(fieldType); + if (actualConverter == null) { + throw new AssertionError("Cannot find converter for field of type " + + field.getType() + " named " + field.getName() + + " in class " + field.getDeclaringClass().getName()); + } + converter = actualConverter.getClass(); + } + if (Modifier.isAbstract(converter.getModifiers())) { + throw new AssertionError("The converter type (" + converter + + ") must be a concrete type"); + } + Type converterResultType; + try { + Method convertMethod = converter.getMethod("convert", String.class); + converterResultType = GenericTypeHelper.getActualReturnType(converter, convertMethod); + } catch (NoSuchMethodException e) { + throw new AssertionError("A known converter object doesn't implement the convert" + + " method"); + } + + if (annotation.allowMultiple()) { + if (GenericTypeHelper.getRawType(converterResultType) == List.class) { + Type elementType = + ((ParameterizedType) converterResultType).getActualTypeArguments()[0]; + if (!GenericTypeHelper.isAssignableFrom(fieldType, elementType)) { + throw new AssertionError("If the converter return type of a multiple occurance " + + "option is a list, then the type of list elements (" + fieldType + ") must be " + + "assignable from the converter list element type (" + elementType + ")"); + } + } else { + if (!GenericTypeHelper.isAssignableFrom(fieldType, converterResultType)) { + throw new AssertionError("Type of list elements (" + fieldType + + ") for multiple occurrence option must be assignable from the converter " + + "return type (" + converterResultType + ")"); + } + } + } else { + if (!GenericTypeHelper.isAssignableFrom(fieldType, converterResultType)) { + throw new AssertionError("Type of field (" + fieldType + + ") must be assignable from the converter " + + "return type (" + converterResultType + ")"); + } + } + + if (annotation.name() == null) { + throw new AssertionError( + "Option cannot have a null name"); + } + if (nameToFieldBuilder.put(annotation.name(), field) != null) { + throw new DuplicateOptionDeclarationException( + "Duplicate option name: --" + annotation.name()); + } + if (annotation.abbrev() != '\0') { + if (abbrevToFieldBuilder.put(annotation.abbrev(), field) != null) { + throw new DuplicateOptionDeclarationException( + "Duplicate option abbrev: -" + annotation.abbrev()); + } + } + optionDefaultsBuilder.put(field, retrieveDefaultFromAnnotation(field)); + + convertersBuilder.put(field, OptionsParserImpl.findConverter(field)); + } + } + return new OptionsData(constructorBuilder, nameToFieldBuilder, abbrevToFieldBuilder, + allOptionsFieldsBuilder, optionDefaultsBuilder, convertersBuilder); + } +}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsParser.java b/src/main/java/com/google/devtools/common/options/OptionsParser.java new file mode 100644 index 0000000..9564daa --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/OptionsParser.java
@@ -0,0 +1,526 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.common.options; + +import com.google.common.base.Function; +import com.google.common.base.Functions; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * A parser for options. Typical use case in a main method: + * + * <pre> + * OptionsParser parser = OptionsParser.newOptionsParser(FooOptions.class, BarOptions.class); + * parser.parseAndExitUponError(args); + * FooOptions foo = parser.getOptions(FooOptions.class); + * BarOptions bar = parser.getOptions(BarOptions.class); + * List<String> otherArguments = parser.getResidue(); + * </pre> + * + * <p>FooOptions and BarOptions would be options specification classes, derived + * from OptionsBase, that contain fields annotated with @Option(...). + * + * <p>Alternatively, rather than calling + * {@link #parseAndExitUponError(OptionPriority, String, String[])}, + * client code may call {@link #parse(OptionPriority,String,List)}, and handle + * parser exceptions usage messages themselves. + * + * <p>This options parsing implementation has (at least) one design flaw. It + * allows both '--foo=baz' and '--foo baz' for all options except void, boolean + * and tristate options. For these, the 'baz' in '--foo baz' is not treated as + * a parameter to the option, making it is impossible to switch options between + * void/boolean/tristate and everything else without breaking backwards + * compatibility. + * + * @see Options a simpler class which you can use if you only have one options + * specification class + */ +public class OptionsParser implements OptionsProvider { + + /** + * A cache for the parsed options data. Both keys and values are immutable, so + * this is always safe. Only access this field through the {@link + * #getOptionsData} method for thread-safety! The cache is very unlikely to + * grow to a significant amount of memory, because there's only a fixed set of + * options classes on the classpath. + */ + private static final Map<ImmutableList<Class<? extends OptionsBase>>, OptionsData> optionsData = + Maps.newHashMap(); + + private static synchronized OptionsData getOptionsData( + ImmutableList<Class<? extends OptionsBase>> optionsClasses) { + OptionsData result = optionsData.get(optionsClasses); + if (result == null) { + result = OptionsData.of(optionsClasses); + optionsData.put(optionsClasses, result); + } + return result; + } + + /** + * Returns all the annotated fields for the given class, including inherited + * ones. + */ + static Collection<Field> getAllAnnotatedFields(Class<? extends OptionsBase> optionsClass) { + OptionsData data = getOptionsData(ImmutableList.<Class<? extends OptionsBase>>of(optionsClass)); + return data.getFieldsForClass(optionsClass); + } + + /** + * @see #newOptionsParser(Iterable) + */ + public static OptionsParser newOptionsParser(Class<? extends OptionsBase> class1) { + return newOptionsParser(ImmutableList.<Class<? extends OptionsBase>>of(class1)); + } + + /** + * @see #newOptionsParser(Iterable) + */ + public static OptionsParser newOptionsParser(Class<? extends OptionsBase> class1, + Class<? extends OptionsBase> class2) { + return newOptionsParser(ImmutableList.of(class1, class2)); + } + + /** + * Create a new {@link OptionsParser}. + */ + public static OptionsParser newOptionsParser( + Iterable<Class<? extends OptionsBase>> optionsClasses) { + return new OptionsParser(getOptionsData(ImmutableList.copyOf(optionsClasses))); + } + + /** + * Canonicalizes a list of options using the given option classes. The + * contract is that if the returned set of options is passed to an options + * parser with the same options classes, then that will have the same effect + * as using the original args (which are passed in here), except for cosmetic + * differences. + */ + public static List<String> canonicalize( + Collection<Class<? extends OptionsBase>> optionsClasses, List<String> args) + throws OptionsParsingException { + OptionsParser parser = new OptionsParser(optionsClasses); + parser.setAllowResidue(false); + parser.parse(args); + return parser.impl.asCanonicalizedList(); + } + + private final OptionsParserImpl impl; + private final List<String> residue = new ArrayList<String>(); + private boolean allowResidue = true; + + OptionsParser(Collection<Class<? extends OptionsBase>> optionsClasses) { + this(OptionsData.of(optionsClasses)); + } + + OptionsParser(OptionsData optionsData) { + impl = new OptionsParserImpl(optionsData); + } + + /** + * Indicates whether or not the parser will allow a non-empty residue; that + * is, iff this value is true then a call to one of the {@code parse} + * methods will throw {@link OptionsParsingException} unless + * {@link #getResidue()} is empty after parsing. + */ + public void setAllowResidue(boolean allowResidue) { + this.allowResidue = allowResidue; + } + + /** + * Indicates whether or not the parser will allow long options with a + * single-dash, instead of the usual double-dash, too, eg. -example instead of just --example. + */ + public void setAllowSingleDashLongOptions(boolean allowSingleDashLongOptions) { + this.impl.setAllowSingleDashLongOptions(allowSingleDashLongOptions); + } + + public void parseAndExitUponError(String[] args) { + parseAndExitUponError(OptionPriority.COMMAND_LINE, "unknown", args); + } + + /** + * A convenience function for use in main methods. Parses the command line + * parameters, and exits upon error. Also, prints out the usage message + * if "--help" appears anywhere within {@code args}. + */ + public void parseAndExitUponError(OptionPriority priority, String source, String[] args) { + try { + parse(priority, source, Arrays.asList(args)); + } catch (OptionsParsingException e) { + System.err.println("Error parsing command line: " + e.getMessage()); + System.err.println("Try --help."); + System.exit(2); + } + for (String arg : args) { + if (arg.equals("--help")) { + System.out.println(describeOptions(Collections.<String, String>emptyMap(), + HelpVerbosity.LONG)); + System.exit(0); + } + } + } + + /** + * The name and value of an option with additional metadata describing its + * priority, source, whether it was set via an implicit dependency, and if so, + * by which other option. + */ + public static class OptionValueDescription { + private final String name; + private final Object value; + private final OptionPriority priority; + private final String source; + private final String implicitDependant; + private final String expandedFrom; + + public OptionValueDescription(String name, Object value, + OptionPriority priority, String source, String implicitDependant, String expandedFrom) { + this.name = name; + this.value = value; + this.priority = priority; + this.source = source; + this.implicitDependant = implicitDependant; + this.expandedFrom = expandedFrom; + } + + public String getName() { + return name; + } + + public Object getValue() { + return value; + } + + public OptionPriority getPriority() { + return priority; + } + + public String getSource() { + return source; + } + + public String getImplicitDependant() { + return implicitDependant; + } + + public boolean isImplicitDependency() { + return implicitDependant != null; + } + + public String getExpansionParent() { + return expandedFrom; + } + + public boolean isExpansion() { + return expandedFrom != null; + } + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("option '").append(name).append("' "); + result.append("set to '").append(value).append("' "); + result.append("with priority ").append(priority); + if (source != null) { + result.append(" and source '").append(source).append("'"); + } + if (implicitDependant != null) { + result.append(" implicitly by "); + } + return result.toString(); + } + } + + /** + * The name and unparsed value of an option with additional metadata describing its + * priority, source, whether it was set via an implicit dependency, and if so, + * by which other option. + * + * <p>Note that the unparsed value and the source parameters can both be null. + */ + public static class UnparsedOptionValueDescription { + private final String name; + private final Field field; + private final String unparsedValue; + private final OptionPriority priority; + private final String source; + private final boolean explicit; + + public UnparsedOptionValueDescription(String name, Field field, String unparsedValue, + OptionPriority priority, String source, boolean explicit) { + this.name = name; + this.field = field; + this.unparsedValue = unparsedValue; + this.priority = priority; + this.source = source; + this.explicit = explicit; + } + + public String getName() { + return name; + } + + Field getField() { + return field; + } + + public boolean isBooleanOption() { + return field.getType().equals(boolean.class); + } + + private DocumentationLevel documentationLevel() { + Option option = field.getAnnotation(Option.class); + return OptionsParser.documentationLevel(option.category()); + } + + public boolean isDocumented() { + return documentationLevel() == DocumentationLevel.DOCUMENTED; + } + + public boolean isHidden() { + return documentationLevel() == DocumentationLevel.HIDDEN; + } + + boolean isExpansion() { + Option option = field.getAnnotation(Option.class); + return option.expansion().length > 0; + } + + boolean isImplicitRequirement() { + Option option = field.getAnnotation(Option.class); + return option.implicitRequirements().length > 0; + } + + boolean allowMultiple() { + Option option = field.getAnnotation(Option.class); + return option.allowMultiple(); + } + + public String getUnparsedValue() { + return unparsedValue; + } + + OptionPriority getPriority() { + return priority; + } + + public String getSource() { + return source; + } + + public boolean isExplicit() { + return explicit; + } + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("option '").append(name).append("' "); + result.append("set to '").append(unparsedValue).append("' "); + result.append("with priority ").append(priority); + if (source != null) { + result.append(" and source '").append(source).append("'"); + } + return result.toString(); + } + } + + /** + * The verbosity with which option help messages are displayed: short (just + * the name), medium (name, type, default, abbreviation), and long (full + * description). + */ + public enum HelpVerbosity { LONG, MEDIUM, SHORT } + + /** + * The level of documentation. Only documented options are output as part of + * the help. + * + * <p>We use 'hidden' so that options that form the protocol between the + * client and the server are not logged. + */ + enum DocumentationLevel { + DOCUMENTED, UNDOCUMENTED, HIDDEN + } + + /** + * Returns a description of all the options this parser can digest. + * In addition to {@link Option} annotations, this method also + * interprets {@link OptionsUsage} annotations which give an intuitive short + * description for the options. + * + * @param categoryDescriptions a mapping from category names to category + * descriptions. Options of the same category (see {@link + * Option#category}) will be grouped together, preceded by the description + * of the category. + * @param helpVerbosity if {@code long}, the options will be described + * verbosely, including their types, defaults and descriptions. If {@code + * medium}, the descriptions are omitted, and if {@code short}, the options + * are just enumerated. + */ + public String describeOptions(Map<String, String> categoryDescriptions, + HelpVerbosity helpVerbosity) { + StringBuilder desc = new StringBuilder(); + if (!impl.getOptionsClasses().isEmpty()) { + + List<Field> allFields = Lists.newArrayList(); + for (Class<? extends OptionsBase> optionsClass : impl.getOptionsClasses()) { + allFields.addAll(impl.getAnnotatedFieldsFor(optionsClass)); + } + Collections.sort(allFields, OptionsUsage.BY_CATEGORY); + String prevCategory = null; + + for (Field optionField : allFields) { + String category = optionField.getAnnotation(Option.class).category(); + if (!category.equals(prevCategory)) { + prevCategory = category; + String description = categoryDescriptions.get(category); + if (description == null) { + description = "Options category '" + category + "'"; + } + if (documentationLevel(category) == DocumentationLevel.DOCUMENTED) { + desc.append("\n").append(description).append(":\n"); + } + } + + if (documentationLevel(prevCategory) == DocumentationLevel.DOCUMENTED) { + OptionsUsage.getUsage(optionField, desc, helpVerbosity); + } + } + } + return desc.toString().trim(); + } + + /** + * Returns a description of the option value set by the last previous call to + * {@link #parse(OptionPriority, String, List)} that successfully set the given + * option. If the option is of type {@link List}, the description will + * correspond to any one of the calls, but not necessarily the last. + */ + public OptionValueDescription getOptionValueDescription(String name) { + return impl.getOptionValueDescription(name); + } + + static DocumentationLevel documentationLevel(String category) { + if ("undocumented".equals(category)) { + return DocumentationLevel.UNDOCUMENTED; + } else if ("hidden".equals(category)) { + return DocumentationLevel.HIDDEN; + } else { + return DocumentationLevel.DOCUMENTED; + } + } + + /** + * A convenience method, equivalent to + * {@code parse(OptionPriority.COMMAND_LINE, null, Arrays.asList(args))}. + */ + public void parse(String... args) throws OptionsParsingException { + parse(OptionPriority.COMMAND_LINE, (String) null, Arrays.asList(args)); + } + + /** + * A convenience method, equivalent to + * {@code parse(OptionPriority.COMMAND_LINE, null, args)}. + */ + public void parse(List<String> args) throws OptionsParsingException { + parse(OptionPriority.COMMAND_LINE, (String) null, args); + } + + /** + * Parses {@code args}, using the classes registered with this parser. + * {@link #getOptions(Class)} and {@link #getResidue()} return the results. + * May be called multiple times; later options override existing ones if they + * have equal or higher priority. The source of options is a free-form string + * that can be used for debugging. Strings that cannot be parsed as options + * accumulates as residue, if this parser allows it. + * + * @see OptionPriority + */ + public void parse(OptionPriority priority, String source, + List<String> args) throws OptionsParsingException { + parseWithSourceFunction(priority, Functions.constant(source), args); + } + + /** + * Parses {@code args}, using the classes registered with this parser. + * {@link #getOptions(Class)} and {@link #getResidue()} return the results. May be called + * multiple times; later options override existing ones if they have equal or higher priority. + * The source of options is given as a function that maps option names to the source of the + * option. Strings that cannot be parsed as options accumulates as* residue, if this parser + * allows it. + */ + public void parseWithSourceFunction(OptionPriority priority, + Function<? super String, String> sourceFunction, List<String> args) + throws OptionsParsingException { + Preconditions.checkNotNull(priority); + Preconditions.checkArgument(priority != OptionPriority.DEFAULT); + residue.addAll(impl.parse(priority, sourceFunction, args)); + if (!allowResidue && !residue.isEmpty()) { + String errorMsg = "Unrecognized arguments: " + Joiner.on(' ').join(residue); + throw new OptionsParsingException(errorMsg); + } + } + + @Override + public List<String> getResidue() { + return ImmutableList.copyOf(residue); + } + + /** + * Returns a list of warnings about problems encountered by previous parse calls. + */ + public List<String> getWarnings() { + return impl.getWarnings(); + } + + @Override + public <O extends OptionsBase> O getOptions(Class<O> optionsClass) { + return impl.getParsedOptions(optionsClass); + } + + @Override + public boolean containsExplicitOption(String name) { + return impl.containsExplicitOption(name); + } + + @Override + public List<UnparsedOptionValueDescription> asListOfUnparsedOptions() { + return impl.asListOfUnparsedOptions(); + } + + @Override + public List<UnparsedOptionValueDescription> asListOfExplicitOptions() { + return impl.asListOfExplicitOptions(); + } + + @Override + public List<OptionValueDescription> asListOfEffectiveOptions() { + return impl.asListOfEffectiveOptions(); + } +}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsParserImpl.java b/src/main/java/com/google/devtools/common/options/OptionsParserImpl.java new file mode 100644 index 0000000..e339dcd --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/OptionsParserImpl.java
@@ -0,0 +1,722 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.common.options; + +import com.google.common.base.Function; +import com.google.common.base.Functions; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.devtools.common.options.OptionsParser.OptionValueDescription; +import com.google.devtools.common.options.OptionsParser.UnparsedOptionValueDescription; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The implementation of the options parser. This is intentionally package + * private for full flexibility. Use {@link OptionsParser} or {@link Options} + * if you're a consumer. + */ +class OptionsParserImpl { + + /** + * A bunch of default converters in case the user doesn't specify a + * different one in the field annotation. + */ + static final Map<Class<?>, Converter<?>> DEFAULT_CONVERTERS = Maps.newHashMap(); + + static { + DEFAULT_CONVERTERS.put(String.class, new Converter<String>() { + @Override + public String convert(String input) { + return input; + } + @Override + public String getTypeDescription() { + return "a string"; + }}); + DEFAULT_CONVERTERS.put(int.class, new Converter<Integer>() { + @Override + public Integer convert(String input) throws OptionsParsingException { + try { + return Integer.decode(input); + } catch (NumberFormatException e) { + throw new OptionsParsingException("'" + input + "' is not an int"); + } + } + @Override + public String getTypeDescription() { + return "an integer"; + }}); + DEFAULT_CONVERTERS.put(double.class, new Converter<Double>() { + @Override + public Double convert(String input) throws OptionsParsingException { + try { + return Double.parseDouble(input); + } catch (NumberFormatException e) { + throw new OptionsParsingException("'" + input + "' is not a double"); + } + } + @Override + public String getTypeDescription() { + return "a double"; + }}); + DEFAULT_CONVERTERS.put(boolean.class, new Converters.BooleanConverter()); + DEFAULT_CONVERTERS.put(TriState.class, new Converter<TriState>() { + @Override + public TriState convert(String input) throws OptionsParsingException { + if (input == null) { + return TriState.AUTO; + } + input = input.toLowerCase(); + if (input.equals("auto")) { + return TriState.AUTO; + } + if (input.equals("true") || input.equals("1") || input.equals("yes") || + input.equals("t") || input.equals("y")) { + return TriState.YES; + } + if (input.equals("false") || input.equals("0") || input.equals("no") || + input.equals("f") || input.equals("n")) { + return TriState.NO; + } + throw new OptionsParsingException("'" + input + "' is not a boolean"); + } + @Override + public String getTypeDescription() { + return "a tri-state (auto, yes, no)"; + }}); + DEFAULT_CONVERTERS.put(Void.class, new Converter<Void>() { + @Override + public Void convert(String input) throws OptionsParsingException { + if (input == null) { + return null; // expected input, return is unused so null is fine. + } + throw new OptionsParsingException("'" + input + "' unexpected"); + } + @Override + public String getTypeDescription() { + return ""; + }}); + DEFAULT_CONVERTERS.put(long.class, new Converter<Long>() { + @Override + public Long convert(String input) throws OptionsParsingException { + try { + return Long.decode(input); + } catch (NumberFormatException e) { + throw new OptionsParsingException("'" + input + "' is not a long"); + } + } + @Override + public String getTypeDescription() { + return "a long integer"; + }}); + } + + /** + * For every value, this class keeps track of its priority, its free-form source + * description, whether it was set as an implicit dependency, and the value. + */ + private static final class ParsedOptionEntry { + private final Object value; + private final OptionPriority priority; + private final String source; + private final String implicitDependant; + private final String expandedFrom; + private final boolean allowMultiple; + + ParsedOptionEntry(Object value, + OptionPriority priority, String source, String implicitDependant, String expandedFrom, + boolean allowMultiple) { + this.value = value; + this.priority = priority; + this.source = source; + this.implicitDependant = implicitDependant; + this.expandedFrom = expandedFrom; + this.allowMultiple = allowMultiple; + } + + // Need to suppress unchecked warnings, because the "multiple occurrence" + // options use unchecked ListMultimaps due to limitations of Java generics. + @SuppressWarnings({"unchecked", "rawtypes"}) + Object getValue() { + if (allowMultiple) { + // Sort the results by option priority and return them in a new list. + // The generic type of the list is not known at runtime, so we can't + // use it here. It was already checked in the constructor, so this is + // type-safe. + List result = Lists.newArrayList(); + ListMultimap realValue = (ListMultimap) value; + for (OptionPriority priority : OptionPriority.values()) { + // If there is no mapping for this key, this check avoids object creation (because + // ListMultimap has to return a new object on get) and also an unnecessary addAll call. + if (realValue.containsKey(priority)) { + result.addAll(realValue.get(priority)); + } + } + return result; + } + return value; + } + + // Need to suppress unchecked warnings, because the "multiple occurrence" + // options use unchecked ListMultimaps due to limitations of Java generics. + @SuppressWarnings({"unchecked", "rawtypes"}) + void addValue(OptionPriority addedPriority, Object addedValue) { + Preconditions.checkState(allowMultiple); + ListMultimap optionValueList = (ListMultimap) value; + if (addedValue instanceof List<?>) { + for (Object element : (List<?>) addedValue) { + optionValueList.put(addedPriority, element); + } + } else { + optionValueList.put(addedPriority, addedValue); + } + } + + OptionValueDescription asOptionValueDescription(String fieldName) { + return new OptionValueDescription(fieldName, getValue(), priority, + source, implicitDependant, expandedFrom); + } + } + + private final OptionsData optionsData; + + /** + * We store the results of parsing the arguments in here. It'll look like + * <pre> + * Field("--host") -> "www.google.com" + * Field("--port") -> 80 + * </pre> + * This map is modified by repeated calls to + * {@link #parse(OptionPriority,Function,List)}. + */ + private final Map<Field, ParsedOptionEntry> parsedValues = Maps.newHashMap(); + + /** + * We store the pre-parsed, explicit options for each priority in here. + * We use partially preparsed options, which can be different from the original + * representation, e.g. "--nofoo" becomes "--foo=0". + */ + private final List<UnparsedOptionValueDescription> unparsedValues = + Lists.newArrayList(); + + private final List<String> warnings = Lists.newArrayList(); + + private boolean allowSingleDashLongOptions = false; + + /** + * Create a new parser object + */ + OptionsParserImpl(OptionsData optionsData) { + this.optionsData = optionsData; + } + + /** + * Indicates whether or not the parser will allow long options with a + * single-dash, instead of the usual double-dash, too, eg. -example instead of just --example. + */ + void setAllowSingleDashLongOptions(boolean allowSingleDashLongOptions) { + this.allowSingleDashLongOptions = allowSingleDashLongOptions; + } + + /** + * The implementation of {@link OptionsBase#asMap}. + */ + static Map<String, Object> optionsAsMap(OptionsBase optionsInstance) { + Map<String, Object> map = Maps.newHashMap(); + for (Field field : OptionsParser.getAllAnnotatedFields(optionsInstance.getClass())) { + try { + String name = field.getAnnotation(Option.class).name(); + Object value = field.get(optionsInstance); + map.put(name, value); + } catch (IllegalAccessException e) { + throw new IllegalStateException(e); // unreachable + } + } + return map; + } + + List<Field> getAnnotatedFieldsFor(Class<? extends OptionsBase> clazz) { + return optionsData.getFieldsForClass(clazz); + } + + /** + * Implements {@link OptionsParser#asListOfUnparsedOptions()}. + */ + List<UnparsedOptionValueDescription> asListOfUnparsedOptions() { + List<UnparsedOptionValueDescription> result = Lists.newArrayList(unparsedValues); + // It is vital that this sort is stable so that options on the same priority are not reordered. + Collections.sort(result, new Comparator<UnparsedOptionValueDescription>() { + @Override + public int compare(UnparsedOptionValueDescription o1, + UnparsedOptionValueDescription o2) { + return o1.getPriority().compareTo(o2.getPriority()); + } + }); + return result; + } + + /** + * Implements {@link OptionsParser#asListOfExplicitOptions()}. + */ + List<UnparsedOptionValueDescription> asListOfExplicitOptions() { + List<UnparsedOptionValueDescription> result = Lists.newArrayList(Iterables.filter( + unparsedValues, + new Predicate<UnparsedOptionValueDescription>() { + @Override + public boolean apply(UnparsedOptionValueDescription input) { + return input.isExplicit(); + } + })); + // It is vital that this sort is stable so that options on the same priority are not reordered. + Collections.sort(result, new Comparator<UnparsedOptionValueDescription>() { + @Override + public int compare(UnparsedOptionValueDescription o1, + UnparsedOptionValueDescription o2) { + return o1.getPriority().compareTo(o2.getPriority()); + } + }); + return result; + } + + /** + * Implements {@link OptionsParser#canonicalize}. + */ + List<String> asCanonicalizedList() { + List<UnparsedOptionValueDescription> processed = Lists.newArrayList(unparsedValues); + Collections.sort(processed, new Comparator<UnparsedOptionValueDescription>() { + // This Comparator sorts implicit requirement options to the end, keeping their existing + // order, and sorts the other options alphabetically. + @Override + public int compare(UnparsedOptionValueDescription o1, + UnparsedOptionValueDescription o2) { + if (o1.isImplicitRequirement()) { + return o2.isImplicitRequirement() ? 0 : 1; + } + if (o2.isImplicitRequirement()) { + return -1; + } + return o1.getName().compareTo(o2.getName()); + } + }); + + List<String> result = Lists.newArrayList(); + for (int i = 0; i < processed.size(); i++) { + UnparsedOptionValueDescription value = processed.get(i); + // Skip an option if the next option is the same, but only if the option does not allow + // multiple values. + if (!value.allowMultiple()) { + if ((i < processed.size() - 1) && value.getName().equals(processed.get(i + 1).getName())) { + continue; + } + } + + // Ignore expansion options. + if (value.isExpansion()) { + continue; + } + + result.add("--" + value.getName() + "=" + value.getUnparsedValue()); + } + return result; + } + + /** + * Implements {@link OptionsParser#asListOfEffectiveOptions()}. + */ + List<OptionValueDescription> asListOfEffectiveOptions() { + List<OptionValueDescription> result = Lists.newArrayList(); + for (Map.Entry<String,Field> mapEntry : optionsData.getAllNamedFields()) { + String fieldName = mapEntry.getKey(); + Field field = mapEntry.getValue(); + ParsedOptionEntry entry = parsedValues.get(field); + if (entry == null) { + Object value = optionsData.getDefaultValue(field); + result.add(new OptionValueDescription(fieldName, value, OptionPriority.DEFAULT, + null, null, null)); + } else { + result.add(entry.asOptionValueDescription(fieldName)); + } + } + return result; + } + + Collection<Class<? extends OptionsBase>> getOptionsClasses() { + return optionsData.getOptionsClasses(); + } + + private void maybeAddDeprecationWarning(Field field) { + Option option = field.getAnnotation(Option.class); + // Continue to support the old behavior for @Deprecated options. + String warning = option.deprecationWarning(); + if (!warning.equals("") || (field.getAnnotation(Deprecated.class) != null)) { + warnings.add("Option '" + option.name() + "' is deprecated" + + (warning.equals("") ? "" : ": " + warning)); + } + } + + // Warnings should not end with a '.' because the internal reporter adds one automatically. + private void setValue(Field field, String name, Object value, + OptionPriority priority, String source, String implicitDependant, String expandedFrom) { + ParsedOptionEntry entry = parsedValues.get(field); + if (entry != null) { + // Override existing option if the new value has higher or equal priority. + if (priority.compareTo(entry.priority) >= 0) { + // Output warnings: + if ((implicitDependant != null) && (entry.implicitDependant != null)) { + if (!implicitDependant.equals(entry.implicitDependant)) { + warnings.add("Option '" + name + "' is implicitly defined by both option '" + + entry.implicitDependant + "' and option '" + implicitDependant + "'"); + } + } else if ((implicitDependant != null) && priority.equals(entry.priority)) { + warnings.add("Option '" + name + "' is implicitly defined by option '" + + implicitDependant + "'; the implicitly set value overrides the previous one"); + } else if (entry.implicitDependant != null) { + warnings.add("A new value for option '" + name + "' overrides a previous " + + "implicit setting of that option by option '" + entry.implicitDependant + "'"); + } else if ((priority == entry.priority) && + ((entry.expandedFrom == null) && (expandedFrom != null))) { + // Create a warning if an expansion option overrides an explicit option: + warnings.add("The option '" + expandedFrom + "' was expanded and now overrides a " + + "previous explicitly specified option '" + name + "'"); + } + + // Record the new value: + parsedValues.put(field, + new ParsedOptionEntry(value, priority, source, implicitDependant, expandedFrom, false)); + } + } else { + parsedValues.put(field, + new ParsedOptionEntry(value, priority, source, implicitDependant, expandedFrom, false)); + maybeAddDeprecationWarning(field); + } + } + + private void addListValue(Field field, String name, Object value, + OptionPriority priority, String source, String implicitDependant, String expandedFrom) { + ParsedOptionEntry entry = parsedValues.get(field); + if (entry == null) { + entry = new ParsedOptionEntry(ArrayListMultimap.create(), priority, source, + implicitDependant, expandedFrom, true); + parsedValues.put(field, entry); + maybeAddDeprecationWarning(field); + } + entry.addValue(priority, value); + } + + private Object getValue(Field field) { + ParsedOptionEntry entry = parsedValues.get(field); + return entry == null ? null : entry.getValue(); + } + + OptionValueDescription getOptionValueDescription(String name) { + Field field = optionsData.getFieldFromName(name); + if (field == null) { + throw new IllegalArgumentException("No such option '" + name + "'"); + } + ParsedOptionEntry entry = parsedValues.get(field); + if (entry == null) { + return null; + } + return entry.asOptionValueDescription(name); + } + + boolean containsExplicitOption(String name) { + Field field = optionsData.getFieldFromName(name); + if (field == null) { + throw new IllegalArgumentException("No such option '" + name + "'"); + } + return parsedValues.get(field) != null; + } + + /** + * Parses the args, and returns what it doesn't parse. May be called multiple + * times, and may be called recursively. In each call, there may be no + * duplicates, but separate calls may contain intersecting sets of options; in + * that case, the arg seen last takes precedence. + */ + List<String> parse(OptionPriority priority, Function<? super String, String> sourceFunction, + List<String> args) throws OptionsParsingException { + return parse(priority, sourceFunction, null, null, args); + } + + /** + * Parses the args, and returns what it doesn't parse. May be called multiple + * times, and may be called recursively. Calls may contain intersecting sets + * of options; in that case, the arg seen last takes precedence. + * + * <p>The method uses the invariant that if an option has neither an implicit + * dependant nor an expanded from value, then it must have been explicitly + * set. + */ + private List<String> parse(OptionPriority priority, + final Function<? super String, String> sourceFunction, String implicitDependant, + String expandedFrom, List<String> args) throws OptionsParsingException { + List<String> unparsedArgs = Lists.newArrayList(); + LinkedHashMap<String,List<String>> implicitRequirements = Maps.newLinkedHashMap(); + for (int pos = 0; pos < args.size(); pos++) { + String arg = args.get(pos); + if (!arg.startsWith("-")) { + unparsedArgs.add(arg); + continue; // not an option arg + } + if (arg.equals("--")) { // "--" means all remaining args aren't options + while (++pos < args.size()) { + unparsedArgs.add(args.get(pos)); + } + break; + } + + String value = null; + Field field; + boolean booleanValue = true; + + if (arg.length() == 2) { // -l (may be nullary or unary) + field = optionsData.getFieldForAbbrev(arg.charAt(1)); + booleanValue = true; + + } else if (arg.length() == 3 && arg.charAt(2) == '-') { // -l- (boolean) + field = optionsData.getFieldForAbbrev(arg.charAt(1)); + booleanValue = false; + + } else if (allowSingleDashLongOptions // -long_option + || arg.startsWith("--")) { // or --long_option + int equalsAt = arg.indexOf('='); + int nameStartsAt = arg.startsWith("--") ? 2 : 1; + String name = + equalsAt == -1 ? arg.substring(nameStartsAt) : arg.substring(nameStartsAt, equalsAt); + if (name.trim().equals("")) { + throw new OptionsParsingException("Invalid options syntax: " + arg, arg); + } + value = equalsAt == -1 ? null : arg.substring(equalsAt + 1); + field = optionsData.getFieldFromName(name); + + // look for a "no"-prefixed option name: "no<optionname>"; + // (Undocumented: we also allow --no_foo. We're generous like that.) + if (field == null && name.startsWith("no")) { + String realname = name.substring(name.startsWith("no_") ? 3 : 2); + field = optionsData.getFieldFromName(realname); + booleanValue = false; + if (field != null) { + // TODO(bazel-team): Add tests for these cases. + if (!OptionsParserImpl.isBooleanField(field)) { + throw new OptionsParsingException( + "Illegal use of 'no' prefix on non-boolean option: " + arg, arg); + } + if (value != null) { + throw new OptionsParsingException( + "Unexpected value after boolean option: " + arg, arg); + } + // "no<optionname>" signifies a boolean option w/ false value + value = "0"; + } + } + + } else { + throw new OptionsParsingException("Invalid options syntax: " + arg, arg); + } + + if (field == null) { + throw new OptionsParsingException("Unrecognized option: " + arg, arg); + } + + if (value == null) { + // special case boolean to supply value based on presence of "no" prefix + if (OptionsParserImpl.isBooleanField(field)) { + value = booleanValue ? "1" : "0"; + } else if (field.getType().equals(Void.class)) { + // this is expected, Void type options have no args + } else if (pos != args.size() - 1) { + value = args.get(++pos); // "--flag value" form + } else { + throw new OptionsParsingException("Expected value after " + arg); + } + } + + Option option = field.getAnnotation(Option.class); + final String originalName = option.name(); + if (implicitDependant == null) { + // Log explicit options and expanded options in the order they are parsed (can be sorted + // later). Also remember whether they were expanded or not. This information is needed to + // correctly canonicalize flags. + unparsedValues.add(new UnparsedOptionValueDescription(originalName, field, value, + priority, sourceFunction.apply(originalName), expandedFrom == null)); + } + + // Handle expansion options. + if (option.expansion().length > 0) { + Function<Object, String> expansionSourceFunction = Functions.<String>constant( + "expanded from option --" + originalName + " from " + + sourceFunction.apply(originalName)); + maybeAddDeprecationWarning(field); + List<String> unparsed = parse(priority, expansionSourceFunction, null, originalName, + ImmutableList.copyOf(option.expansion())); + if (!unparsed.isEmpty()) { + // Throw an assertion, because this indicates an error in the code that specified the + // expansion for the current option. + throw new AssertionError("Unparsed options remain after parsing expansion of " + + arg + ":" + Joiner.on(' ').join(unparsed)); + } + } else { + Converter<?> converter = optionsData.getConverter(field); + Object convertedValue; + try { + convertedValue = converter.convert(value); + } catch (OptionsParsingException e) { + // The converter doesn't know the option name, so we supply it here by + // re-throwing: + throw new OptionsParsingException("While parsing option " + arg + + ": " + e.getMessage(), e); + } + + // ...but allow duplicates of single-use options across separate calls to + // parse(); latest wins: + if (!option.allowMultiple()) { + setValue(field, originalName, convertedValue, + priority, sourceFunction.apply(originalName), implicitDependant, expandedFrom); + } else { + // But if it's a multiple-use option, then just accumulate the + // values, in the order in which they were seen. + // Note: The type of the list member is not known; Java introspection + // only makes it available in String form via the signature string + // for the field declaration. + addListValue(field, originalName, convertedValue, + priority, sourceFunction.apply(originalName), implicitDependant, expandedFrom); + } + } + + // Collect any implicit requirements. + if (option.implicitRequirements().length > 0) { + implicitRequirements.put(option.name(), Arrays.asList(option.implicitRequirements())); + } + } + + // Now parse any implicit requirements that were collected. + // TODO(bazel-team): this should happen when the option is encountered. + if (!implicitRequirements.isEmpty()) { + for (Map.Entry<String,List<String>> entry : implicitRequirements.entrySet()) { + Function<Object, String> requirementSourceFunction = Functions.<String>constant( + "implicit requirement of option --" + entry.getKey() + " from " + + sourceFunction.apply(entry.getKey())); + + List<String> unparsed = parse(priority, requirementSourceFunction, entry.getKey(), null, + entry.getValue()); + if (!unparsed.isEmpty()) { + // Throw an assertion, because this indicates an error in the code that specified in the + // implicit requirements for the option(s). + throw new AssertionError("Unparsed options remain after parsing implicit options:" + + Joiner.on(' ').join(unparsed)); + } + } + } + + return unparsedArgs; + } + + /** + * Gets the result of parsing the options. + */ + <O extends OptionsBase> O getParsedOptions(Class<O> optionsClass) { + // Create the instance: + O optionsInstance; + try { + Constructor<O> constructor = optionsData.getConstructor(optionsClass); + if (constructor == null) { + return null; + } + optionsInstance = constructor.newInstance(new Object[0]); + } catch (Exception e) { + throw new IllegalStateException(e); + } + + // Set the fields + for (Field field : optionsData.getFieldsForClass(optionsClass)) { + Object value = getValue(field); + if (value == null) { + value = optionsData.getDefaultValue(field); + } + try { + field.set(optionsInstance, value); + } catch (IllegalAccessException e) { + throw new IllegalStateException(e); + } + } + return optionsInstance; + } + + List<String> getWarnings() { + return ImmutableList.copyOf(warnings); + } + + static String getDefaultOptionString(Field optionField) { + Option annotation = optionField.getAnnotation(Option.class); + return annotation.defaultValue(); + } + + static boolean isBooleanField(Field field) { + return field.getType().equals(boolean.class) || field.getType().equals(TriState.class); + } + + static boolean isSpecialNullDefault(String defaultValueString, Field optionField) { + return defaultValueString.equals("null") && !optionField.getType().isPrimitive(); + } + + static Converter<?> findConverter(Field optionField) { + Option annotation = optionField.getAnnotation(Option.class); + if (annotation.converter() == Converter.class) { + Type type; + if (annotation.allowMultiple()) { + // The OptionParserImpl already checked that the type is List<T> for some T; + // here we extract the type T. + type = ((ParameterizedType) optionField.getGenericType()).getActualTypeArguments()[0]; + } else { + type = optionField.getType(); + } + Converter<?> converter = DEFAULT_CONVERTERS.get(type); + if (converter == null) { + throw new AssertionError("No converter found for " + + type + "; possible fix: add " + + "converter=... to @Option annotation for " + + optionField.getName()); + } + return converter; + } + try { + Class<?> converter = annotation.converter(); + Constructor<?> constructor = converter.getConstructor(new Class<?>[0]); + return (Converter<?>) constructor.newInstance(new Object[0]); + } catch (Exception e) { + throw new AssertionError(e); + } + } + +}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsParsingException.java b/src/main/java/com/google/devtools/common/options/OptionsParsingException.java new file mode 100644 index 0000000..9d2916a --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/OptionsParsingException.java
@@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.common.options; + +/** + * An exception that's thrown when the {@link OptionsParser} fails. + * + * @see OptionsParser#parse(OptionPriority,String,java.util.List) + */ +public class OptionsParsingException extends Exception { + private final String invalidArgument; + + public OptionsParsingException(String message) { + this(message, (String) null); + } + + public OptionsParsingException(String message, String argument) { + super(message); + this.invalidArgument = argument; + } + + public OptionsParsingException(String message, Throwable throwable) { + this(message, null, throwable); + } + + public OptionsParsingException(String message, String argument, Throwable throwable) { + super(message, throwable); + this.invalidArgument = argument; + } + + /** + * Gets the name of the invalid argument or {@code null} if the exception + * can not determine the exact invalid arguments + */ + public String getInvalidArgument() { + return invalidArgument; + } +}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsProvider.java b/src/main/java/com/google/devtools/common/options/OptionsProvider.java new file mode 100644 index 0000000..be399a7 --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/OptionsProvider.java
@@ -0,0 +1,67 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.common.options; + +import com.google.devtools.common.options.OptionsParser.OptionValueDescription; +import com.google.devtools.common.options.OptionsParser.UnparsedOptionValueDescription; + +import java.util.List; + +/** + * A read-only interface for options parser results, which does not allow any + * further parsing of options. + */ +public interface OptionsProvider extends OptionsClassProvider { + + /** + * Returns an immutable copy of the residue, that is, the arguments that + * have not been parsed. + */ + List<String> getResidue(); + + /** + * Returns if the named option was specified explicitly in a call to parse. + */ + boolean containsExplicitOption(String string); + + /** + * Returns a mutable copy of the list of all options that were specified + * either explicitly or implicitly. These options are sorted by priority, and + * by the order in which they were specified. If an option was specified + * multiple times, it is included in the result multiple times. Does not + * include the residue. + * + * <p>The returned list can be filtered if undocumented, hidden or implicit + * options should not be displayed. + */ + List<UnparsedOptionValueDescription> asListOfUnparsedOptions(); + + /** + * Returns a list of all explicitly specified options, suitable for logging + * or for displaying back to the user. These options are sorted by priority, + * and by the order in which they were specified. If an option was + * explicitly specified multiple times, it is included in the result + * multiple times. Does not include the residue. + * + * <p>The list includes undocumented options. + */ + public List<UnparsedOptionValueDescription> asListOfExplicitOptions(); + + /** + * Returns a list of all options, including undocumented ones, and their + * effective values. There is no guaranteed ordering for the result. + */ + public List<OptionValueDescription> asListOfEffectiveOptions(); +}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsUsage.java b/src/main/java/com/google/devtools/common/options/OptionsUsage.java new file mode 100644 index 0000000..c48a532 --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/OptionsUsage.java
@@ -0,0 +1,156 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.common.options; + +import static com.google.devtools.common.options.OptionsParserImpl.findConverter; + +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; + +import java.lang.reflect.Field; +import java.text.BreakIterator; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +/** + * A renderer for usage messages. For now this is very simple. + */ +class OptionsUsage { + + private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n'); + + /** + * Given an options class, render the usage string into the usage, + * which is passed in as an argument. + */ + static void getUsage(Class<? extends OptionsBase> optionsClass, StringBuilder usage) { + List<Field> optionFields = + Lists.newArrayList(OptionsParser.getAllAnnotatedFields(optionsClass)); + Collections.sort(optionFields, BY_NAME); + for (Field optionField : optionFields) { + getUsage(optionField, usage, OptionsParser.HelpVerbosity.LONG); + } + } + + /** + * Paragraph-fill the specified input text, indenting lines to 'indent' and + * wrapping lines at 'width'. Returns the formatted result. + */ + static String paragraphFill(String in, int indent, int width) { + String indentString = Strings.repeat(" ", indent); + StringBuilder out = new StringBuilder(); + String sep = ""; + for (String paragraph : NEWLINE_SPLITTER.split(in)) { + BreakIterator boundary = BreakIterator.getLineInstance(); // (factory) + boundary.setText(paragraph); + out.append(sep).append(indentString); + int cursor = indent; + for (int start = boundary.first(), end = boundary.next(); + end != BreakIterator.DONE; + start = end, end = boundary.next()) { + String word = + paragraph.substring(start, end); // (may include trailing space) + if (word.length() + cursor > width) { + out.append('\n').append(indentString); + cursor = indent; + } + out.append(word); + cursor += word.length(); + } + sep = "\n"; + } + return out.toString(); + } + + /** + * Append the usage message for a single option-field message to 'usage'. + */ + static void getUsage(Field optionField, StringBuilder usage, + OptionsParser.HelpVerbosity helpVerbosity) { + String flagName = getFlagName(optionField); + String typeDescription = getTypeDescription(optionField); + Option annotation = optionField.getAnnotation(Option.class); + usage.append(" --" + flagName); + if (helpVerbosity == OptionsParser.HelpVerbosity.SHORT) { // just the name + usage.append('\n'); + return; + } + if (annotation.abbrev() != '\0') { + usage.append(" [-").append(annotation.abbrev()).append(']'); + } + if (!typeDescription.equals("")) { + usage.append(" (" + typeDescription + "; "); + if (annotation.allowMultiple()) { + usage.append("may be used multiple times"); + } else { + // Don't call the annotation directly (we must allow overrides to certain defaults) + String defaultValueString = OptionsParserImpl.getDefaultOptionString(optionField); + if (OptionsParserImpl.isSpecialNullDefault(defaultValueString, optionField)) { + usage.append("default: see description"); + } else { + usage.append("default: \"" + defaultValueString + "\""); + } + } + usage.append(")"); + } + usage.append("\n"); + if (helpVerbosity == OptionsParser.HelpVerbosity.MEDIUM) { // just the name and type. + return; + } + if (!annotation.help().equals("")) { + usage.append(paragraphFill(annotation.help(), 4, 80)); // (indent, width) + usage.append('\n'); + } + if (annotation.expansion().length > 0) { + StringBuilder expandsMsg = new StringBuilder("Expands to: "); + for (String exp : annotation.expansion()) { + expandsMsg.append(exp).append(" "); + } + usage.append(paragraphFill(expandsMsg.toString(), 4, 80)); // (indent, width) + usage.append('\n'); + } + } + + private static final Comparator<Field> BY_NAME = new Comparator<Field>() { + @Override + public int compare(Field left, Field right) { + return left.getName().compareTo(right.getName()); + } + }; + + /** + * An ordering relation for option-field fields that first groups together + * options of the same category, then sorts by name within the category. + */ + static final Comparator<Field> BY_CATEGORY = new Comparator<Field>() { + @Override + public int compare(Field left, Field right) { + int r = left.getAnnotation(Option.class).category().compareTo( + right.getAnnotation(Option.class).category()); + return r == 0 ? BY_NAME.compare(left, right) : r; + } + }; + + private static String getTypeDescription(Field optionsField) { + return findConverter(optionsField).getTypeDescription(); + } + + static String getFlagName(Field field) { + String name = field.getAnnotation(Option.class).name(); + return OptionsParserImpl.isBooleanField(field) ? "[no]" + name : name; + } + +}
diff --git a/src/main/java/com/google/devtools/common/options/TriState.java b/src/main/java/com/google/devtools/common/options/TriState.java new file mode 100644 index 0000000..9e873ea --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/TriState.java
@@ -0,0 +1,21 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.common.options; + +/** + * Enum used to represent tri-state options (yes/no/auto). + */ +public enum TriState { + YES, NO, AUTO +}