Implement parsing Ninja targets together with variable expanding.
Closes #10236.
PiperOrigin-RevId: 281270814
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
index d56e086..1a5477f 100644
--- a/src/main/java/com/google/devtools/build/lib/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -701,6 +701,8 @@
deps = [
"//src/main/java/com/google/devtools/build/lib:util",
"//src/main/java/com/google/devtools/build/lib/concurrent",
+ "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
+ "//third_party:auto_value_value",
"//third_party:guava",
"//third_party:jsr305",
],
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexer.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexer.java
index 4c0af09..600eda4 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexer.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexer.java
@@ -43,8 +43,8 @@
private NinjaLexerStep step;
private final List<Pair<Integer, Integer>> ranges;
private final List<NinjaToken> tokens;
- /** Flag to give a hint that letters should be interpreted as text, not as identifier. */
- private boolean expectTextUntilEol;
+ /** Flag to give a hint how letters should be interpreted (as text, identifier, path). */
+ private TextKind expectedTextKind = TextKind.IDENTIFIER;
/** @param fragment fragment to do the lexing on */
public NinjaLexer(ByteBufferFragment fragment) {
@@ -90,17 +90,17 @@
step.forceError("Tabs are not allowed, use spaces.");
return push(NinjaToken.ERROR);
case '\r':
- expectTextUntilEol = false;
+ expectedTextKind = TextKind.IDENTIFIER;
step.processLineFeedNewLine();
return push(NinjaToken.NEWLINE);
case '\n':
- expectTextUntilEol = false;
+ expectedTextKind = TextKind.IDENTIFIER;
return push(NinjaToken.NEWLINE);
case '#':
step.skipComment();
break;
case '=':
- if (expectTextUntilEol) {
+ if (TextKind.TEXT.equals(expectedTextKind)) {
step.readText();
return push(NinjaToken.TEXT);
}
@@ -108,7 +108,7 @@
case ':':
return push(NinjaToken.COLON);
case '|':
- if (expectTextUntilEol) {
+ if (TextKind.TEXT.equals(expectedTextKind)) {
step.readText();
return push(NinjaToken.TEXT);
}
@@ -129,20 +129,25 @@
step.forceError("Bad $-escape (literal $ must be written as $$)");
return push(NinjaToken.ERROR);
default:
- if (expectTextUntilEol) {
- step.readText();
- return push(NinjaToken.TEXT);
- } else {
- step.tryReadIdentifier();
- if (step.getError() == null) {
- byte[] bytes = step.getBytes();
- NinjaToken keywordToken = KEYWORD_MAP.get(bytes[0]);
- if (keywordToken != null && Arrays.equals(keywordToken.getBytes(), bytes)) {
- return push(keywordToken);
+ switch (expectedTextKind) {
+ case TEXT:
+ step.readText();
+ return push(NinjaToken.TEXT);
+ case PATH:
+ step.readPath();
+ return push(NinjaToken.TEXT);
+ case IDENTIFIER:
+ step.tryReadIdentifier();
+ if (step.getError() == null) {
+ byte[] bytes = step.getBytes();
+ NinjaToken keywordToken = KEYWORD_MAP.get(bytes[0]);
+ if (keywordToken != null && Arrays.equals(keywordToken.getBytes(), bytes)) {
+ return push(keywordToken);
+ }
}
- }
- return push(NinjaToken.IDENTIFIER);
+ return push(NinjaToken.IDENTIFIER);
}
+ throw new IllegalStateException();
}
if (step.canAdvance()) {
step.ensureEnd();
@@ -193,9 +198,9 @@
return Preconditions.checkNotNull(Iterables.getLast(ranges).getSecond());
}
- /** Give a hint that letters should be interpreted as text, not as identifier. */
- public void expectTextUntilEol() {
- this.expectTextUntilEol = true;
+ /** Give a hint how letters should be interpreted (as text, identifier, path). */
+ public void setExpectedTextKind(TextKind expectedTextKind) {
+ this.expectedTextKind = expectedTextKind;
}
/** Undo the previously read token. */
@@ -204,7 +209,7 @@
ranges.remove(ranges.size() - 1);
tokens.remove(tokens.size() - 1);
step = new NinjaLexerStep(fragment, ranges.isEmpty() ? 0 : getLastEnd());
- expectTextUntilEol = false;
+ expectedTextKind = TextKind.IDENTIFIER;
}
public String getError() {
@@ -214,4 +219,14 @@
public ByteBufferFragment getFragment() {
return fragment;
}
+
+ /**
+ * Enum with variants of text fragments parsing: as identifier (most restricted set of symbols),
+ * path (all spaces should be $-escaped, and | symbol has a special meaning), or text.
+ */
+ public enum TextKind {
+ IDENTIFIER,
+ PATH,
+ TEXT
+ }
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexerStep.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexerStep.java
index c4d2f51..41fefb1 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexerStep.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexerStep.java
@@ -47,6 +47,7 @@
private static final ImmutableSortedSet<Byte> IDENTIFIER_SYMBOLS =
createByteSet("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-");
private static final ImmutableSortedSet<Byte> TEXT_STOPPERS = createByteSet("\n\r \t#$:\u0000");
+ private static final ImmutableSortedSet<Byte> PATH_STOPPERS = createByteSet("\n\r \t#$:|\u0000");
private static ImmutableSortedSet<Byte> createByteSet(String variants) {
ImmutableSortedSet.Builder<Byte> builder = ImmutableSortedSet.naturalOrder();
@@ -238,6 +239,10 @@
end = eatSequence(position, TEXT_STOPPERS::contains);
}
+ public void readPath() {
+ end = eatSequence(position, PATH_STOPPERS::contains);
+ }
+
private int readIdentifier(int startFrom, boolean withDot) {
if (withDot) {
return eatSequence(startFrom, b -> !IDENTIFIER_SYMBOLS.contains(b) && '.' != b);
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParser.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParser.java
index cd50c4a..74f00d4 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParser.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParser.java
@@ -20,11 +20,22 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
+import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer.TextKind;
import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaToken;
+import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget.InputKind;
+import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget.InputOutputKind;
+import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget.OutputKind;
import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.PathFragment;
import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
/** Ninja files parser. The types of tokens: {@link NinjaToken}. Ninja lexer: {@link NinjaLexer}. */
public class NinjaParser {
@@ -39,24 +50,21 @@
String name = asString(parseExpected(NinjaToken.IDENTIFIER));
parseExpected(NinjaToken.EQUALS);
- NinjaVariableValue value = parseVariableValue(true, name);
+ NinjaVariableValue value = parseVariableValue(name);
return Pair.of(name, value);
}
@VisibleForTesting
- public NinjaVariableValue parseVariableValue(boolean allowUnescapedColon, String name)
- throws GenericParsingException {
- return parseVariableValueImpl(
- allowUnescapedColon, () -> String.format("Variable '%s' has no value.", name));
+ public NinjaVariableValue parseVariableValue(String name) throws GenericParsingException {
+ return parseVariableValueImpl(() -> String.format("Variable '%s' has no value.", name));
}
- private NinjaVariableValue parseVariableValueImpl(
- boolean allowUnescapedColon, Supplier<String> messageForNoValue)
+ private NinjaVariableValue parseVariableValueImpl(Supplier<String> messageForNoValue)
throws GenericParsingException {
NinjaVariableValue.Builder varBuilder = NinjaVariableValue.builder();
int previous = -1;
while (lexer.hasNextToken()) {
- lexer.expectTextUntilEol();
+ lexer.setExpectedTextKind(TextKind.TEXT);
NinjaToken token = lexer.nextToken();
if (NinjaToken.VARIABLE.equals(token)) {
if (previous >= 0) {
@@ -67,7 +75,7 @@
varBuilder.addVariable(normalizeVariableName(asString(lexer.getTokenBytes())));
} else if (NinjaToken.TEXT.equals(token)
|| NinjaToken.ESCAPED_TEXT.equals(token)
- || (allowUnescapedColon && NinjaToken.COLON.equals(token))) {
+ || NinjaToken.COLON.equals(token)) {
// Add text together with the spaces between current and previous token.
int start = previous >= 0 ? previous : lexer.getLastStart();
String rawText = asString(lexer.getFragment().getBytes(start, lexer.getLastEnd()));
@@ -86,6 +94,41 @@
return varBuilder.build();
}
+ /**
+ * Paths variable is a sequence of text and variable references until space, newline, eof or |
+ * symbol.
+ */
+ @Nullable
+ private NinjaVariableValue parsePathVariableValue() {
+ NinjaVariableValue.Builder varBuilder = NinjaVariableValue.builder();
+ int previous = -1;
+ while (lexer.hasNextToken()) {
+ lexer.setExpectedTextKind(TextKind.PATH);
+ NinjaToken token = lexer.nextToken();
+ if (previous >= 0 && lexer.getLastStart() != previous) {
+ // no spaces.
+ lexer.undo();
+ break;
+ }
+ if (NinjaToken.VARIABLE.equals(token)) {
+ varBuilder.addVariable(normalizeVariableName(asString(lexer.getTokenBytes())));
+ } else if (NinjaToken.TEXT.equals(token) || NinjaToken.ESCAPED_TEXT.equals(token)) {
+ String rawText = asString(lexer.getTokenBytes());
+ String text = NinjaToken.ESCAPED_TEXT.equals(token) ? unescapeText(rawText) : rawText;
+ varBuilder.addText(text);
+ } else {
+ lexer.undo();
+ break;
+ }
+ previous = lexer.getLastEnd();
+ }
+ if (previous == -1) {
+ // We read no value.
+ return null;
+ }
+ return varBuilder.build();
+ }
+
private static String unescapeText(String text) {
StringBuilder sb = new StringBuilder(text.length());
for (int i = 0; i < text.length(); i++) {
@@ -114,7 +157,6 @@
parseExpected(token);
NinjaVariableValue value =
parseVariableValueImpl(
- true,
() -> String.format("%s statement has no path.", Ascii.toLowerCase(token.name())));
if (lexer.hasNextToken()) {
parseExpected(NinjaToken.NEWLINE);
@@ -137,7 +179,7 @@
parseExpected(NinjaToken.INDENT);
String variableName = asString(parseExpected(NinjaToken.IDENTIFIER));
parseExpected(NinjaToken.EQUALS);
- NinjaVariableValue value = parseVariableValue(true, variableName);
+ NinjaVariableValue value = parseVariableValue(variableName);
NinjaRuleVariable ninjaRuleVariable = NinjaRuleVariable.nullOrValue(variableName);
if (ninjaRuleVariable == null) {
@@ -151,6 +193,185 @@
return new NinjaRule(variablesBuilder.build());
}
+ private enum NinjaTargetParsingPart {
+ OUTPUTS(OutputKind.USUAL, true),
+ IMPLICIT_OUTPUTS(OutputKind.IMPLICIT, true),
+ INPUTS(InputKind.USUAL, false),
+ IMPLICIT_INPUTS(InputKind.IMPLICIT, false),
+ ORDER_ONLY_INPUTS(InputKind.ORDER_ONLY, false),
+ RULE_NAME(null, false),
+ VARIABLES(null, false);
+
+ @Nullable private final InputOutputKind inputOutputKind;
+ private final boolean transitionRequired;
+
+ NinjaTargetParsingPart(@Nullable InputOutputKind inputOutputKind, boolean transitionRequired) {
+ this.inputOutputKind = inputOutputKind;
+ this.transitionRequired = transitionRequired;
+ }
+
+ @Nullable
+ public InputOutputKind getInputOutputKind() {
+ return inputOutputKind;
+ }
+
+ public boolean isTransitionRequired() {
+ return transitionRequired;
+ }
+ }
+
+ /**
+ * Mapping for changing the {@link NinjaTargetParsingPart} according to the next separator symbol.
+ */
+ private static final ImmutableSortedMap<
+ NinjaTargetParsingPart, ImmutableSortedMap<NinjaToken, NinjaTargetParsingPart>>
+ TARGET_PARTS_TRANSITIONS_MAP =
+ ImmutableSortedMap.of(
+ NinjaTargetParsingPart.OUTPUTS,
+ ImmutableSortedMap.of(
+ NinjaToken.PIPE, NinjaTargetParsingPart.IMPLICIT_OUTPUTS,
+ NinjaToken.COLON, NinjaTargetParsingPart.RULE_NAME),
+ NinjaTargetParsingPart.IMPLICIT_OUTPUTS,
+ ImmutableSortedMap.of(NinjaToken.COLON, NinjaTargetParsingPart.RULE_NAME),
+ NinjaTargetParsingPart.INPUTS,
+ ImmutableSortedMap.of(
+ NinjaToken.PIPE, NinjaTargetParsingPart.IMPLICIT_INPUTS,
+ NinjaToken.PIPE2, NinjaTargetParsingPart.ORDER_ONLY_INPUTS,
+ NinjaToken.NEWLINE, NinjaTargetParsingPart.VARIABLES),
+ NinjaTargetParsingPart.IMPLICIT_INPUTS,
+ ImmutableSortedMap.of(
+ NinjaToken.PIPE2, NinjaTargetParsingPart.ORDER_ONLY_INPUTS,
+ NinjaToken.NEWLINE, NinjaTargetParsingPart.VARIABLES),
+ NinjaTargetParsingPart.ORDER_ONLY_INPUTS,
+ ImmutableSortedMap.of(NinjaToken.NEWLINE, NinjaTargetParsingPart.VARIABLES));
+
+ /**
+ * Parses Ninja target using {@link NinjaScope} of the file, where it is defined, to expand
+ * variables.
+ */
+ public NinjaTarget parseNinjaTarget(NinjaScope fileScope, int offset)
+ throws GenericParsingException {
+ NinjaTarget.Builder builder = NinjaTarget.builder();
+ parseExpected(NinjaToken.BUILD);
+
+ Map<InputOutputKind, List<NinjaVariableValue>> pathValuesMap =
+ parseTargetDependenciesPart(builder);
+
+ NinjaScope targetScope = parseTargetVariables(offset, fileScope, builder);
+
+ // Variables from the build statement can be used in the input and output paths, so
+ // we are using targetScope to resolve paths values.
+ for (Map.Entry<InputOutputKind, List<NinjaVariableValue>> entry : pathValuesMap.entrySet()) {
+ List<PathFragment> paths =
+ entry.getValue().stream()
+ .map(
+ value ->
+ PathFragment.create(targetScope.getExpandedValue(Integer.MAX_VALUE, value)))
+ .collect(Collectors.toList());
+ InputOutputKind inputOutputKind = entry.getKey();
+ if (inputOutputKind instanceof InputKind) {
+ builder.addInputs((InputKind) inputOutputKind, paths);
+ } else {
+ builder.addOutputs((OutputKind) inputOutputKind, paths);
+ }
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * We resolve build statement variables values, using the file scope: build statement variable
+ * values can not refer to each other. Then we are constructing the target's {@link NinjaScope}
+ * with already expanded variables; it will be used for resolving target's input and output paths
+ * (which can also refer to file-level variables, so we better reuse resolve logic that we already
+ * have in NinjaScope).
+ *
+ * <p>As we expand variable values, we are adding them to {@link NinjaTarget.Builder}.
+ *
+ * <p>Ninja targets can not refer to the rule's variables values, because the rule variables are
+ * only expanded when the rule is used, and the rule is used for already parsed target. However,
+ * target's variables can override values of rule's variables.
+ *
+ * @return Ninja scope for expanding input and output paths of that statement
+ */
+ private NinjaScope parseTargetVariables(
+ int offset, NinjaScope fileScope, NinjaTarget.Builder builder)
+ throws GenericParsingException {
+ Map<String, List<Pair<Integer, String>>> expandedVariables = Maps.newHashMap();
+ while (lexer.hasNextToken()) {
+ parseExpected(NinjaToken.INDENT);
+
+ Pair<String, NinjaVariableValue> pair = parseVariable();
+ String name = Preconditions.checkNotNull(pair.getFirst());
+ NinjaVariableValue value = Preconditions.checkNotNull(pair.getSecond());
+ String expandedValue = fileScope.getExpandedValue(offset, value);
+ expandedVariables
+ .computeIfAbsent(name, k -> Lists.newArrayList())
+ .add(Pair.of(0, expandedValue));
+ builder.addVariable(name, expandedValue);
+
+ if (lexer.hasNextToken()) {
+ parseExpected(NinjaToken.NEWLINE);
+ }
+ }
+ return fileScope.createTargetsScope(ImmutableSortedMap.copyOf(expandedVariables));
+ }
+
+ /**
+ * Parse build statement dependencies part: output1..k [| implicit_output1..k]: rule input1..k [|
+ * implicit_input1..k] [|| order_only_input1..k]
+ */
+ private Map<InputOutputKind, List<NinjaVariableValue>> parseTargetDependenciesPart(
+ NinjaTarget.Builder builder) throws GenericParsingException {
+ Map<InputOutputKind, List<NinjaVariableValue>> pathValuesMap = Maps.newHashMap();
+ boolean ruleNameParsed = false;
+ NinjaTargetParsingPart parsingPart = NinjaTargetParsingPart.OUTPUTS;
+ while (lexer.hasNextToken() && !NinjaTargetParsingPart.VARIABLES.equals(parsingPart)) {
+ if (NinjaTargetParsingPart.RULE_NAME.equals(parsingPart)) {
+ ruleNameParsed = true;
+ builder.setRuleName(asString(parseExpected(NinjaToken.IDENTIFIER)));
+ parsingPart = NinjaTargetParsingPart.INPUTS;
+ continue;
+ }
+ List<NinjaVariableValue> paths = parsePaths();
+ if (paths.isEmpty() && !NinjaTargetParsingPart.INPUTS.equals(parsingPart)) {
+ throw new GenericParsingException("Expected paths sequence");
+ }
+ if (!paths.isEmpty()) {
+ pathValuesMap.put(Preconditions.checkNotNull(parsingPart.getInputOutputKind()), paths);
+ }
+ if (!lexer.hasNextToken()) {
+ if (parsingPart.isTransitionRequired()) {
+ throw new GenericParsingException("Unexpected end of target");
+ }
+ break;
+ }
+ NinjaToken lexicalSeparator = lexer.nextToken();
+ parsingPart =
+ Preconditions.checkNotNull(TARGET_PARTS_TRANSITIONS_MAP.get(parsingPart))
+ .get(lexicalSeparator);
+
+ if (parsingPart == null) {
+ throw new GenericParsingException("Unexpected token: " + lexicalSeparator);
+ }
+ }
+ if (!ruleNameParsed) {
+ throw new GenericParsingException("Expected rule name");
+ }
+ Preconditions.checkState(
+ !lexer.hasNextToken() || NinjaTargetParsingPart.VARIABLES.equals(parsingPart));
+ return pathValuesMap;
+ }
+
+ private List<NinjaVariableValue> parsePaths() {
+ List<NinjaVariableValue> result = Lists.newArrayList();
+ NinjaVariableValue value;
+ while (lexer.hasNextToken() && (value = parsePathVariableValue()) != null) {
+ result.add(value);
+ }
+ return result;
+ }
+
@VisibleForTesting
public static String normalizeVariableName(String raw) {
// We start from 1 because it is always at least $ marker symbol in the beginning
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaScope.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaScope.java
index 91b8275..100cce5 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaScope.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaScope.java
@@ -19,6 +19,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -45,8 +46,9 @@
private final NavigableMap<Integer, NinjaScope> includedScopes;
private final Map<String, List<Pair<Integer, NinjaVariableValue>>> variables;
- private final Map<String, List<Pair<Integer, String>>> expandedVariables;
+ private Map<String, List<Pair<Integer, String>>> expandedVariables;
private final Map<String, List<Pair<Integer, NinjaRule>>> rules;
+ private boolean isExpanded;
public NinjaScope() {
this(null, null);
@@ -61,17 +63,38 @@
rules = Maps.newHashMap();
}
+ private NinjaScope(
+ @Nullable NinjaScope parentScope,
+ @Nullable Integer includePoint,
+ ImmutableSortedMap<String, List<Pair<Integer, String>>> expandedVariables) {
+ this.parentScope = parentScope;
+ this.includePoint = includePoint;
+ this.includedScopes = Maps.newTreeMap();
+ this.variables = Maps.newHashMap();
+ this.expandedVariables = expandedVariables;
+ this.rules = Maps.newHashMap();
+ this.isExpanded = true;
+ }
+
+ NinjaScope createTargetsScope(
+ ImmutableSortedMap<String, List<Pair<Integer, String>>> expandedVariables) {
+ return new NinjaScope(this, Integer.MAX_VALUE, expandedVariables);
+ }
+
public NinjaScope createIncludeScope(int offset) {
+ Preconditions.checkState(!isExpanded);
NinjaScope includeScope = new NinjaScope(this, offset);
includedScopes.put(offset, includeScope);
return includeScope;
}
public void addVariable(String name, int offset, NinjaVariableValue value) {
+ Preconditions.checkState(!isExpanded);
variables.computeIfAbsent(name, k -> Lists.newArrayList()).add(Pair.of(offset, value));
}
public void addRule(int offset, NinjaRule rule) {
+ Preconditions.checkState(!isExpanded);
rules.computeIfAbsent(rule.getName(), k -> Lists.newArrayList()).add(Pair.of(offset, rule));
}
@@ -95,9 +118,15 @@
}
}
- private void expandVariable(String name, int offset, NinjaVariableValue value) {
- List<Pair<Integer, String>> targetList =
- expandedVariables.computeIfAbsent(name, k -> Lists.newArrayList());
+ /**
+ * Expands variable value at the given offset. If some of the variable references, used in the
+ * value, can not be found, uses an empty string as their value.
+ *
+ * <p>This method is used either on already expanded scope, or in the process of the scope
+ * expansion in {@link #expandVariables}.
+ */
+ public String getExpandedValue(int offset, NinjaVariableValue value) {
+ Preconditions.checkState(isExpanded);
// Cache expanded variables values to save time replacing several references to the same
// variable.
// This cache is local to the offset, it depends on the offset of the variable we are expanding.
@@ -106,18 +135,27 @@
// Do the same as Ninja implementation: if the variable is not found, use empty string.
Function<String, String> expander =
ref -> cache.computeIfAbsent(ref, (key) -> nullToEmpty(findExpandedVariable(offset, key)));
- targetList.add(Pair.of(offset, value.getExpandedValue(expander)));
+ return value.getExpandedValue(expander);
}
/** Resolve variables inside this scope and included scopes. */
public void expandVariables() {
+ Preconditions.checkState(!isExpanded);
+ /* To allow assertion in {@link #getExpandedValue} to pass. */
+ isExpanded = true;
+
TreeMap<Integer, Runnable> resolvables = Maps.newTreeMap();
for (Map.Entry<String, List<Pair<Integer, NinjaVariableValue>>> entry : variables.entrySet()) {
String name = entry.getKey();
for (Pair<Integer, NinjaVariableValue> pair : entry.getValue()) {
int offset = Preconditions.checkNotNull(pair.getFirst());
NinjaVariableValue variableValue = Preconditions.checkNotNull(pair.getSecond());
- resolvables.put(offset, () -> expandVariable(name, offset, variableValue));
+ resolvables.put(
+ offset,
+ () ->
+ expandedVariables
+ .computeIfAbsent(name, k -> Lists.newArrayList())
+ .add(Pair.of(offset, getExpandedValue(offset, variableValue))));
}
}
for (Map.Entry<Integer, NinjaScope> entry : includedScopes.entrySet()) {
@@ -125,6 +163,8 @@
}
resolvables.values().forEach(Runnable::run);
+ expandedVariables = ImmutableSortedMap.copyOf(expandedVariables);
+ variables.clear();
}
/**
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaTarget.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaTarget.java
new file mode 100644
index 0000000..4046918
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaTarget.java
@@ -0,0 +1,150 @@
+// Copyright 2019 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package com.google.devtools.build.lib.bazel.rules.ninja.parser;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.devtools.build.lib.collect.ImmutableSortedKeyListMultimap;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.errorprone.annotations.Immutable;
+import java.util.Collection;
+import java.util.List;
+
+/** Ninja target (build statement) representation. */
+public final class NinjaTarget {
+ /** Builder for {@link NinjaTarget}. */
+ public static class Builder {
+ private String ruleName;
+ private final ImmutableSortedKeyListMultimap.Builder<InputKind, PathFragment> inputsBuilder;
+ private final ImmutableSortedKeyListMultimap.Builder<OutputKind, PathFragment> outputsBuilder;
+
+ private final ImmutableSortedMap.Builder<String, String> variablesBuilder;
+
+ private Builder() {
+ inputsBuilder = ImmutableSortedKeyListMultimap.builder();
+ outputsBuilder = ImmutableSortedKeyListMultimap.builder();
+ variablesBuilder = ImmutableSortedMap.naturalOrder();
+ }
+
+ public Builder setRuleName(String ruleName) {
+ this.ruleName = ruleName;
+ return this;
+ }
+
+ public Builder addInputs(InputKind kind, Collection<PathFragment> inputs) {
+ inputsBuilder.putAll(kind, inputs);
+ return this;
+ }
+
+ public Builder addOutputs(OutputKind kind, Collection<PathFragment> outputs) {
+ outputsBuilder.putAll(kind, outputs);
+ return this;
+ }
+
+ public Builder addVariable(String key, String value) {
+ variablesBuilder.put(key, value);
+ return this;
+ }
+
+ public NinjaTarget build() {
+ Preconditions.checkNotNull(ruleName);
+ return new NinjaTarget(
+ ruleName, inputsBuilder.build(), outputsBuilder.build(), variablesBuilder.build());
+ }
+ }
+
+ /** Enum with possible kinds of inputs. */
+ @Immutable
+ public enum InputKind implements InputOutputKind {
+ USUAL,
+ IMPLICIT,
+ ORDER_ONLY
+ }
+
+ /** Enum with possible kinds of outputs. */
+ @Immutable
+ public enum OutputKind implements InputOutputKind {
+ USUAL,
+ IMPLICIT
+ }
+
+ /**
+ * Marker interface, so that it is possible to address {@link InputKind} and {@link OutputKind}
+ * together in one map.
+ */
+ @Immutable
+ public interface InputOutputKind {}
+
+ private final String ruleName;
+ private final ImmutableSortedKeyListMultimap<InputKind, PathFragment> inputs;
+ private final ImmutableSortedKeyListMultimap<OutputKind, PathFragment> outputs;
+ private final ImmutableSortedMap<String, String> variables;
+
+ public NinjaTarget(
+ String ruleName,
+ ImmutableSortedKeyListMultimap<InputKind, PathFragment> inputs,
+ ImmutableSortedKeyListMultimap<OutputKind, PathFragment> outputs,
+ ImmutableSortedMap<String, String> variables) {
+ this.ruleName = ruleName;
+ this.inputs = inputs;
+ this.outputs = outputs;
+ this.variables = variables;
+ }
+
+ public String getRuleName() {
+ return ruleName;
+ }
+
+ public ImmutableSortedMap<String, String> getVariables() {
+ return variables;
+ }
+
+ public boolean hasInputs() {
+ return !inputs.isEmpty();
+ }
+
+ public List<PathFragment> getOutputs() {
+ return outputs.get(OutputKind.USUAL);
+ }
+
+ public List<PathFragment> getImplicitOutputs() {
+ return outputs.get(OutputKind.IMPLICIT);
+ }
+
+ public Collection<PathFragment> getAllOutputs() {
+ return outputs.values();
+ }
+
+ public Collection<PathFragment> getAllInputs() {
+ return inputs.values();
+ }
+
+ public Collection<PathFragment> getUsualInputs() {
+ return inputs.get(InputKind.USUAL);
+ }
+
+ public Collection<PathFragment> getImplicitInputs() {
+ return inputs.get(InputKind.IMPLICIT);
+ }
+
+ public Collection<PathFragment> getOrderOnlyInputs() {
+ return inputs.get(InputKind.ORDER_ONLY);
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaVariableValue.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaVariableValue.java
index 8cc1a06..0431f92 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaVariableValue.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaVariableValue.java
@@ -15,6 +15,7 @@
package com.google.devtools.build.lib.bazel.rules.ninja.parser;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -46,6 +47,7 @@
}
/** Compute the expanded value, using the passed <code>expander</code> function. */
+ @VisibleForTesting
public String getExpandedValue(Function<String, String> expander) {
return parts.stream().map(fun -> fun.apply(expander)).collect(Collectors.joining(""));
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/BUILD
index 7f0c602..cbdfa21 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/bazel/BUILD
@@ -42,6 +42,7 @@
"//src/main/java/com/google/devtools/build/lib:bazel-ninja",
"//src/main/java/com/google/devtools/build/lib:util",
"//src/main/java/com/google/devtools/build/lib/concurrent",
+ "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/test/java/com/google/devtools/build/lib:test_runner",
"//src/test/java/com/google/devtools/build/lib:testutil",
"//third_party:guava",
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerTest.java
index a40e9eb..88a50a1 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerTest.java
@@ -19,6 +19,7 @@
import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
+import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer.TextKind;
import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaToken;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@@ -144,7 +145,7 @@
assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "my.var");
assertTokenBytes(lexer, NinjaToken.EQUALS, null);
- lexer.expectTextUntilEol();
+ lexer.setExpectedTextKind(TextKind.TEXT);
assertTokenBytes(lexer, NinjaToken.TEXT, "Any");
assertTokenBytes(lexer, NinjaToken.TEXT, "text");
assertTokenBytes(lexer, NinjaToken.TEXT, "^&%=@&!*");
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaParserTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaParserTest.java
index b86a725..fecfbb9 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaParserTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaParserTest.java
@@ -26,8 +26,11 @@
import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaParser;
import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaRule;
import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaRuleVariable;
+import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaScope;
+import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget;
import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaVariableValue;
import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.PathFragment;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.function.Function;
@@ -169,6 +172,104 @@
doTestNinjaRuleParsingException("rule testRule\n custom = a", "Unexpected variable 'custom'");
}
+ @Test
+ public void testNinjaTargets() throws GenericParsingException {
+ NinjaTarget target = parseNinjaTarget("build output: command input");
+ assertThat(target.getRuleName()).isEqualTo("command");
+ assertThat(target.getOutputs()).containsExactly(PathFragment.create("output"));
+ assertThat(target.getUsualInputs()).containsExactly(PathFragment.create("input"));
+
+ NinjaTarget target1 =
+ parseNinjaTarget("build o1 o2 | io1 io2: command i1 i2 | ii1 ii2 || ooi1 ooi2");
+ assertThat(target1.getRuleName()).isEqualTo("command");
+ assertThat(target1.getOutputs())
+ .containsExactly(PathFragment.create("o1"), PathFragment.create("o2"));
+ assertThat(target1.getImplicitOutputs())
+ .containsExactly(PathFragment.create("io1"), PathFragment.create("io2"));
+ assertThat(target1.getUsualInputs())
+ .containsExactly(PathFragment.create("i1"), PathFragment.create("i2"));
+ assertThat(target1.getImplicitInputs())
+ .containsExactly(PathFragment.create("ii1"), PathFragment.create("ii2"));
+ assertThat(target1.getOrderOnlyInputs())
+ .containsExactly(PathFragment.create("ooi1"), PathFragment.create("ooi2"));
+
+ NinjaTarget target2 = parseNinjaTarget("build output: phony");
+ assertThat(target2.getRuleName()).isEqualTo("phony");
+ assertThat(target2.getOutputs()).containsExactly(PathFragment.create("output"));
+
+ NinjaTarget target3 = parseNinjaTarget("build output: command $\n || order-only-input");
+ assertThat(target3.getRuleName()).isEqualTo("command");
+ assertThat(target3.getOutputs()).containsExactly(PathFragment.create("output"));
+ assertThat(target3.getOrderOnlyInputs())
+ .containsExactly(PathFragment.create("order-only-input"));
+ }
+
+ @Test
+ public void testNinjaTargetParsingErrors() {
+ testNinjaTargetParsingError("build xxx", "Unexpected end of target");
+ testNinjaTargetParsingError("build xxx yyy:", "Expected rule name");
+ testNinjaTargetParsingError("build xxx || yyy: command", "Unexpected token: PIPE2");
+ testNinjaTargetParsingError("build xxx: command :", "Unexpected token: COLON");
+ testNinjaTargetParsingError("build xxx: command | || a", "Expected paths sequence");
+ }
+
+ @Test
+ public void testNinjaTargetsWithVariables() throws GenericParsingException {
+ NinjaScope scope = new NinjaScope();
+ scope.addVariable("output", 1, NinjaVariableValue.createPlainText("out123"));
+ scope.addVariable("input", 2, NinjaVariableValue.createPlainText("in123"));
+
+ scope.expandVariables();
+
+ // Variables, defined inside build statement, are used for input and output paths,
+ // but not for the values of the other variables.
+ // Test it.
+ NinjaTarget target =
+ createParser(
+ "build $output : command $input $dir/abcde\n"
+ + " dir = def$input\n empty = '$dir'")
+ .parseNinjaTarget(scope, 5);
+ assertThat(target.getRuleName()).isEqualTo("command");
+ assertThat(target.getOutputs()).containsExactly(PathFragment.create("out123"));
+ assertThat(target.getUsualInputs())
+ .containsExactly(PathFragment.create("in123"), PathFragment.create("defin123/abcde"));
+ assertThat(target.getVariables())
+ .containsExactlyEntriesIn(ImmutableSortedMap.of("dir", "defin123", "empty", "''"));
+ }
+
+ @Test
+ public void testPseudoCyclesOfVariables() {
+ NinjaScope scope = new NinjaScope();
+ scope.addVariable(
+ "output", 1, NinjaVariableValue.builder().addText("'out'").addVariable("input").build());
+ scope.addVariable(
+ "input", 2, NinjaVariableValue.builder().addText("'in'").addVariable("output").build());
+ scope.expandVariables();
+ assertThat(scope.findExpandedVariable(3, "input")).isEqualTo("'in''out'");
+ assertThat(scope.findExpandedVariable(3, "output")).isEqualTo("'out'");
+ }
+
+ @Test
+ public void testNinjaTargetsPathWithEscapedSpace() throws GenericParsingException {
+ NinjaTarget target = parseNinjaTarget("build output : command input$ with$ space other");
+ assertThat(target.getRuleName()).isEqualTo("command");
+ assertThat(target.getOutputs()).containsExactly(PathFragment.create("output"));
+ assertThat(target.getUsualInputs())
+ .containsExactly(PathFragment.create("input with space"), PathFragment.create("other"));
+ }
+
+ private static void testNinjaTargetParsingError(String text, String error) {
+ GenericParsingException exception =
+ assertThrows(GenericParsingException.class, () -> parseNinjaTarget(text));
+ assertThat(exception).hasMessageThat().isEqualTo(error);
+ }
+
+ private static NinjaTarget parseNinjaTarget(String text) throws GenericParsingException {
+ NinjaScope fileScope = new NinjaScope();
+ fileScope.expandVariables();
+ return createParser(text).parseNinjaTarget(fileScope, 0);
+ }
+
private static void doTestNinjaRuleParsingException(String text, String message) {
GenericParsingException exception =
assertThrows(GenericParsingException.class, () -> createParser(text).parseNinjaRule());
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaScopeTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaScopeTest.java
index fc49b55..a73dd42 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaScopeTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaScopeTest.java
@@ -292,6 +292,6 @@
private static NinjaVariableValue parseValue(String text) throws GenericParsingException {
ByteBuffer bb = ByteBuffer.wrap(text.getBytes(StandardCharsets.ISO_8859_1));
NinjaLexer lexer = new NinjaLexer(new ByteBufferFragment(bb, 0, bb.limit()));
- return new NinjaParser(lexer).parseVariableValue(true, "test");
+ return new NinjaParser(lexer).parseVariableValue("test");
}
}
