Make the parser look at only one Token at a time.
Remove all references to `Token` (except one) in the parser. In particular,
remove the mechanism to push tokens back. This change will make possible for
the lexer to reuse tokens instead of allocating new objects each time.
RELNOTES: None.
PiperOrigin-RevId: 197585185
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Token.java b/src/main/java/com/google/devtools/build/lib/syntax/Token.java
index aef2d0e..3c99df5 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/Token.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Token.java
@@ -13,22 +13,29 @@
// limitations under the License.
package com.google.devtools.build.lib.syntax;
+import javax.annotation.Nullable;
+
/**
* A Token represents an actual lexeme; that is, a lexical unit, its location in
* the input text, its lexical kind (TokenKind) and any associated value.
*/
-public class Token {
+class Token {
- public final TokenKind kind;
- public final int left;
- public final int right;
- public final Object value;
+ TokenKind kind;
+ final int left;
+ final int right;
+ /**
+ * value is an Integer if the kind is INT.
+ * It is a String if the kind is STRING, IDENTIFIER, or COMMENT.
+ * It is null otherwise.
+ */
+ @Nullable final Object value;
- public Token(TokenKind kind, int left, int right) {
+ Token(TokenKind kind, int left, int right) {
this(kind, left, right, null);
}
- public Token(TokenKind kind, int left, int right, Object value) {
+ Token(TokenKind kind, int left, int right, Object value) {
this.kind = kind;
this.left = left;
this.right = right;
