Add operator // for division. In both Python 2 and Python 3, the operator // is used for int division. With Python 3, operator / is for float division. RELNOTES: None. PiperOrigin-RevId: 156582262
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java index ea8280a..0e03157 100644 --- a/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java +++ b/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java
@@ -134,6 +134,7 @@ return mult(lval, rval, env, location); case DIVIDE: + case FLOOR_DIVIDE: return divide(lval, rval, location); case PERCENT:
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java index d7ee319..9257de7 100644 --- a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java +++ b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
@@ -296,7 +296,7 @@ * delimiter (3 x quot), and advances 'pos' by two if so. */ private boolean skipTripleQuote(char quot) { - if (pos + 1 < buffer.length && buffer[pos] == quot && buffer[pos + 1] == quot) { + if (lookaheadIs(0, quot) && lookaheadIs(1, quot)) { pos += 2; return true; } else { @@ -340,7 +340,7 @@ // Insert \ and the following character. // As in Python, it means that a raw string can never end with a single \. literal.append('\\'); - if (pos + 1 < buffer.length && buffer[pos] == '\r' && buffer[pos + 1] == '\n') { + if (lookaheadIs(0, '\r') && lookaheadIs(1, '\n')) { literal.append("\n"); pos += 2; } else if (buffer[pos] == '\r' || buffer[pos] == '\n') { @@ -356,7 +356,7 @@ pos++; switch (c) { case '\r': - if (pos < buffer.length && buffer[pos] == '\n') { + if (lookaheadIs(0, '\n')) { pos += 1; break; } else { @@ -466,7 +466,7 @@ return t; case '\\': if (isRaw) { - if (pos + 1 < buffer.length && buffer[pos] == '\r' && buffer[pos + 1] == '\n') { + if (lookaheadIs(0, '\r') && lookaheadIs(1, '\n')) { // There was a CRLF after the newline. No shortcut possible, since it needs to be // transformed into a single LF. pos = oldPos + 1; @@ -483,9 +483,10 @@ case '"': if (c == quot) { // close-quote, all done. - return new Token(TokenKind.STRING, oldPos, pos, - bufferSlice(oldPos + 1, pos - 1)); + return new Token(TokenKind.STRING, oldPos, pos, bufferSlice(oldPos + 1, pos - 1)); } + break; + default: // fall out } } @@ -667,6 +668,11 @@ } } + /** Test if the character at pos+p is c. */ + private boolean lookaheadIs(int p, char c) { + return pos + p < buffer.length && buffer[pos + p] == c; + } + /** * Performs tokenization of the character buffer of file contents provided to * the constructor. @@ -747,7 +753,16 @@ break; } case '/': { - addToken(new Token(TokenKind.SLASH, pos - 1, pos)); + if (lookaheadIs(0, '/') && lookaheadIs(1, '=')) { + addToken(new Token(TokenKind.SLASH_SLASH_EQUALS, pos - 1, pos + 2)); + pos += 2; + } else if (lookaheadIs(0, '/')) { + addToken(new Token(TokenKind.SLASH_SLASH, pos - 1, pos + 1)); + pos += 1; + } else { + // /= is handled by tokenizeTwoChars. + addToken(new Token(TokenKind.SLASH, pos - 1, pos)); + } break; } case ';': { @@ -770,9 +785,9 @@ } case '\\': { // Backslash character is valid only at the end of a line (or in a string) - if (pos + 1 < buffer.length && buffer[pos] == '\n') { + if (lookaheadIs(0, '\n')) { pos += 1; // skip the end of line character - } else if (pos + 2 < buffer.length && buffer[pos] == '\r' && buffer[pos + 1] == '\n') { + } else if (lookaheadIs(0, '\r') && lookaheadIs(1, '\n')) { pos += 2; // skip the CRLF at the end of line } else { addToken(new Token(TokenKind.ILLEGAL, pos - 1, pos, Character.toString(c)));
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Operator.java b/src/main/java/com/google/devtools/build/lib/syntax/Operator.java index 91bd49d..b839865 100644 --- a/src/main/java/com/google/devtools/build/lib/syntax/Operator.java +++ b/src/main/java/com/google/devtools/build/lib/syntax/Operator.java
@@ -21,6 +21,7 @@ AND("and"), DIVIDE("/"), EQUALS_EQUALS("=="), + FLOOR_DIVIDE("//"), GREATER(">"), GREATER_EQUALS(">="), IN("in"),
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Parser.java b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java index 23bb0cd..1cb224d 100644 --- a/src/main/java/com/google/devtools/build/lib/syntax/Parser.java +++ b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
@@ -38,7 +38,6 @@ import java.util.List; import java.util.Map; - /** * Recursive descent parser for LL(2) BUILD language. * Loosely based on Python 2 grammar. @@ -159,6 +158,7 @@ .put(TokenKind.OR, Operator.OR) .put(TokenKind.PERCENT, Operator.PERCENT) .put(TokenKind.SLASH, Operator.DIVIDE) + .put(TokenKind.SLASH_SLASH, Operator.FLOOR_DIVIDE) .put(TokenKind.PLUS, Operator.PLUS) .put(TokenKind.PIPE, Operator.PIPE) .put(TokenKind.STAR, Operator.MULT) @@ -171,6 +171,7 @@ .put(TokenKind.MINUS_EQUALS, Operator.MINUS) .put(TokenKind.STAR_EQUALS, Operator.MULT) .put(TokenKind.SLASH_EQUALS, Operator.DIVIDE) + .put(TokenKind.SLASH_SLASH_EQUALS, Operator.FLOOR_DIVIDE) .put(TokenKind.PERCENT_EQUALS, Operator.PERCENT) .build(); @@ -185,7 +186,7 @@ Operator.GREATER, Operator.GREATER_EQUALS, Operator.IN, Operator.NOT_IN), EnumSet.of(Operator.PIPE), EnumSet.of(Operator.MINUS, Operator.PLUS), - EnumSet.of(Operator.DIVIDE, Operator.MULT, Operator.PERCENT)); + EnumSet.of(Operator.DIVIDE, Operator.FLOOR_DIVIDE, Operator.MULT, Operator.PERCENT)); private final Iterator<Token> tokens; private int errorsCount; @@ -1198,7 +1199,8 @@ if (token.kind == TokenKind.EQUALS) { nextToken(); Expression rvalue = parseExpression(); - return setLocation(new AssignmentStatement(expression, rvalue), start, rvalue); + return setLocation( + new AssignmentStatement(/*lvalue=*/ expression, /*expression=*/ rvalue), start, rvalue); } else if (augmentedAssignmentMethods.containsKey(token.kind)) { Operator operator = augmentedAssignmentMethods.get(token.kind); nextToken();
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java b/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java index 8000546..52acce1 100644 --- a/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java +++ b/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java
@@ -19,9 +19,9 @@ */ public enum TokenKind { - ASSERT("assert"), AND("and"), AS("as"), + ASSERT("assert"), BREAK("break"), CLASS("class"), COLON(":"), @@ -58,6 +58,7 @@ LESS_EQUALS("<="), LPAREN("("), MINUS("-"), + MINUS_EQUALS("-="), NEWLINE("newline"), NONLOCAL("nonlocal"), NOT("not"), @@ -67,13 +68,10 @@ OUTDENT("outdent"), PASS("pass"), PERCENT("%"), + PERCENT_EQUALS("%="), PIPE("|"), PLUS("+"), PLUS_EQUALS("+="), - MINUS_EQUALS("-="), - STAR_EQUALS("*="), - SLASH_EQUALS("/="), - PERCENT_EQUALS("%="), RAISE("raise"), RBRACE("}"), RBRACKET("]"), @@ -81,7 +79,11 @@ RPAREN(")"), SEMI(";"), SLASH("/"), + SLASH_EQUALS("/="), + SLASH_SLASH("//"), + SLASH_SLASH_EQUALS("//="), STAR("*"), + STAR_EQUALS("*="), STAR_STAR("**"), STRING("string"), TRY("try"),