Update from Google.
--
MOE_MIGRATED_REVID=85702957
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ASTNode.java b/src/main/java/com/google/devtools/build/lib/syntax/ASTNode.java
new file mode 100644
index 0000000..81ca584
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ASTNode.java
@@ -0,0 +1,65 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.devtools.build.lib.events.Location;
+
+import java.io.Serializable;
+
+/**
+ * Root class for nodes in the Abstract Syntax Tree of the Build language.
+ */
+public abstract class ASTNode implements Serializable {
+
+ private Location location;
+
+ protected ASTNode() {}
+
+ @VisibleForTesting // productionVisibility = Visibility.PACKAGE_PRIVATE
+ public void setLocation(Location location) {
+ this.location = location;
+ }
+
+ public Location getLocation() {
+ return location;
+ }
+
+ /**
+ * Print the syntax node in a form useful for debugging. The output is not
+ * precisely specified, and should not be used by pretty-printing routines.
+ */
+ @Override
+ public abstract String toString();
+
+ @Override
+ public int hashCode() {
+ throw new UnsupportedOperationException(); // avoid nondeterminism
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Implements the double dispatch by calling into the node specific
+ * <code>visit</code> method of the {@link SyntaxTreeVisitor}
+ *
+ * @param visitor the {@link SyntaxTreeVisitor} instance to dispatch to.
+ */
+ public abstract void accept(SyntaxTreeVisitor visitor);
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/AbstractFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/AbstractFunction.java
new file mode 100644
index 0000000..f444c23
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/AbstractFunction.java
@@ -0,0 +1,64 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Partial implementation of Function interface.
+ */
+public abstract class AbstractFunction implements Function {
+
+ private final String name;
+
+ protected AbstractFunction(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Returns the name of this function.
+ */
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Class<?> getObjectType() {
+ return null;
+ }
+
+ /**
+ * Abstract implementation of Function that accepts no parameters.
+ */
+ public abstract static class NoArgFunction extends AbstractFunction {
+
+ public NoArgFunction(String name) {
+ super(name);
+ }
+
+ @Override
+ public Object call(List<Object> args, Map<String, Object> kwargs, FuncallExpression ast,
+ Environment env) throws EvalException, InterruptedException {
+ if (args.size() != 1 || kwargs.size() != 0) {
+ throw new EvalException(ast.getLocation(), "Invalid number of arguments (expected 0)");
+ }
+ return call(args.get(0), ast, env);
+ }
+
+ public abstract Object call(Object self, FuncallExpression ast, Environment env)
+ throws EvalException, InterruptedException;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Argument.java b/src/main/java/com/google/devtools/build/lib/syntax/Argument.java
new file mode 100644
index 0000000..0706dee
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Argument.java
@@ -0,0 +1,122 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Syntax node for a function argument. This can be a key/value pair such as
+ * appears as a keyword argument to a function call or just an expression that
+ * is used as a positional argument. It also can be used for function definitions
+ * to identify the name (and optionally the default value) of the argument.
+ */
+public final class Argument extends ASTNode {
+
+ private final Ident name;
+
+ private final Expression value;
+
+ private final boolean kwargs;
+
+ /**
+ * Create a new argument.
+ * At call site: name is optional, value is mandatory. kwargs is true for ** arguments.
+ * At definition site: name is mandatory, (default) value is optional.
+ */
+ public Argument(Ident name, Expression value, boolean kwargs) {
+ this.name = name;
+ this.value = value;
+ this.kwargs = kwargs;
+ }
+
+ public Argument(Ident name, Expression value) {
+ this.name = name;
+ this.value = value;
+ this.kwargs = false;
+ }
+
+ /**
+ * Creates an Argument with null as name. It can be used as positional arguments
+ * of function calls.
+ */
+ public Argument(Expression value) {
+ this(null, value);
+ }
+
+ /**
+ * Creates an Argument with null as value. It can be used as a mandatory keyword argument
+ * of a function definition.
+ */
+ public Argument(Ident name) {
+ this(name, null);
+ }
+
+ /**
+ * Returns the name of this keyword argument or null if this argument is
+ * positional.
+ */
+ public Ident getName() {
+ return name;
+ }
+
+ /**
+ * Returns the String value of the Ident of this argument. Shortcut for arg.getName().getName().
+ */
+ public String getArgName() {
+ return name.getName();
+ }
+
+ /**
+ * Returns the syntax of this argument expression.
+ */
+ public Expression getValue() {
+ return value;
+ }
+
+ /**
+ * Returns true if this argument is positional.
+ */
+ public boolean isPositional() {
+ return name == null && !kwargs;
+ }
+
+ /**
+ * Returns true if this argument is a keyword argument.
+ */
+ public boolean isNamed() {
+ return name != null;
+ }
+
+ /**
+ * Returns true if this argument is a **kwargs argument.
+ */
+ public boolean isKwargs() {
+ return kwargs;
+ }
+
+ /**
+ * Returns true if this argument has value.
+ */
+ public boolean hasValue() {
+ return value != null;
+ }
+
+ @Override
+ public String toString() {
+ return isNamed() ? name + "=" + value : String.valueOf(value);
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/AssignmentStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/AssignmentStatement.java
new file mode 100644
index 0000000..619e841
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/AssignmentStatement.java
@@ -0,0 +1,108 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Syntax node for an assignment statement.
+ */
+public final class AssignmentStatement extends Statement {
+
+ private final Expression lvalue;
+
+ private final Expression expression;
+
+ /**
+ * Constructs an assignment: "lvalue := value".
+ */
+ AssignmentStatement(Expression lvalue, Expression expression) {
+ this.lvalue = lvalue;
+ this.expression = expression;
+ }
+
+ /**
+ * Returns the LHS of the assignment.
+ */
+ public Expression getLValue() {
+ return lvalue;
+ }
+
+ /**
+ * Returns the RHS of the assignment.
+ */
+ public Expression getExpression() {
+ return expression;
+ }
+
+ @Override
+ public String toString() {
+ return lvalue + " = " + expression + '\n';
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ if (!(lvalue instanceof Ident)) {
+ throw new EvalException(getLocation(),
+ "can only assign to variables, not to '" + lvalue + "'");
+ }
+
+ Ident ident = (Ident) lvalue;
+ Object result = expression.eval(env);
+ Preconditions.checkNotNull(result, "result of " + expression + " is null");
+
+ if (env.isSkylarkEnabled()) {
+ // The variable may have been referenced successfully if a global variable
+ // with the same name exists. In this case an Exception needs to be thrown.
+ SkylarkEnvironment skylarkEnv = (SkylarkEnvironment) env;
+ if (skylarkEnv.hasBeenReadGlobalVariable(ident.getName())) {
+ throw new EvalException(getLocation(), "Variable '" + ident.getName()
+ + "' is referenced before assignment."
+ + "The variable is defined in the global scope.");
+ }
+ Class<?> variableType = skylarkEnv.getVariableType(ident.getName());
+ Class<?> resultType = EvalUtils.getSkylarkType(result.getClass());
+ if (variableType != null && !variableType.equals(resultType)
+ && !resultType.equals(Environment.NoneType.class)
+ && !variableType.equals(Environment.NoneType.class)) {
+ throw new EvalException(getLocation(), String.format("Incompatible variable types, "
+ + "trying to assign %s (type of %s) to variable %s which is already %s",
+ EvalUtils.prettyPrintValue(result),
+ EvalUtils.getDatatypeName(result),
+ ident.getName(),
+ EvalUtils.getDataTypeNameFromClass(variableType)));
+ }
+ }
+ env.update(ident.getName(), result);
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ // TODO(bazel-team): Implement other validations.
+ if (lvalue instanceof Ident) {
+ Ident ident = (Ident) lvalue;
+ SkylarkType resultType = expression.validate(env);
+ env.update(ident.getName(), resultType, getLocation());
+ } else {
+ throw new EvalException(getLocation(),
+ "can only assign to variables, not to '" + lvalue + "'");
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java
new file mode 100644
index 0000000..f53538f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java
@@ -0,0 +1,412 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.IllegalFormatException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Syntax node for a binary operator expression.
+ */
+public final class BinaryOperatorExpression extends Expression {
+
+ private final Expression lhs;
+
+ private final Expression rhs;
+
+ private final Operator operator;
+
+ public BinaryOperatorExpression(Operator operator,
+ Expression lhs,
+ Expression rhs) {
+ this.lhs = lhs;
+ this.rhs = rhs;
+ this.operator = operator;
+ }
+
+ public Expression getLhs() {
+ return lhs;
+ }
+
+ public Expression getRhs() {
+ return rhs;
+ }
+
+ /**
+ * Returns the operator kind for this binary operation.
+ */
+ public Operator getOperator() {
+ return operator;
+ }
+
+ @Override
+ public String toString() {
+ return lhs + " " + operator + " " + rhs;
+ }
+
+ private int compare(Object lval, Object rval) throws EvalException {
+ if (!(lval instanceof Comparable)) {
+ throw new EvalException(getLocation(), lval + " is not comparable");
+ }
+ try {
+ return ((Comparable) lval).compareTo(rval);
+ } catch (ClassCastException e) {
+ throw new EvalException(getLocation(), "Cannot compare " + EvalUtils.getDatatypeName(lval)
+ + " with " + EvalUtils.getDatatypeName(rval));
+ }
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ Object lval = lhs.eval(env);
+
+ // Short-circuit operators
+ if (operator == Operator.AND) {
+ if (EvalUtils.toBoolean(lval)) {
+ return rhs.eval(env);
+ } else {
+ return lval;
+ }
+ }
+
+ if (operator == Operator.OR) {
+ if (EvalUtils.toBoolean(lval)) {
+ return lval;
+ } else {
+ return rhs.eval(env);
+ }
+ }
+
+ Object rval = rhs.eval(env);
+
+ switch (operator) {
+ case PLUS: {
+ // int + int
+ if (lval instanceof Integer && rval instanceof Integer) {
+ return ((Integer) lval).intValue() + ((Integer) rval).intValue();
+ }
+
+ // string + string
+ if (lval instanceof String && rval instanceof String) {
+ return (String) lval + (String) rval;
+ }
+
+ // list + list, tuple + tuple (list + tuple, tuple + list => error)
+ if (lval instanceof List<?> && rval instanceof List<?>) {
+ List<?> llist = (List<?>) lval;
+ List<?> rlist = (List<?>) rval;
+ if (EvalUtils.isImmutable(llist) != EvalUtils.isImmutable(rlist)) {
+ throw new EvalException(getLocation(), "can only concatenate "
+ + EvalUtils.getDatatypeName(rlist) + " (not \""
+ + EvalUtils.getDatatypeName(llist) + "\") to "
+ + EvalUtils.getDatatypeName(rlist));
+ }
+ if (llist instanceof GlobList<?> || rlist instanceof GlobList<?>) {
+ return GlobList.concat(llist, rlist);
+ } else {
+ List<Object> result = Lists.newArrayListWithCapacity(llist.size() + rlist.size());
+ result.addAll(llist);
+ result.addAll(rlist);
+ return EvalUtils.makeSequence(result, EvalUtils.isImmutable(llist));
+ }
+ }
+
+ if (lval instanceof SkylarkList && rval instanceof SkylarkList) {
+ return SkylarkList.concat((SkylarkList) lval, (SkylarkList) rval, getLocation());
+ }
+
+ if (env.isSkylarkEnabled() && lval instanceof Map<?, ?> && rval instanceof Map<?, ?>) {
+ Map<?, ?> ldict = (Map<?, ?>) lval;
+ Map<?, ?> rdict = (Map<?, ?>) rval;
+ Map<Object, Object> result = Maps.newHashMapWithExpectedSize(ldict.size() + rdict.size());
+ result.putAll(ldict);
+ result.putAll(rdict);
+ return result;
+ }
+
+ if (env.isSkylarkEnabled()
+ && lval instanceof SkylarkClassObject && rval instanceof SkylarkClassObject) {
+ return SkylarkClassObject.concat(
+ (SkylarkClassObject) lval, (SkylarkClassObject) rval, getLocation());
+ }
+
+ if (env.isSkylarkEnabled() && lval instanceof SkylarkNestedSet) {
+ return new SkylarkNestedSet((SkylarkNestedSet) lval, rval, getLocation());
+ }
+ break;
+ }
+
+ case MINUS: {
+ if (lval instanceof Integer && rval instanceof Integer) {
+ return ((Integer) lval).intValue() - ((Integer) rval).intValue();
+ }
+ break;
+ }
+
+ case MULT: {
+ // int * int
+ if (lval instanceof Integer && rval instanceof Integer) {
+ return ((Integer) lval).intValue() * ((Integer) rval).intValue();
+ }
+
+ // string * int
+ if (lval instanceof String && rval instanceof Integer) {
+ return Strings.repeat((String) lval, ((Integer) rval).intValue());
+ }
+
+ // int * string
+ if (lval instanceof Integer && rval instanceof String) {
+ return Strings.repeat((String) rval, ((Integer) lval).intValue());
+ }
+ break;
+ }
+
+ case PERCENT: {
+ // int % int
+ if (lval instanceof Integer && rval instanceof Integer) {
+ return ((Integer) lval).intValue() % ((Integer) rval).intValue();
+ }
+
+ // string % tuple, string % dict, string % anything-else
+ if (lval instanceof String) {
+ try {
+ String pattern = (String) lval;
+ if (rval instanceof List<?>) {
+ List<?> rlist = (List<?>) rval;
+ if (EvalUtils.isTuple(rlist)) {
+ return EvalUtils.formatString(pattern, rlist);
+ }
+ /* string % list: fall thru */
+ }
+ if (rval instanceof SkylarkList) {
+ SkylarkList rlist = (SkylarkList) rval;
+ if (rlist.isTuple()) {
+ return EvalUtils.formatString(pattern, rlist.toList());
+ }
+ }
+
+ return EvalUtils.formatString(pattern,
+ Collections.singletonList(rval));
+ } catch (IllegalFormatException e) {
+ throw new EvalException(getLocation(), e.getMessage());
+ }
+ }
+ break;
+ }
+
+ case EQUALS_EQUALS: {
+ return lval.equals(rval);
+ }
+
+ case NOT_EQUALS: {
+ return !lval.equals(rval);
+ }
+
+ case LESS: {
+ return compare(lval, rval) < 0;
+ }
+
+ case LESS_EQUALS: {
+ return compare(lval, rval) <= 0;
+ }
+
+ case GREATER: {
+ return compare(lval, rval) > 0;
+ }
+
+ case GREATER_EQUALS: {
+ return compare(lval, rval) >= 0;
+ }
+
+ case IN: {
+ if (rval instanceof SkylarkList) {
+ for (Object obj : (SkylarkList) rval) {
+ if (obj.equals(lval)) {
+ return true;
+ }
+ }
+ return false;
+ } else if (rval instanceof Collection<?>) {
+ return ((Collection<?>) rval).contains(lval);
+ } else if (rval instanceof Map<?, ?>) {
+ return ((Map<?, ?>) rval).containsKey(lval);
+ } else if (rval instanceof String) {
+ if (lval instanceof String) {
+ return ((String) rval).contains((String) lval);
+ } else {
+ throw new EvalException(getLocation(),
+ "in operator only works on strings if the left operand is also a string");
+ }
+ } else {
+ throw new EvalException(getLocation(),
+ "in operator only works on lists, tuples, dictionaries and strings");
+ }
+ }
+
+ default: {
+ throw new AssertionError("Unsupported binary operator: " + operator);
+ }
+ } // endswitch
+
+ throw new EvalException(getLocation(),
+ "unsupported operand types for '" + operator + "': '"
+ + EvalUtils.getDatatypeName(lval) + "' and '"
+ + EvalUtils.getDatatypeName(rval) + "'");
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ SkylarkType ltype = lhs.validate(env);
+ SkylarkType rtype = rhs.validate(env);
+ String lname = EvalUtils.getDataTypeNameFromClass(ltype.getType());
+ String rname = EvalUtils.getDataTypeNameFromClass(rtype.getType());
+
+ switch (operator) {
+ case AND: {
+ return ltype.infer(rtype, "and operator", rhs.getLocation(), lhs.getLocation());
+ }
+
+ case OR: {
+ return ltype.infer(rtype, "or operator", rhs.getLocation(), lhs.getLocation());
+ }
+
+ case PLUS: {
+ // int + int
+ if (ltype == SkylarkType.INT && rtype == SkylarkType.INT) {
+ return SkylarkType.INT;
+ }
+
+ // string + string
+ if (ltype == SkylarkType.STRING && rtype == SkylarkType.STRING) {
+ return SkylarkType.STRING;
+ }
+
+ // list + list
+ if (ltype.isList() && rtype.isList()) {
+ return ltype.infer(rtype, "list concatenation", rhs.getLocation(), lhs.getLocation());
+ }
+
+ // dict + dict
+ if (ltype.isDict() && rtype.isDict()) {
+ return ltype.infer(rtype, "dict concatenation", rhs.getLocation(), lhs.getLocation());
+ }
+
+ // struct + struct
+ if (ltype.isStruct() && rtype.isStruct()) {
+ return SkylarkType.of(ClassObject.class);
+ }
+
+ if (ltype.isNset()) {
+ if (rtype.isNset()) {
+ return ltype.infer(rtype, "nested set", rhs.getLocation(), lhs.getLocation());
+ } else if (rtype.isList()) {
+ return ltype.infer(SkylarkType.of(SkylarkNestedSet.class, rtype.getGenericType1()),
+ "nested set", rhs.getLocation(), lhs.getLocation());
+ }
+ if (rtype != SkylarkType.UNKNOWN) {
+ throw new EvalException(getLocation(), String.format("can only concatenate nested sets "
+ + "with other nested sets or list of items, not '" + rname + "'"));
+ }
+ }
+
+ break;
+ }
+
+ case MULT: {
+ // int * int
+ if (ltype == SkylarkType.INT && rtype == SkylarkType.INT) {
+ return SkylarkType.INT;
+ }
+
+ // string * int
+ if (ltype == SkylarkType.STRING && rtype == SkylarkType.INT) {
+ return SkylarkType.STRING;
+ }
+
+ // int * string
+ if (ltype == SkylarkType.INT && rtype == SkylarkType.STRING) {
+ return SkylarkType.STRING;
+ }
+ break;
+ }
+
+ case MINUS: {
+ if (ltype == SkylarkType.INT && rtype == SkylarkType.INT) {
+ return SkylarkType.INT;
+ }
+ break;
+ }
+
+ case PERCENT: {
+ // int % int
+ if (ltype == SkylarkType.INT && rtype == SkylarkType.INT) {
+ return SkylarkType.INT;
+ }
+
+ // string % tuple, string % dict, string % anything-else
+ if (ltype == SkylarkType.STRING) {
+ return SkylarkType.STRING;
+ }
+ break;
+ }
+
+ case EQUALS_EQUALS:
+ case NOT_EQUALS:
+ case LESS:
+ case LESS_EQUALS:
+ case GREATER:
+ case GREATER_EQUALS: {
+ if (ltype != SkylarkType.UNKNOWN && !(Comparable.class.isAssignableFrom(ltype.getType()))) {
+ throw new EvalException(getLocation(), lname + " is not comparable");
+ }
+ ltype.infer(rtype, "comparison", lhs.getLocation(), rhs.getLocation());
+ return SkylarkType.BOOL;
+ }
+
+ case IN: {
+ if (rtype.isList()
+ || rtype.isSet()
+ || rtype.isDict()
+ || rtype == SkylarkType.STRING) {
+ return SkylarkType.BOOL;
+ } else {
+ if (rtype != SkylarkType.UNKNOWN) {
+ throw new EvalException(getLocation(), String.format("operand 'in' only works on "
+ + "strings, dictionaries, lists, sets or tuples, not on a(n) %s",
+ EvalUtils.getDataTypeNameFromClass(rtype.getType())));
+ }
+ }
+ }
+ } // endswitch
+
+ if (ltype != SkylarkType.UNKNOWN && rtype != SkylarkType.UNKNOWN) {
+ throw new EvalException(getLocation(),
+ "unsupported operand types for '" + operator + "': '" + lname + "' and '" + rname + "'");
+ }
+ return SkylarkType.UNKNOWN;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java b/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java
new file mode 100644
index 0000000..6c85ab1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java
@@ -0,0 +1,244 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.CachingPackageLocator;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * Abstract syntax node for an entire BUILD file.
+ */
+public class BuildFileAST extends ASTNode {
+
+ private final ImmutableList<Statement> stmts;
+
+ private final ImmutableList<Comment> comments;
+
+ private final ImmutableSet<PathFragment> imports;
+
+ /**
+ * Whether any errors were encountered during scanning or parsing.
+ */
+ private final boolean containsErrors;
+
+ private final String contentHashCode;
+
+ private BuildFileAST(Lexer lexer, List<Statement> preludeStatements, Parser.ParseResult result) {
+ this(lexer, preludeStatements, result, null);
+ }
+
+ private BuildFileAST(Lexer lexer, List<Statement> preludeStatements,
+ Parser.ParseResult result, String contentHashCode) {
+ this.stmts = ImmutableList.<Statement>builder()
+ .addAll(preludeStatements)
+ .addAll(result.statements)
+ .build();
+ this.comments = ImmutableList.copyOf(result.comments);
+ this.containsErrors = result.containsErrors;
+ this.contentHashCode = contentHashCode;
+ this.imports = fetchImports(this.stmts);
+ if (result.statements.size() > 0) {
+ setLocation(lexer.createLocation(
+ result.statements.get(0).getLocation().getStartOffset(),
+ result.statements.get(result.statements.size() - 1).getLocation().getEndOffset()));
+ } else {
+ setLocation(Location.fromFile(lexer.getFilename()));
+ }
+ }
+
+ private ImmutableSet<PathFragment> fetchImports(List<Statement> stmts) {
+ Set<PathFragment> imports = new HashSet<>();
+ for (Statement stmt : stmts) {
+ if (stmt instanceof LoadStatement) {
+ LoadStatement imp = (LoadStatement) stmt;
+ imports.add(imp.getImportPath());
+ }
+ }
+ return ImmutableSet.copyOf(imports);
+ }
+
+ /**
+ * Returns true if any errors were encountered during scanning or parsing. If
+ * set, clients should not rely on the correctness of the AST for builds or
+ * BUILD-file editing.
+ */
+ public boolean containsErrors() {
+ return containsErrors;
+ }
+
+ /**
+ * Returns an (immutable, ordered) list of statements in this BUILD file.
+ */
+ public ImmutableList<Statement> getStatements() {
+ return stmts;
+ }
+
+ /**
+ * Returns an (immutable, ordered) list of comments in this BUILD file.
+ */
+ public ImmutableList<Comment> getComments() {
+ return comments;
+ }
+
+ /**
+ * Returns an (immutable) set of imports in this BUILD file.
+ */
+ public ImmutableCollection<PathFragment> getImports() {
+ return imports;
+ }
+
+ /**
+ * Executes this build file in a given Environment.
+ *
+ * <p>If, for any reason, execution of a statement cannot be completed, an
+ * {@link EvalException} is thrown by {@link Statement#exec(Environment)}.
+ * This exception is caught here and reported through reporter and execution
+ * continues on the next statement. In effect, there is a "try/except" block
+ * around every top level statement. Such exceptions are not ignored, though:
+ * they are visible via the return value. Rules declared in a package
+ * containing any error (including loading-phase semantical errors that
+ * cannot be checked here) must also be considered "in error".
+ *
+ * <p>Note that this method will not affect the value of {@link
+ * #containsErrors()}; that refers only to lexer/parser errors.
+ *
+ * @return true if no error occurred during execution.
+ */
+ public boolean exec(Environment env, EventHandler eventHandler) throws InterruptedException {
+ boolean ok = true;
+ for (Statement stmt : stmts) {
+ try {
+ stmt.exec(env);
+ } catch (EvalException e) {
+ ok = false;
+ // Do not report errors caused by a previous parsing error, as it has already been
+ // reported.
+ if (e.isDueToIncompleteAST()) {
+ continue;
+ }
+ // When the exception is raised from another file, report first the location in the
+ // BUILD file (as it is the most probable cause for the error).
+ Location exnLoc = e.getLocation();
+ Location nodeLoc = stmt.getLocation();
+ if (exnLoc == null || !nodeLoc.getPath().equals(exnLoc.getPath())) {
+ eventHandler.handle(Event.error(nodeLoc,
+ e.getMessage() + " (raised from " + exnLoc + ")"));
+ } else {
+ eventHandler.handle(Event.error(exnLoc, e.getMessage()));
+ }
+ }
+ }
+ return ok;
+ }
+
+ @Override
+ public String toString() {
+ return "BuildFileAST" + getStatements();
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ /**
+ * Parse the specified build file, returning its AST. All errors during
+ * scanning or parsing will be reported to the reporter.
+ *
+ * @throws IOException if the file cannot not be read.
+ */
+ public static BuildFileAST parseBuildFile(Path buildFile, EventHandler eventHandler,
+ CachingPackageLocator locator, boolean parsePython)
+ throws IOException {
+ ParserInputSource inputSource = ParserInputSource.create(buildFile);
+ return parseBuildFile(inputSource, eventHandler, locator, parsePython);
+ }
+
+ /**
+ * Parse the specified build file, returning its AST. All errors during
+ * scanning or parsing will be reported to the reporter.
+ */
+ public static BuildFileAST parseBuildFile(ParserInputSource input,
+ List<Statement> preludeStatements,
+ EventHandler eventHandler,
+ CachingPackageLocator locator,
+ boolean parsePython) {
+ Lexer lexer = new Lexer(input, eventHandler, parsePython);
+ Parser.ParseResult result = Parser.parseFile(lexer, eventHandler, locator, parsePython);
+ return new BuildFileAST(lexer, preludeStatements, result);
+ }
+
+ public static BuildFileAST parseBuildFile(ParserInputSource input, EventHandler eventHandler,
+ CachingPackageLocator locator, boolean parsePython) {
+ Lexer lexer = new Lexer(input, eventHandler, parsePython);
+ Parser.ParseResult result = Parser.parseFile(lexer, eventHandler, locator, parsePython);
+ return new BuildFileAST(lexer, ImmutableList.<Statement>of(), result);
+ }
+
+ /**
+ * Parse the specified build file, returning its AST. All errors during
+ * scanning or parsing will be reported to the reporter.
+ */
+ public static BuildFileAST parseBuildFile(Lexer lexer, EventHandler eventHandler) {
+ Parser.ParseResult result = Parser.parseFile(lexer, eventHandler, null, false);
+ return new BuildFileAST(lexer, ImmutableList.<Statement>of(), result);
+ }
+
+ /**
+ * Parse the specified Skylark file, returning its AST. All errors during
+ * scanning or parsing will be reported to the reporter.
+ *
+ * @throws IOException if the file cannot not be read.
+ */
+ public static BuildFileAST parseSkylarkFile(Path file, EventHandler eventHandler,
+ CachingPackageLocator locator, ValidationEnvironment validationEnvironment)
+ throws IOException {
+ ParserInputSource input = ParserInputSource.create(file);
+ Lexer lexer = new Lexer(input, eventHandler, false);
+ Parser.ParseResult result =
+ Parser.parseFileForSkylark(lexer, eventHandler, locator, validationEnvironment);
+ return new BuildFileAST(lexer, ImmutableList.<Statement>of(), result, input.contentHashCode());
+ }
+
+ /**
+ * Parse the specified build file, without building the AST.
+ *
+ * @return true if the input file is syntactically valid
+ */
+ public static boolean checkSyntax(ParserInputSource input,
+ EventHandler eventHandler, boolean parsePython) {
+ return !parseBuildFile(input, eventHandler, null, parsePython).containsErrors();
+ }
+
+ /**
+ * Returns a hash code calculated from the string content of the source file of this AST.
+ */
+ @Nullable public String getContentHashCode() {
+ return contentHashCode;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ClassObject.java b/src/main/java/com/google/devtools/build/lib/syntax/ClassObject.java
new file mode 100644
index 0000000..3b1cccf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ClassObject.java
@@ -0,0 +1,113 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Sets.SetView;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Location;
+
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * An interface for objects behaving like Skylark structs.
+ */
+// TODO(bazel-team): type checks
+public interface ClassObject {
+
+ /**
+ * Returns the value associated with the name field in this struct,
+ * or null if the field does not exist.
+ */
+ @Nullable
+ Object getValue(String name);
+
+ /**
+ * Returns the fields of this struct.
+ */
+ ImmutableCollection<String> getKeys();
+
+ /**
+ * Returns a customized error message to print if the name is not a valid struct field
+ * of this struct, or returns null to use the default error message.
+ */
+ @Nullable String errorMessage(String name);
+
+ /**
+ * An implementation class of ClassObject for structs created in Skylark code.
+ */
+ @Immutable
+ @SkylarkModule(name = "struct",
+ doc = "A special language element to support structs (i.e. simple value objects). "
+ + "See the global <code>struct</code> method for more details.")
+ public class SkylarkClassObject implements ClassObject {
+
+ private final ImmutableMap<String, Object> values;
+ private final Location creationLoc;
+ private final String errorMessage;
+
+ /**
+ * Creates a built-in struct (i.e. without creation loc). The errorMessage has to have
+ * exactly one '%s' parameter to substitute the struct field name.
+ */
+ public SkylarkClassObject(Map<String, Object> values, String errorMessage) {
+ this.values = ImmutableMap.copyOf(values);
+ this.creationLoc = null;
+ this.errorMessage = errorMessage;
+ }
+
+ public SkylarkClassObject(Map<String, Object> values, Location creationLoc) {
+ this.values = ImmutableMap.copyOf(values);
+ this.creationLoc = Preconditions.checkNotNull(creationLoc);
+ this.errorMessage = null;
+ }
+
+ @Override
+ public Object getValue(String name) {
+ return values.get(name);
+ }
+
+ @Override
+ public ImmutableCollection<String> getKeys() {
+ return values.keySet();
+ }
+
+ public Location getCreationLoc() {
+ return Preconditions.checkNotNull(creationLoc,
+ "This struct was not created in a Skylark code");
+ }
+
+ static SkylarkClassObject concat(
+ SkylarkClassObject lval, SkylarkClassObject rval, Location loc) throws EvalException {
+ SetView<String> commonFields = Sets.intersection(lval.values.keySet(), rval.values.keySet());
+ if (!commonFields.isEmpty()) {
+ throw new EvalException(loc, "Cannot concat structs with common field(s): "
+ + Joiner.on(",").join(commonFields));
+ }
+ return new SkylarkClassObject(ImmutableMap.<String, Object>builder()
+ .putAll(lval.values).putAll(rval.values).build(), loc);
+ }
+
+ @Override
+ public String errorMessage(String name) {
+ return errorMessage != null ? String.format(errorMessage, name) : null;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/CommaSeparatedPackageNameListConverter.java b/src/main/java/com/google/devtools/build/lib/syntax/CommaSeparatedPackageNameListConverter.java
new file mode 100644
index 0000000..070e928
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/CommaSeparatedPackageNameListConverter.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.util.List;
+
+/**
+ * A converter from strings containing comma-separated names of packages to lists of strings.
+ */
+public class CommaSeparatedPackageNameListConverter
+ implements Converter<List<String>> {
+
+ private static final Splitter SPACE_SPLITTER = Splitter.on(',');
+
+ @Override
+ public List<String> convert(String input) throws OptionsParsingException {
+ if (Strings.isNullOrEmpty(input)) {
+ return ImmutableList.of();
+ }
+ ImmutableList.Builder<String> list = ImmutableList.builder();
+ for (String s : SPACE_SPLITTER.split(input)) {
+ String errorMessage = LabelValidator.validatePackageName(s);
+ if (errorMessage != null) {
+ throw new OptionsParsingException(errorMessage);
+ }
+ list.add(s);
+ }
+ return list.build();
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "comma-separated list of package names";
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Comment.java b/src/main/java/com/google/devtools/build/lib/syntax/Comment.java
new file mode 100644
index 0000000..29d9474
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Comment.java
@@ -0,0 +1,40 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Syntax node for comments.
+ */
+public final class Comment extends ASTNode {
+
+ protected final String value;
+
+ public Comment(String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ public String toString() {
+ return value;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/DictComprehension.java b/src/main/java/com/google/devtools/build/lib/syntax/DictComprehension.java
new file mode 100644
index 0000000..a69605e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/DictComprehension.java
@@ -0,0 +1,102 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Syntax node for dictionary comprehension expressions.
+ */
+public class DictComprehension extends Expression {
+
+ private final Expression keyExpression;
+ private final Expression valueExpression;
+ private final Ident loopVar;
+ private final Expression listExpression;
+
+ public DictComprehension(Expression keyExpression, Expression valueExpression, Ident loopVar,
+ Expression listExpression) {
+ this.keyExpression = keyExpression;
+ this.valueExpression = valueExpression;
+ this.loopVar = loopVar;
+ this.listExpression = listExpression;
+ }
+
+ Expression getKeyExpression() {
+ return keyExpression;
+ }
+
+ Expression getValueExpression() {
+ return valueExpression;
+ }
+
+ Ident getLoopVar() {
+ return loopVar;
+ }
+
+ Expression getListExpression() {
+ return listExpression;
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ // We want to keep the iteration order
+ LinkedHashMap<Object, Object> map = new LinkedHashMap<>();
+ Iterable<?> elements = EvalUtils.toIterable(listExpression.eval(env), getLocation());
+ for (Object element : elements) {
+ env.update(loopVar.getName(), element);
+ Object key = keyExpression.eval(env);
+ map.put(key, valueExpression.eval(env));
+ }
+ return ImmutableMap.copyOf(map);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ SkylarkType elementsType = listExpression.validate(env);
+ // TODO(bazel-team): GenericType1 should be a SkylarkType.
+ Class<?> listElementType = elementsType.getGenericType1();
+ SkylarkType listElementSkylarkType = listElementType.equals(Object.class)
+ ? SkylarkType.UNKNOWN : SkylarkType.of(listElementType);
+ env.update(loopVar.getName(), listElementSkylarkType, getLocation());
+ SkylarkType keyType = keyExpression.validate(env);
+ if (!keyType.isSimple()) {
+ // TODO(bazel-team): this is most probably dead code but it's better to have it here
+ // in case we enable e.g. list of lists or we validate function calls on Java objects
+ throw new EvalException(getLocation(), "Dict comprehension key must be of a simple type");
+ }
+ valueExpression.validate(env);
+ if (elementsType != SkylarkType.UNKNOWN && !elementsType.isList()) {
+ throw new EvalException(getLocation(), "Dict comprehension elements must be a list");
+ }
+ return SkylarkType.of(Map.class, keyType.getType());
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append('{').append(keyExpression).append(": ").append(valueExpression);
+ sb.append(" for ").append(loopVar).append(" in ").append(listExpression);
+ sb.append('}');
+ return sb.toString();
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.accept(this);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/DictionaryLiteral.java b/src/main/java/com/google/devtools/build/lib/syntax/DictionaryLiteral.java
new file mode 100644
index 0000000..8f79739
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/DictionaryLiteral.java
@@ -0,0 +1,117 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Syntax node for dictionary literals.
+ */
+public class DictionaryLiteral extends Expression {
+
+ static final class DictionaryEntryLiteral extends ASTNode {
+
+ private final Expression key;
+ private final Expression value;
+
+ public DictionaryEntryLiteral(Expression key, Expression value) {
+ this.key = key;
+ this.value = value;
+ }
+
+ Expression getKey() {
+ return key;
+ }
+
+ Expression getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(key);
+ sb.append(": ");
+ sb.append(value);
+ return sb.toString();
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+ }
+
+ private final ImmutableList<DictionaryEntryLiteral> entries;
+
+ public DictionaryLiteral(List<DictionaryEntryLiteral> exprs) {
+ this.entries = ImmutableList.copyOf(exprs);
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ // We need LinkedHashMap to maintain the order during iteration (e.g. for loops)
+ Map<Object, Object> map = new LinkedHashMap<>();
+ for (DictionaryEntryLiteral entry : entries) {
+ if (entry == null) {
+ throw new EvalException(getLocation(), "null expression in " + this);
+ }
+ map.put(entry.key.eval(env), entry.value.eval(env));
+
+ }
+ return map;
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append("{");
+ String sep = "";
+ for (DictionaryEntryLiteral e : entries) {
+ sb.append(sep);
+ sb.append(e);
+ sep = ", ";
+ }
+ sb.append("}");
+ return sb.toString();
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ public ImmutableList<DictionaryEntryLiteral> getEntries() {
+ return entries;
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ SkylarkType type = SkylarkType.UNKNOWN;
+ for (DictionaryEntryLiteral entry : entries) {
+ SkylarkType nextType = entry.key.validate(env);
+ entry.value.validate(env);
+ if (!nextType.isSimple()) {
+ throw new EvalException(getLocation(),
+ String.format("Dict cannot contain composite type '%s' as key", nextType));
+ }
+ type = type.infer(nextType, "dict literal", entry.getLocation(), getLocation());
+ }
+ return SkylarkType.of(Map.class, type.getType());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/DotExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/DotExpression.java
new file mode 100644
index 0000000..b0ae5a9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/DotExpression.java
@@ -0,0 +1,110 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.FuncallExpression.MethodDescriptor;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Syntax node for a dot expression.
+ * e.g. obj.field, but not obj.method()
+ */
+public final class DotExpression extends Expression {
+
+ private final Expression obj;
+
+ private final Ident field;
+
+ public DotExpression(Expression obj, Ident field) {
+ this.obj = obj;
+ this.field = field;
+ }
+
+ public Expression getObj() {
+ return obj;
+ }
+
+ public Ident getField() {
+ return field;
+ }
+
+ @Override
+ public String toString() {
+ return obj + "." + field;
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ Object objValue = obj.eval(env);
+ String name = field.getName();
+ Object result = eval(objValue, name, getLocation());
+ if (result == null) {
+ if (objValue instanceof ClassObject) {
+ String customErrorMessage = ((ClassObject) objValue).errorMessage(name);
+ if (customErrorMessage != null) {
+ throw new EvalException(getLocation(), customErrorMessage);
+ }
+ }
+ throw new EvalException(getLocation(), "Object of type '"
+ + EvalUtils.getDatatypeName(objValue) + "' has no field '" + name + "'");
+ }
+ return result;
+ }
+
+ /**
+ * Returns the field of the given name of the struct objValue, or null if no such field exists.
+ */
+ public static Object eval(Object objValue, String name, Location loc) throws EvalException {
+ Object result = null;
+ if (objValue instanceof ClassObject) {
+ result = ((ClassObject) objValue).getValue(name);
+ result = SkylarkType.convertToSkylark(result, loc);
+ // If we access NestedSets using ClassObject.getValue() we won't know the generic type,
+ // so we have to disable it. This should not happen.
+ SkylarkType.checkTypeAllowedInSkylark(result, loc);
+ } else {
+ try {
+ List<MethodDescriptor> methods = FuncallExpression.getMethods(objValue.getClass(), name, 0);
+ if (methods != null && methods.size() > 0) {
+ MethodDescriptor method = Iterables.getOnlyElement(methods);
+ if (method.getAnnotation().structField()) {
+ result = FuncallExpression.callMethod(
+ method, name, objValue, new Object[] {}, loc);
+ }
+ }
+ } catch (ExecutionException | IllegalAccessException | InvocationTargetException e) {
+ throw new EvalException(loc, "Method invocation failed: " + e);
+ }
+ }
+
+ return result;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ obj.validate(env);
+ // TODO(bazel-team): check existance of field
+ return SkylarkType.UNKNOWN;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Environment.java b/src/main/java/com/google/devtools/build/lib/syntax/Environment.java
new file mode 100644
index 0000000..a148a70
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Environment.java
@@ -0,0 +1,345 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * The BUILD environment.
+ */
+public class Environment {
+
+ @SkylarkBuiltin(name = "True", returnType = Boolean.class, doc = "Literal for the boolean true.")
+ private static final Boolean TRUE = true;
+
+ @SkylarkBuiltin(name = "False", returnType = Boolean.class,
+ doc = "Literal for the boolean false.")
+ private static final Boolean FALSE = false;
+
+ @SkylarkBuiltin(name = "PACKAGE_NAME", returnType = String.class,
+ doc = "The name of the package the rule or build extension is called from. "
+ + "This variable is special, because its value comes from outside of the extension "
+ + "module (it comes from the BUILD file), so it can only be accessed in functions "
+ + "(transitively) called from BUILD files. For example:<br>"
+ + "<pre class=language-python>def extension():\n"
+ + " return PACKAGE_NAME</pre>"
+ + "In this case calling <code>extension()</code> works from the BUILD file (if the "
+ + "function is loaded), but not as a top level function call in the extension module.")
+ public static final String PKG_NAME = "PACKAGE_NAME";
+
+ /**
+ * There should be only one instance of this type to allow "== None" tests.
+ */
+ @Immutable
+ public static final class NoneType {
+ @Override
+ public String toString() { return "None"; }
+ private NoneType() {}
+ }
+
+ @SkylarkBuiltin(name = "None", returnType = NoneType.class, doc = "Literal for the None value.")
+ public static final NoneType NONE = new NoneType();
+
+ protected final Map<String, Object> env = new HashMap<>();
+
+ // Functions with namespaces. Works only in the global environment.
+ protected final Map<Class<?>, Map<String, Function>> functions = new HashMap<>();
+
+ /**
+ * The parent environment. For Skylark it's the global environment,
+ * used for global read only variable lookup.
+ */
+ protected final Environment parent;
+
+ /**
+ * Map from a Skylark extension to an environment, which contains all symbols defined in the
+ * extension.
+ */
+ protected Map<PathFragment, SkylarkEnvironment> importedExtensions;
+
+ /**
+ * A set of disable variables propagating through function calling. This is needed because
+ * UserDefinedFunctions lock the definition Environment which should be immutable.
+ */
+ protected Set<String> disabledVariables = new HashSet<>();
+
+ /**
+ * A set of disable namespaces propagating through function calling. See disabledVariables.
+ */
+ protected Set<Class<?>> disabledNameSpaces = new HashSet<>();
+
+ /**
+ * A set of variables propagating through function calling. It's only used to call
+ * native rules from Skylark build extensions.
+ */
+ protected Set<String> propagatingVariables = new HashSet<>();
+
+ /**
+ * An EventHandler for errors and warnings. This is not used in the BUILD language,
+ * however it might be used in Skylark code called from the BUILD language.
+ */
+ @Nullable protected EventHandler eventHandler;
+
+ /**
+ * Constructs an empty root non-Skylark environment.
+ * The root environment is also the global environment.
+ */
+ public Environment() {
+ this.parent = null;
+ this.importedExtensions = new HashMap<>();
+ setupGlobal();
+ }
+
+ /**
+ * Constructs an empty child environment.
+ */
+ public Environment(Environment parent) {
+ Preconditions.checkNotNull(parent);
+ this.parent = parent;
+ this.importedExtensions = new HashMap<>();
+ }
+
+ /**
+ * Constructs an empty child environment with an EventHandler.
+ */
+ public Environment(Environment parent, EventHandler eventHandler) {
+ this(parent);
+ this.eventHandler = Preconditions.checkNotNull(eventHandler);
+ }
+
+ // Sets up the global environment
+ private void setupGlobal() {
+ // In Python 2.x, True and False are global values and can be redefined by the user.
+ // In Python 3.x, they are keywords. We implement them as values, for the sake of
+ // simplicity. We define them as Boolean objects.
+ env.put("False", FALSE);
+ env.put("True", TRUE);
+ env.put("None", NONE);
+ }
+
+ public boolean isSkylarkEnabled() {
+ return false;
+ }
+
+ protected boolean hasVariable(String varname) {
+ return env.containsKey(varname);
+ }
+
+ /**
+ * @return the value from the environment whose name is "varname".
+ * @throws NoSuchVariableException if the variable is not defined in the Environment.
+ *
+ */
+ public Object lookup(String varname) throws NoSuchVariableException {
+ if (disabledVariables.contains(varname)) {
+ throw new NoSuchVariableException(varname);
+ }
+ Object value = env.get(varname);
+ if (value == null) {
+ if (parent != null) {
+ return parent.lookup(varname);
+ }
+ throw new NoSuchVariableException(varname);
+ }
+ return value;
+ }
+
+ /**
+ * Like <code>lookup(String)</code>, but instead of throwing an exception in
+ * the case where "varname" is not defined, "defaultValue" is returned instead.
+ *
+ */
+ public Object lookup(String varname, Object defaultValue) {
+ Object value = env.get(varname);
+ if (value == null) {
+ if (parent != null) {
+ return parent.lookup(varname, defaultValue);
+ }
+ return defaultValue;
+ }
+ return value;
+ }
+
+ /**
+ * Updates the value of variable "varname" in the environment, corresponding
+ * to an {@link AssignmentStatement}.
+ */
+ public void update(String varname, Object value) {
+ Preconditions.checkNotNull(value, "update(value == null)");
+ env.put(varname, value);
+ }
+
+ /**
+ * Same as {@link #update}, but also marks the variable propagating, meaning it will
+ * be present in the execution environment of a UserDefinedFunction called from this
+ * Environment. Using this method is discouraged.
+ */
+ public void updateAndPropagate(String varname, Object value) {
+ update(varname, value);
+ propagatingVariables.add(varname);
+ }
+
+ /**
+ * Remove the variable from the environment, returning
+ * any previous mapping (null if there was none).
+ */
+ public Object remove(String varname) {
+ return env.remove(varname);
+ }
+
+ /**
+ * Returns the (immutable) set of names of all variables defined in this
+ * environment. Exposed for testing; not very efficient!
+ */
+ @VisibleForTesting
+ public Set<String> getVariableNames() {
+ if (parent == null) {
+ return env.keySet();
+ } else {
+ Set<String> vars = new HashSet<>();
+ vars.addAll(env.keySet());
+ vars.addAll(parent.getVariableNames());
+ return vars;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ throw new UnsupportedOperationException(); // avoid nondeterminism
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder out = new StringBuilder();
+ out.append("Environment{");
+ List<String> keys = new ArrayList<>(env.keySet());
+ Collections.sort(keys);
+ for (String key: keys) {
+ out.append(key).append(" -> ").append(env.get(key)).append(", ");
+ }
+ out.append("}");
+ if (parent != null) {
+ out.append("=>");
+ out.append(parent.toString());
+ }
+ return out.toString();
+ }
+
+ /**
+ * An exception thrown when an attempt is made to lookup a non-existent
+ * variable in the environment.
+ */
+ public static class NoSuchVariableException extends Exception {
+ NoSuchVariableException(String variable) {
+ super("no such variable: " + variable);
+ }
+ }
+
+ /**
+ * An exception thrown when an attempt is made to import a symbol from a file
+ * that was not properly loaded.
+ */
+ public static class LoadFailedException extends Exception {
+ LoadFailedException(String file) {
+ super("file '" + file + "' was not correctly loaded. Make sure the 'load' statement appears "
+ + "in the global scope, in the BUILD file");
+ }
+ }
+
+ public void setImportedExtensions(Map<PathFragment, SkylarkEnvironment> importedExtensions) {
+ this.importedExtensions = importedExtensions;
+ }
+
+ public void importSymbol(PathFragment extension, String symbol)
+ throws NoSuchVariableException, LoadFailedException {
+ if (!importedExtensions.containsKey(extension)) {
+ throw new LoadFailedException(extension.toString());
+ }
+ Object value = importedExtensions.get(extension).lookup(symbol);
+ if (!isSkylarkEnabled()) {
+ value = SkylarkType.convertFromSkylark(value);
+ }
+ update(symbol, value);
+ }
+
+ /**
+ * Registers a function with namespace to this global environment.
+ */
+ public void registerFunction(Class<?> nameSpace, String name, Function function) {
+ Preconditions.checkArgument(parent == null);
+ if (!functions.containsKey(nameSpace)) {
+ functions.put(nameSpace, new HashMap<String, Function>());
+ }
+ functions.get(nameSpace).put(name, function);
+ }
+
+ private Map<String, Function> getNamespaceFunctions(Class<?> nameSpace) {
+ if (disabledNameSpaces.contains(nameSpace)
+ || (parent != null && parent.disabledNameSpaces.contains(nameSpace))) {
+ return null;
+ }
+ Environment topLevel = this;
+ while (topLevel.parent != null) {
+ topLevel = topLevel.parent;
+ }
+ return topLevel.functions.get(nameSpace);
+ }
+
+ /**
+ * Returns the function of the namespace of the given name or null of it does not exists.
+ */
+ public Function getFunction(Class<?> nameSpace, String name) {
+ Map<String, Function> nameSpaceFunctions = getNamespaceFunctions(nameSpace);
+ return nameSpaceFunctions != null ? nameSpaceFunctions.get(name) : null;
+ }
+
+ /**
+ * Returns the function names registered with the namespace.
+ */
+ public Set<String> getFunctionNames(Class<?> nameSpace) {
+ Map<String, Function> nameSpaceFunctions = getNamespaceFunctions(nameSpace);
+ return nameSpaceFunctions != null ? nameSpaceFunctions.keySet() : ImmutableSet.<String>of();
+ }
+
+ /**
+ * Return the current stack trace (list of function names).
+ */
+ public ImmutableList<String> getStackTrace() {
+ // Empty list, since this environment does not allow function definition
+ // (see SkylarkEnvironment)
+ return ImmutableList.of();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/EvalException.java b/src/main/java/com/google/devtools/build/lib/syntax/EvalException.java
new file mode 100644
index 0000000..27aba0f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/EvalException.java
@@ -0,0 +1,105 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.events.Location;
+
+/**
+ * Exceptions thrown during evaluation of BUILD ASTs or Skylark extensions.
+ *
+ * <p>This exception must always correspond to a repeatable, permanent error, i.e. evaluating the
+ * same package again must yield the same exception. Notably, do not use this for reporting I/O
+ * errors.
+ *
+ * <p>This requirement is in place so that we can cache packages where an error is reported by way
+ * of {@link EvalException}.
+ */
+public class EvalException extends Exception {
+
+ private final Location location;
+ private final String message;
+ private final boolean dueToIncompleteAST;
+
+ /**
+ * @param location the location where evaluation/execution failed.
+ * @param message the error message.
+ */
+ public EvalException(Location location, String message) {
+ this.location = location;
+ this.message = Preconditions.checkNotNull(message);
+ this.dueToIncompleteAST = false;
+ }
+
+ /**
+ * @param location the location where evaluation/execution failed.
+ * @param message the error message.
+ * @param dueToIncompleteAST if the error is caused by a previous error, such as parsing.
+ */
+ public EvalException(Location location, String message, boolean dueToIncompleteAST) {
+ this.location = location;
+ this.message = Preconditions.checkNotNull(message);
+ this.dueToIncompleteAST = dueToIncompleteAST;
+ }
+
+ private EvalException(Location location, Throwable cause) {
+ super(cause);
+ this.location = location;
+ // This is only used from Skylark, it's useful for debugging. Note that this only happens
+ // when the Precondition below kills the execution anyway.
+ if (cause.getMessage() == null) {
+ cause.printStackTrace();
+ }
+ this.message = Preconditions.checkNotNull(cause.getMessage());
+ this.dueToIncompleteAST = false;
+ }
+
+ /**
+ * Returns the error message.
+ */
+ @Override
+ public String getMessage() {
+ return message;
+ }
+
+ /**
+ * Returns the location of the evaluation error.
+ */
+ public Location getLocation() {
+ return location;
+ }
+
+ public boolean isDueToIncompleteAST() {
+ return dueToIncompleteAST;
+ }
+
+ /**
+ * A class to support a special case of EvalException when the cause of the error is an
+ * Exception during a direct Java call.
+ */
+ public static final class EvalExceptionWithJavaCause extends EvalException {
+
+ public EvalExceptionWithJavaCause(Location location, Throwable cause) {
+ super(location, cause);
+ }
+ }
+
+ /**
+ * Returns the error message with location info if exists.
+ */
+ public String print() {
+ return getLocation() == null ? getMessage() : getLocation().print() + ": " + getMessage();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/EvalUtils.java b/src/main/java/com/google/devtools/build/lib/syntax/EvalUtils.java
new file mode 100644
index 0000000..70d89bc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/EvalUtils.java
@@ -0,0 +1,590 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.ClassObject.SkylarkClassObject;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Formattable;
+import java.util.Formatter;
+import java.util.IllegalFormatException;
+import java.util.List;
+import java.util.Map;
+import java.util.MissingFormatWidthException;
+import java.util.Set;
+
+/**
+ * Utilities used by the evaluator.
+ */
+public abstract class EvalUtils {
+
+ // TODO(bazel-team): Yet an other hack committed in the name of Skylark. One problem is that the
+ // syntax package cannot depend on actions so we have to have this until Actions are immutable.
+ // The other is that BuildConfigurations are technically not immutable but they cannot be modified
+ // from Skylark.
+ private static final ImmutableSet<Class<?>> quasiImmutableClasses;
+ static {
+ try {
+ ImmutableSet.Builder<Class<?>> builder = ImmutableSet.builder();
+ builder.add(Class.forName("com.google.devtools.build.lib.actions.Action"));
+ builder.add(Class.forName("com.google.devtools.build.lib.analysis.config.BuildConfiguration"));
+ builder.add(Class.forName("com.google.devtools.build.lib.actions.Root"));
+ quasiImmutableClasses = builder.build();
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private EvalUtils() {
+ }
+
+ /**
+ * @return true if the specified sequence is a tuple; false if it's a modifiable list.
+ */
+ public static boolean isTuple(List<?> l) {
+ return isTuple(l.getClass());
+ }
+
+ public static boolean isTuple(Class<?> c) {
+ Preconditions.checkState(List.class.isAssignableFrom(c));
+ if (ImmutableList.class.isAssignableFrom(c)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * @return true if the specified value is immutable (suitable for use as a
+ * dictionary key) according to the rules of the Build language.
+ */
+ public static boolean isImmutable(Object o) {
+ if (o instanceof Map<?, ?> || o instanceof Function
+ || o instanceof FilesetEntry || o instanceof GlobList<?>) {
+ return false;
+ } else if (o instanceof List<?>) {
+ return isTuple((List<?>) o); // tuples are immutable, lists are not.
+ } else {
+ return true; // string/int
+ }
+ }
+
+ /**
+ * Returns true if the type is immutable in the skylark language.
+ */
+ public static boolean isSkylarkImmutable(Class<?> c) {
+ if (c.isAnnotationPresent(Immutable.class)) {
+ return true;
+ } else if (c.equals(String.class) || c.equals(Integer.class) || c.equals(Boolean.class)
+ || SkylarkList.class.isAssignableFrom(c) || ImmutableMap.class.isAssignableFrom(c)
+ || NestedSet.class.isAssignableFrom(c)) {
+ return true;
+ } else {
+ for (Class<?> classObject : quasiImmutableClasses) {
+ if (classObject.isAssignableFrom(c)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns a transitive superclass or interface implemented by c which is annotated
+ * with SkylarkModule. Returns null if no such class or interface exists.
+ */
+ @VisibleForTesting
+ static Class<?> getParentWithSkylarkModule(Class<?> c) {
+ if (c == null) {
+ return null;
+ }
+ if (c.isAnnotationPresent(SkylarkModule.class)) {
+ return c;
+ }
+ Class<?> parent = getParentWithSkylarkModule(c.getSuperclass());
+ if (parent != null) {
+ return parent;
+ }
+ for (Class<?> ifparent : c.getInterfaces()) {
+ ifparent = getParentWithSkylarkModule(ifparent);
+ if (ifparent != null) {
+ return ifparent;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns the Skylark equivalent type of the parameter. Note that the Skylark
+ * language doesn't have inheritance.
+ */
+ public static Class<?> getSkylarkType(Class<?> c) {
+ if (ImmutableList.class.isAssignableFrom(c)) {
+ return ImmutableList.class;
+ } else if (List.class.isAssignableFrom(c)) {
+ return List.class;
+ } else if (SkylarkList.class.isAssignableFrom(c)) {
+ return SkylarkList.class;
+ } else if (Map.class.isAssignableFrom(c)) {
+ return Map.class;
+ } else if (NestedSet.class.isAssignableFrom(c)) {
+ // This could be removed probably
+ return NestedSet.class;
+ } else if (Set.class.isAssignableFrom(c)) {
+ return Set.class;
+ } else {
+ // Check if one of the superclasses or implemented interfaces has the SkylarkModule
+ // annotation. If yes return that class.
+ Class<?> parent = getParentWithSkylarkModule(c);
+ if (parent != null) {
+ return parent;
+ }
+ }
+ return c;
+ }
+
+ /**
+ * Returns a pretty name for the datatype of object 'o' in the Build language.
+ */
+ public static String getDatatypeName(Object o) {
+ Preconditions.checkNotNull(o);
+ if (o instanceof SkylarkList) {
+ return ((SkylarkList) o).isTuple() ? "tuple" : "list";
+ }
+ return getDataTypeNameFromClass(o.getClass());
+ }
+
+ /**
+ * Returns a pretty name for the datatype equivalent of class 'c' in the Build language.
+ */
+ public static String getDataTypeNameFromClass(Class<?> c) {
+ if (c.equals(Object.class)) {
+ return "unknown";
+ } else if (c.equals(String.class)) {
+ return "string";
+ } else if (c.equals(Integer.class)) {
+ return "int";
+ } else if (c.equals(Boolean.class)) {
+ return "bool";
+ } else if (c.equals(Void.TYPE) || c.equals(Environment.NoneType.class)) {
+ return "None";
+ } else if (List.class.isAssignableFrom(c)) {
+ return isTuple(c) ? "tuple" : "list";
+ } else if (GlobList.class.isAssignableFrom(c)) {
+ return "list";
+ } else if (Map.class.isAssignableFrom(c)) {
+ return "dict";
+ } else if (Function.class.isAssignableFrom(c)) {
+ return "function";
+ } else if (c.equals(FilesetEntry.class)) {
+ return "FilesetEntry";
+ } else if (NestedSet.class.isAssignableFrom(c) || SkylarkNestedSet.class.isAssignableFrom(c)) {
+ return "set";
+ } else if (SkylarkClassObject.class.isAssignableFrom(c)) {
+ return "struct";
+ } else if (SkylarkList.class.isAssignableFrom(c)) {
+ // TODO(bazel-team): this is not entirely correct, it can also be a tuple.
+ return "list";
+ } else if (c.isAnnotationPresent(SkylarkModule.class)) {
+ SkylarkModule module = c.getAnnotation(SkylarkModule.class);
+ return c.getAnnotation(SkylarkModule.class).name()
+ + (module.namespace() ? " (a language module)" : "");
+ } else {
+ if (c.getSimpleName().isEmpty()) {
+ return c.getName();
+ } else {
+ return c.getSimpleName();
+ }
+ }
+ }
+
+ /**
+ * Returns a sequence of the appropriate list/tuple datatype for 'seq', based on 'isTuple'.
+ */
+ public static List<?> makeSequence(List<?> seq, boolean isTuple) {
+ return isTuple ? ImmutableList.copyOf(seq) : seq;
+ }
+
+ /**
+ * Print build-language value 'o' in display format into the specified buffer.
+ */
+ public static void printValue(Object o, Appendable buffer) {
+ // Exception-swallowing wrapper due to annoying Appendable interface.
+ try {
+ printValueX(o, buffer);
+ } catch (IOException e) {
+ throw new AssertionError(e); // can't happen
+ }
+ }
+
+ private static void printValueX(Object o, Appendable buffer)
+ throws IOException {
+ if (o == null) {
+ throw new NullPointerException(); // None is not a build language value.
+ } else if (o instanceof String ||
+ o instanceof Integer ||
+ o instanceof Double) {
+ buffer.append(o.toString());
+
+ } else if (o == Environment.NONE) {
+ buffer.append("None");
+
+ } else if (o == Boolean.TRUE) {
+ buffer.append("True");
+
+ } else if (o == Boolean.FALSE) {
+ buffer.append("False");
+
+ } else if (o instanceof List<?>) {
+ List<?> seq = (List<?>) o;
+ boolean isTuple = isImmutable(seq);
+ String sep = "";
+ buffer.append(isTuple ? '(' : '[');
+ for (int ii = 0, len = seq.size(); ii < len; ++ii) {
+ buffer.append(sep);
+ prettyPrintValue(seq.get(ii), buffer);
+ sep = ", ";
+ }
+ buffer.append(isTuple ? ')' : ']');
+
+ } else if (o instanceof Map<?, ?>) {
+ Map<?, ?> dict = (Map<?, ?>) o;
+ buffer.append('{');
+ String sep = "";
+ for (Map.Entry<?, ?> entry : dict.entrySet()) {
+ buffer.append(sep);
+ prettyPrintValue(entry.getKey(), buffer);
+ buffer.append(": ");
+ prettyPrintValue(entry.getValue(), buffer);
+ sep = ", ";
+ }
+ buffer.append('}');
+
+ } else if (o instanceof Function) {
+ Function func = (Function) o;
+ buffer.append("<function " + func.getName() + ">");
+
+ } else if (o instanceof FilesetEntry) {
+ FilesetEntry entry = (FilesetEntry) o;
+ buffer.append("FilesetEntry(srcdir = ");
+ prettyPrintValue(entry.getSrcLabel().toString(), buffer);
+ buffer.append(", files = ");
+ prettyPrintValue(makeStringList(entry.getFiles()), buffer);
+ buffer.append(", excludes = ");
+ prettyPrintValue(makeList(entry.getExcludes()), buffer);
+ buffer.append(", destdir = ");
+ prettyPrintValue(entry.getDestDir().getPathString(), buffer);
+ buffer.append(", strip_prefix = ");
+ prettyPrintValue(entry.getStripPrefix(), buffer);
+ buffer.append(", symlinks = \"");
+ buffer.append(entry.getSymlinkBehavior().toString());
+ buffer.append("\")");
+ } else if (o instanceof PathFragment) {
+ buffer.append(((PathFragment) o).getPathString());
+ } else {
+ buffer.append(o.toString());
+ }
+ }
+
+ private static List<?> makeList(Collection<?> list) {
+ return list == null ? Lists.newArrayList() : Lists.newArrayList(list);
+ }
+
+ private static List<String> makeStringList(List<Label> labels) {
+ if (labels == null) { return Collections.emptyList(); }
+ List<String> strings = Lists.newArrayListWithCapacity(labels.size());
+ for (Label label : labels) {
+ strings.add(label.toString());
+ }
+ return strings;
+ }
+
+ /**
+ * Print build-language value 'o' in parseable format into the specified
+ * buffer. (Only differs from printValueX in treatment of strings at toplevel,
+ * i.e. not within a sequence or dict)
+ */
+ public static void prettyPrintValue(Object o, Appendable buffer) {
+ // Exception-swallowing wrapper due to annoying Appendable interface.
+ try {
+ prettyPrintValueX(o, buffer);
+ } catch (IOException e) {
+ throw new AssertionError(e); // can't happen
+ }
+ }
+
+ private static void prettyPrintValueX(Object o, Appendable buffer)
+ throws IOException {
+ if (o instanceof Label) {
+ o = o.toString(); // Pretty-print a label like a string
+ }
+ if (o instanceof String) {
+ String s = (String) o;
+ buffer.append('"');
+ for (int ii = 0, len = s.length(); ii < len; ++ii) {
+ char c = s.charAt(ii);
+ switch (c) {
+ case '\r':
+ buffer.append('\\').append('r');
+ break;
+ case '\n':
+ buffer.append('\\').append('n');
+ break;
+ case '\t':
+ buffer.append('\\').append('t');
+ break;
+ case '\"':
+ buffer.append('\\').append('"');
+ break;
+ default:
+ if (c < 32) {
+ buffer.append(String.format("\\x%02x", (int) c));
+ } else {
+ buffer.append(c); // no need to support UTF-8
+ }
+ } // endswitch
+ }
+ buffer.append('\"');
+ } else {
+ printValueX(o, buffer);
+ }
+ }
+
+ /**
+ * Pretty-print value 'o' to a string. Convenience overloading of
+ * prettyPrintValue(Object, Appendable).
+ */
+ public static String prettyPrintValue(Object o) {
+ StringBuffer buffer = new StringBuffer();
+ prettyPrintValue(o, buffer);
+ return buffer.toString();
+ }
+
+ /**
+ * Pretty-print values of 'o' separated by the separator.
+ */
+ public static String prettyPrintValues(String separator, Iterable<Object> o) {
+ return Joiner.on(separator).join(Iterables.transform(o,
+ new com.google.common.base.Function<Object, String>() {
+ @Override
+ public String apply(Object input) {
+ return prettyPrintValue(input);
+ }
+ }));
+ }
+
+ /**
+ * Print value 'o' to a string. Convenience overloading of printValue(Object, Appendable).
+ */
+ public static String printValue(Object o) {
+ StringBuffer buffer = new StringBuffer();
+ printValue(o, buffer);
+ return buffer.toString();
+ }
+
+ public static Object checkNotNull(Expression expr, Object obj) throws EvalException {
+ if (obj == null) {
+ throw new EvalException(expr.getLocation(),
+ "Unexpected null value, please send a bug report. "
+ + "This was generated by '" + expr + "'");
+ }
+ return obj;
+ }
+
+ /**
+ * Convert BUILD language objects to Formattable so JDK can render them correctly.
+ * Don't do this for numeric or string types because we want %d, %x, %s to work.
+ */
+ private static Object makeFormattable(final Object o) {
+ if (o instanceof Integer || o instanceof Double || o instanceof String) {
+ return o;
+ } else {
+ return new Formattable() {
+ @Override
+ public String toString() {
+ return "Formattable[" + o + "]";
+ }
+
+ @Override
+ public void formatTo(Formatter formatter, int flags, int width,
+ int precision) {
+ printValue(o, formatter.out());
+ }
+ };
+ }
+ }
+
+ private static final Object[] EMPTY = new Object[0];
+
+ /*
+ * N.B. MissingFormatWidthException is the only kind of IllegalFormatException
+ * whose constructor can take and display arbitrary error message, hence its use below.
+ */
+
+ /**
+ * Perform Python-style string formatting. Implemented by delegation to Java's
+ * own string formatting routine to avoid reinventing the wheel. In more
+ * obscure cases, semantics follow JDK (not Python) rules.
+ *
+ * @param pattern a format string.
+ * @param tuple a tuple containing positional arguments
+ */
+ public static String formatString(String pattern, List<?> tuple)
+ throws IllegalFormatException {
+ int count = countPlaceholders(pattern);
+ if (count < tuple.size()) {
+ throw new MissingFormatWidthException(
+ "not all arguments converted during string formatting");
+ }
+
+ List<Object> args = new ArrayList<>();
+
+ for (Object o : tuple) {
+ args.add(makeFormattable(o));
+ }
+
+ try {
+ return String.format(pattern, args.toArray(EMPTY));
+ } catch (IllegalFormatException e) {
+ throw new MissingFormatWidthException(
+ "invalid arguments for format string");
+ }
+ }
+
+ private static int countPlaceholders(String pattern) {
+ int length = pattern.length();
+ boolean afterPercent = false;
+ int i = 0;
+ int count = 0;
+ while (i < length) {
+ switch (pattern.charAt(i)) {
+ case 's':
+ case 'd':
+ if (afterPercent) {
+ count++;
+ afterPercent = false;
+ }
+ break;
+
+ case '%':
+ afterPercent = !afterPercent;
+ break;
+
+ default:
+ if (afterPercent) {
+ throw new MissingFormatWidthException("invalid arguments for format string");
+ }
+ afterPercent = false;
+ break;
+ }
+ i++;
+ }
+
+ return count;
+ }
+
+ /**
+ * @return the truth value of an object, according to Python rules.
+ * http://docs.python.org/2/library/stdtypes.html#truth-value-testing
+ */
+ public static boolean toBoolean(Object o) {
+ if (o == null || o == Environment.NONE) {
+ return false;
+ } else if (o instanceof Boolean) {
+ return (Boolean) o;
+ } else if (o instanceof String) {
+ return !((String) o).isEmpty();
+ } else if (o instanceof Integer) {
+ return (Integer) o != 0;
+ } else if (o instanceof Collection<?>) {
+ return !((Collection<?>) o).isEmpty();
+ } else if (o instanceof Map<?, ?>) {
+ return !((Map<?, ?>) o).isEmpty();
+ } else if (o instanceof NestedSet<?>) {
+ return !((NestedSet<?>) o).isEmpty();
+ } else if (o instanceof SkylarkNestedSet) {
+ return !((SkylarkNestedSet) o).isEmpty();
+ } else if (o instanceof Iterable<?>) {
+ return !(Iterables.isEmpty((Iterable<?>) o));
+ } else {
+ return true;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public static Collection<Object> toCollection(Object o, Location loc) throws EvalException {
+ if (o instanceof Collection) {
+ return (Collection<Object>) o;
+ } else if (o instanceof Map<?, ?>) {
+ // For dictionaries we iterate through the keys only
+ return ((Map<Object, Object>) o).keySet();
+ } else if (o instanceof SkylarkNestedSet) {
+ return ((SkylarkNestedSet) o).toCollection();
+ } else {
+ throw new EvalException(loc,
+ "type '" + EvalUtils.getDatatypeName(o) + "' is not a collection");
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public static Iterable<Object> toIterable(Object o, Location loc) throws EvalException {
+ if (o instanceof String) {
+ // This is not as efficient as special casing String in for and dict and list comprehension
+ // statements. However this is a more unified way.
+ // The regex matches every character in the string until the end of the string,
+ // so "abc" will be split into ["a", "b", "c"].
+ return ImmutableList.<Object>copyOf(((String) o).split("(?!^)"));
+ } else if (o instanceof Iterable) {
+ return (Iterable<Object>) o;
+ } else if (o instanceof Map<?, ?>) {
+ // For dictionaries we iterate through the keys only
+ return ((Map<Object, Object>) o).keySet();
+ } else {
+ throw new EvalException(loc,
+ "type '" + EvalUtils.getDatatypeName(o) + "' is not an iterable");
+ }
+ }
+
+ /**
+ * Returns the size of the Skylark object or -1 in case the object doesn't have a size.
+ */
+ public static int size(Object arg) {
+ if (arg instanceof String) {
+ return ((String) arg).length();
+ } else if (arg instanceof Map) {
+ return ((Map<?, ?>) arg).size();
+ } else if (arg instanceof SkylarkList) {
+ return ((SkylarkList) arg).size();
+ } else if (arg instanceof Iterable) {
+ // Iterables.size() checks if arg is a Collection so it's efficient in that sense.
+ return Iterables.size((Iterable<?>) arg);
+ }
+ return -1;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Expression.java b/src/main/java/com/google/devtools/build/lib/syntax/Expression.java
new file mode 100644
index 0000000..1659eb0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Expression.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Base class for all expression nodes in the AST.
+ */
+public abstract class Expression extends ASTNode {
+
+ /**
+ * Returns the result of evaluating this build-language expression in the
+ * specified environment. All BUILD language datatypes are mapped onto the
+ * corresponding Java types as follows:
+ *
+ * <pre>
+ * int -> Integer
+ * float -> Double (currently not generated by the grammar)
+ * str -> String
+ * [...] -> List<Object> (mutable)
+ * (...) -> List<Object> (immutable)
+ * {...} -> Map<Object, Object>
+ * func -> Function
+ * </pre>
+ *
+ * @return the result of evaluting the expression: a Java object corresponding
+ * to a datatype in the BUILD language.
+ * @throws EvalException if the expression could not be evaluated.
+ */
+ abstract Object eval(Environment env) throws EvalException, InterruptedException;
+
+ /**
+ * Returns the inferred type of the result of the Expression.
+ *
+ * <p>Checks the semantics of the Expression using the SkylarkEnvironment according to
+ * the rules of the Skylark language, throws EvalException in case of a semantical error.
+ *
+ * @see Statement
+ */
+ abstract SkylarkType validate(ValidationEnvironment env) throws EvalException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ExpressionStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/ExpressionStatement.java
new file mode 100644
index 0000000..f742d40
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ExpressionStatement.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Syntax node for a function call statement. Used for build rules.
+ */
+public final class ExpressionStatement extends Statement {
+
+ private final Expression expr;
+
+ public ExpressionStatement(Expression expr) {
+ this.expr = expr;
+ }
+
+ public Expression getExpression() {
+ return expr;
+ }
+
+ @Override
+ public String toString() {
+ return expr.toString() + '\n';
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ expr.eval(env);
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ expr.validate(env);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/FilesetEntry.java b/src/main/java/com/google/devtools/build/lib/syntax/FilesetEntry.java
new file mode 100644
index 0000000..4586b64
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/FilesetEntry.java
@@ -0,0 +1,175 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * FilesetEntry is a value object used to represent a "FilesetEntry" inside a "Fileset" BUILD rule.
+ */
+public final class FilesetEntry {
+ /** SymlinkBehavior decides what to do when a source file of a FilesetEntry is a symlink. */
+ public enum SymlinkBehavior {
+ /** Just copies the symlink as-is. May result in dangling links. */
+ COPY,
+ /** Follow the link and make the destination point to the absolute path of the final target. */
+ DEREFERENCE;
+
+ public static SymlinkBehavior parse(String value) throws IllegalArgumentException {
+ return valueOf(value.toUpperCase());
+ }
+
+ @Override
+ public String toString() {
+ return super.toString().toLowerCase();
+ }
+ }
+
+ private final Label srcLabel;
+ @Nullable private final ImmutableList<Label> files;
+ @Nullable private final ImmutableSet<String> excludes;
+ private final PathFragment destDir;
+ private final SymlinkBehavior symlinkBehavior;
+ private final String stripPrefix;
+
+ /**
+ * Constructs a FilesetEntry with the given values.
+ *
+ * @param srcLabel the label of the source directory. Must be non-null.
+ * @param files The explicit files to include. May be null.
+ * @param excludes The files to exclude. Man be null. May only be non-null if files is null.
+ * @param destDir The target-relative output directory.
+ * @param symlinkBehavior how to treat symlinks on the input. See
+ * {@link FilesetEntry.SymlinkBehavior}.
+ * @param stripPrefix the prefix to strip from the package-relative path. If ".", keep only the
+ * basename.
+ */
+ public FilesetEntry(Label srcLabel,
+ @Nullable List<Label> files,
+ @Nullable List<String> excludes,
+ String destDir,
+ SymlinkBehavior symlinkBehavior,
+ String stripPrefix) {
+ this.srcLabel = checkNotNull(srcLabel);
+ this.destDir = new PathFragment((destDir == null) ? "" : destDir);
+ this.files = files == null ? null : ImmutableList.copyOf(files);
+ this.excludes = (excludes == null || excludes.isEmpty()) ? null : ImmutableSet.copyOf(excludes);
+ this.symlinkBehavior = symlinkBehavior;
+ this.stripPrefix = stripPrefix;
+ }
+
+ /**
+ * @return the source label.
+ */
+ public Label getSrcLabel() {
+ return srcLabel;
+ }
+
+ /**
+ * @return the destDir. Non null.
+ */
+ public PathFragment getDestDir() {
+ return destDir;
+ }
+
+ /**
+ * @return how symlinks should be handled.
+ */
+ public SymlinkBehavior getSymlinkBehavior() {
+ return symlinkBehavior;
+ }
+
+ /**
+ * @return an immutable list of excludes. Null if none specified.
+ */
+ @Nullable
+ public ImmutableSet<String> getExcludes() {
+ return excludes;
+ }
+
+ /**
+ * @return an immutable list of file labels. Null if none specified.
+ */
+ @Nullable
+ public ImmutableList<Label> getFiles() {
+ return files;
+ }
+
+ /**
+ * @return true if this Fileset should get files from the source directory.
+ */
+ public boolean isSourceFileset() {
+ return "BUILD".equals(srcLabel.getName());
+ }
+
+ /**
+ * @return all prerequisite labels in the FilesetEntry.
+ */
+ public Collection<Label> getLabels() {
+ Set<Label> labels = new LinkedHashSet<>();
+ if (files != null) {
+ labels.addAll(files);
+ } else {
+ labels.add(srcLabel);
+ }
+ return labels;
+ }
+
+ /**
+ * @return the prefix that should be stripped from package-relative path names.
+ */
+ public String getStripPrefix() {
+ return stripPrefix;
+ }
+
+ /**
+ * @return null if the entry is valid, and a human-readable error message otherwise.
+ */
+ @Nullable
+ public String validate() {
+ if (excludes != null && files != null) {
+ return "Cannot specify both 'files' and 'excludes' in a FilesetEntry";
+ } else if (files != null && !isSourceFileset()) {
+ return "Cannot specify files with Fileset label '" + srcLabel + "'";
+ } else if (destDir.isAbsolute()) {
+ return "Cannot specify absolute destdir '" + destDir + "'";
+ } else if (!stripPrefix.equals(".") && files == null) {
+ return "If the strip prefix is not '.', files must be specified";
+ } else if (new PathFragment(stripPrefix).containsUplevelReferences()) {
+ return "Strip prefix must not contain uplevel references";
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return String.format("FilesetEntry(srcdir=%s, destdir=%s, strip_prefix=%s, symlinks=%s, "
+ + "%d file(s) and %d excluded)", srcLabel, destDir, stripPrefix, symlinkBehavior,
+ files != null ? files.size() : 0,
+ excludes != null ? excludes.size() : 0);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ForStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/ForStatement.java
new file mode 100644
index 0000000..34a4eea
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ForStatement.java
@@ -0,0 +1,97 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * Syntax node for a for loop statement.
+ */
+public final class ForStatement extends Statement {
+
+ private final Ident variable;
+ private final Expression collection;
+ private final ImmutableList<Statement> block;
+
+ /**
+ * Constructs a for loop statement.
+ */
+ ForStatement(Ident variable, Expression collection, List<Statement> block) {
+ this.variable = Preconditions.checkNotNull(variable);
+ this.collection = Preconditions.checkNotNull(collection);
+ this.block = ImmutableList.copyOf(block);
+ }
+
+ public Ident getVariable() {
+ return variable;
+ }
+
+ public Expression getCollection() {
+ return collection;
+ }
+
+ public ImmutableList<Statement> block() {
+ return block;
+ }
+
+ @Override
+ public String toString() {
+ // TODO(bazel-team): if we want to print the complete statement, the function
+ // needs an extra argument to specify indentation level.
+ return "for " + variable + " in " + collection + ": ...\n";
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ Object o = collection.eval(env);
+ Iterable<?> col = EvalUtils.toIterable(o, getLocation());
+
+ int i = 0;
+ for (Object it : ImmutableList.copyOf(col)) {
+ env.update(variable.getName(), it);
+ for (Statement stmt : block) {
+ stmt.exec(env);
+ }
+ i++;
+ }
+ // TODO(bazel-team): This should not happen if every collection is immutable.
+ if (i != EvalUtils.size(col)) {
+ throw new EvalException(getLocation(),
+ String.format("Cannot modify '%s' during during iteration.", collection.toString()));
+ }
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ if (env.isTopLevel()) {
+ throw new EvalException(getLocation(),
+ "'For' is not allowed as a the top level statement");
+ }
+ // TODO(bazel-team): validate variable. Maybe make it temporarily readonly.
+ SkylarkType type = collection.validate(env);
+ env.checkIterable(type, getLocation());
+ env.update(variable.getName(), SkylarkType.UNKNOWN, getLocation());
+ for (Statement stmt : block) {
+ stmt.validate(env);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/FuncallExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/FuncallExpression.java
new file mode 100644
index 0000000..e24d97f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/FuncallExpression.java
@@ -0,0 +1,550 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.EvalException.EvalExceptionWithJavaCause;
+import com.google.devtools.build.lib.util.StringUtilities;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Syntax node for a function call expression.
+ */
+public final class FuncallExpression extends Expression {
+
+ private static enum ArgConversion {
+ FROM_SKYLARK,
+ TO_SKYLARK,
+ NO_CONVERSION
+ }
+
+ /**
+ * A value class to store Methods with their corresponding SkylarkCallable annotations.
+ * This is needed because the annotation is sometimes in a superclass.
+ */
+ public static final class MethodDescriptor {
+ private final Method method;
+ private final SkylarkCallable annotation;
+
+ private MethodDescriptor(Method method, SkylarkCallable annotation) {
+ this.method = method;
+ this.annotation = annotation;
+ }
+
+ Method getMethod() {
+ return method;
+ }
+
+ /**
+ * Returns the SkylarkCallable annotation corresponding to this method.
+ */
+ public SkylarkCallable getAnnotation() {
+ return annotation;
+ }
+ }
+
+ private static final LoadingCache<Class<?>, Map<String, List<MethodDescriptor>>> methodCache =
+ CacheBuilder.newBuilder()
+ .initialCapacity(10)
+ .maximumSize(100)
+ .build(new CacheLoader<Class<?>, Map<String, List<MethodDescriptor>>>() {
+
+ @Override
+ public Map<String, List<MethodDescriptor>> load(Class<?> key) throws Exception {
+ Map<String, List<MethodDescriptor>> methodMap = new HashMap<>();
+ for (Method method : key.getMethods()) {
+ // Synthetic methods lead to false multiple matches
+ if (method.isSynthetic()) {
+ continue;
+ }
+ SkylarkCallable callable = getAnnotationFromParentClass(
+ method.getDeclaringClass(), method);
+ if (callable == null) {
+ continue;
+ }
+ String name = callable.name();
+ if (name.isEmpty()) {
+ name = StringUtilities.toPythonStyleFunctionName(method.getName());
+ }
+ String signature = name + "#" + method.getParameterTypes().length;
+ if (methodMap.containsKey(signature)) {
+ methodMap.get(signature).add(new MethodDescriptor(method, callable));
+ } else {
+ methodMap.put(signature, Lists.newArrayList(new MethodDescriptor(method, callable)));
+ }
+ }
+ return ImmutableMap.copyOf(methodMap);
+ }
+ });
+
+ /**
+ * Returns a map of methods and corresponding SkylarkCallable annotations
+ * of the methods of the classObj class reachable from Skylark.
+ */
+ public static ImmutableMap<Method, SkylarkCallable> collectSkylarkMethodsWithAnnotation(
+ Class<?> classObj) {
+ ImmutableMap.Builder<Method, SkylarkCallable> methodMap = ImmutableMap.builder();
+ for (Method method : classObj.getMethods()) {
+ // Synthetic methods lead to false multiple matches
+ if (!method.isSynthetic()) {
+ SkylarkCallable annotation = getAnnotationFromParentClass(classObj, method);
+ if (annotation != null) {
+ methodMap.put(method, annotation);
+ }
+ }
+ }
+ return methodMap.build();
+ }
+
+ private static SkylarkCallable getAnnotationFromParentClass(Class<?> classObj, Method method) {
+ boolean keepLooking = false;
+ try {
+ Method superMethod = classObj.getMethod(method.getName(), method.getParameterTypes());
+ if (classObj.isAnnotationPresent(SkylarkModule.class)
+ && superMethod.isAnnotationPresent(SkylarkCallable.class)) {
+ return superMethod.getAnnotation(SkylarkCallable.class);
+ } else {
+ keepLooking = true;
+ }
+ } catch (NoSuchMethodException e) {
+ // The class might not have the specified method, so an exceptions is OK.
+ keepLooking = true;
+ }
+ if (keepLooking) {
+ if (classObj.getSuperclass() != null) {
+ SkylarkCallable annotation = getAnnotationFromParentClass(classObj.getSuperclass(), method);
+ if (annotation != null) {
+ return annotation;
+ }
+ }
+ for (Class<?> interfaceObj : classObj.getInterfaces()) {
+ SkylarkCallable annotation = getAnnotationFromParentClass(interfaceObj, method);
+ if (annotation != null) {
+ return annotation;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * An exception class to handle exceptions in direct Java API calls.
+ */
+ public static final class FuncallException extends Exception {
+
+ public FuncallException(String msg) {
+ super(msg);
+ }
+ }
+
+ private final Expression obj;
+
+ private final Ident func;
+
+ private final List<Argument> args;
+
+ private final int numPositionalArgs;
+
+ /**
+ * Note: the grammar definition restricts the function value in a function
+ * call expression to be a global identifier; however, the representation of
+ * values in the interpreter is flexible enough to allow functions to be
+ * arbitrary expressions. In any case, the "func" expression is always
+ * evaluated, so functions and variables share a common namespace.
+ */
+ public FuncallExpression(Expression obj, Ident func,
+ List<Argument> args) {
+ for (Argument arg : args) {
+ Preconditions.checkArgument(arg.hasValue());
+ }
+ this.obj = obj;
+ this.func = func;
+ this.args = args;
+ this.numPositionalArgs = countPositionalArguments();
+ }
+
+ /**
+ * Note: the grammar definition restricts the function value in a function
+ * call expression to be a global identifier; however, the representation of
+ * values in the interpreter is flexible enough to allow functions to be
+ * arbitrary expressions. In any case, the "func" expression is always
+ * evaluated, so functions and variables share a common namespace.
+ */
+ public FuncallExpression(Ident func, List<Argument> args) {
+ this(null, func, args);
+ }
+
+ /**
+ * Returns the number of positional arguments.
+ */
+ private int countPositionalArguments() {
+ int num = 0;
+ for (Argument arg : args) {
+ if (arg.isPositional()) {
+ num++;
+ }
+ }
+ return num;
+ }
+
+ /**
+ * Returns the function expression.
+ */
+ public Ident getFunction() {
+ return func;
+ }
+
+ /**
+ * Returns the object the function called on.
+ * It's null if the function is not called on an object.
+ */
+ public Expression getObject() {
+ return obj;
+ }
+
+ /**
+ * Returns an (immutable, ordered) list of function arguments. The first n are
+ * positional and the remaining ones are keyword args, where n =
+ * getNumPositionalArguments().
+ */
+ public List<Argument> getArguments() {
+ return Collections.unmodifiableList(args);
+ }
+
+ /**
+ * Returns the number of arguments which are positional; the remainder are
+ * keyword arguments.
+ */
+ public int getNumPositionalArguments() {
+ return numPositionalArgs;
+ }
+
+ @Override
+ public String toString() {
+ if (func.getName().equals("$substring")) {
+ return obj + "[" + args.get(0) + ":" + args.get(1) + "]";
+ }
+ if (func.getName().equals("$index")) {
+ return obj + "[" + args.get(0) + "]";
+ }
+ if (obj != null) {
+ return obj + "." + func + "(" + args + ")";
+ }
+ return func + "(" + args + ")";
+ }
+
+ /**
+ * Returns the list of Skylark callable Methods of objClass with the given name
+ * and argument number.
+ */
+ public static List<MethodDescriptor> getMethods(Class<?> objClass, String methodName, int argNum)
+ throws ExecutionException {
+ return methodCache.get(objClass).get(methodName + "#" + argNum);
+ }
+
+ /**
+ * Returns the list of the Skylark name of all Skylark callable methods.
+ */
+ public static List<String> getMethodNames(Class<?> objClass)
+ throws ExecutionException {
+ List<String> names = new ArrayList<>();
+ for (List<MethodDescriptor> methods : methodCache.get(objClass).values()) {
+ for (MethodDescriptor method : methods) {
+ // TODO(bazel-team): store the Skylark name in the MethodDescriptor.
+ String name = method.annotation.name();
+ if (name.isEmpty()) {
+ name = StringUtilities.toPythonStyleFunctionName(method.method.getName());
+ }
+ names.add(name);
+ }
+ }
+ return names;
+ }
+
+ static Object callMethod(MethodDescriptor methodDescriptor, String methodName, Object obj,
+ Object[] args, Location loc) throws EvalException, IllegalAccessException,
+ IllegalArgumentException, InvocationTargetException {
+ Method method = methodDescriptor.getMethod();
+ if (obj == null && !Modifier.isStatic(method.getModifiers())) {
+ throw new EvalException(loc, "Method '" + methodName + "' is not static");
+ }
+ // This happens when the interface is public but the implementation classes
+ // have reduced visibility.
+ method.setAccessible(true);
+ Object result = method.invoke(obj, args);
+ if (method.getReturnType().equals(Void.TYPE)) {
+ return Environment.NONE;
+ }
+ if (result == null) {
+ if (methodDescriptor.getAnnotation().allowReturnNones()) {
+ return Environment.NONE;
+ } else {
+ throw new EvalException(loc,
+ "Method invocation returned None, please contact Skylark developers: " + methodName
+ + "(" + EvalUtils.prettyPrintValues(", ", ImmutableList.copyOf(args)) + ")");
+ }
+ }
+ result = SkylarkType.convertToSkylark(result, method);
+ if (result != null && !EvalUtils.isSkylarkImmutable(result.getClass())) {
+ throw new EvalException(loc, "Method '" + methodName
+ + "' returns a mutable object (type of " + EvalUtils.getDatatypeName(result) + ")");
+ }
+ return result;
+ }
+
+ // TODO(bazel-team): If there's exactly one usable method, this works. If there are multiple
+ // matching methods, it still can be a problem. Figure out how the Java compiler does it
+ // exactly and copy that behaviour.
+ // TODO(bazel-team): check if this and SkylarkBuiltInFunctions.createObject can be merged.
+ private Object invokeJavaMethod(
+ Object obj, Class<?> objClass, String methodName, List<Object> args) throws EvalException {
+ try {
+ MethodDescriptor matchingMethod = null;
+ List<MethodDescriptor> methods = getMethods(objClass, methodName, args.size());
+ if (methods != null) {
+ for (MethodDescriptor method : methods) {
+ Class<?>[] params = method.getMethod().getParameterTypes();
+ int i = 0;
+ boolean matching = true;
+ for (Class<?> param : params) {
+ if (!param.isAssignableFrom(args.get(i).getClass())) {
+ matching = false;
+ break;
+ }
+ i++;
+ }
+ if (matching) {
+ if (matchingMethod == null) {
+ matchingMethod = method;
+ } else {
+ throw new EvalException(func.getLocation(),
+ "Multiple matching methods for " + formatMethod(methodName, args)
+ + " in " + EvalUtils.getDataTypeNameFromClass(objClass));
+ }
+ }
+ }
+ }
+ if (matchingMethod != null && !matchingMethod.getAnnotation().structField()) {
+ return callMethod(matchingMethod, methodName, obj, args.toArray(), getLocation());
+ } else {
+ throw new EvalException(getLocation(), "No matching method found for "
+ + formatMethod(methodName, args) + " in "
+ + EvalUtils.getDataTypeNameFromClass(objClass));
+ }
+ } catch (IllegalAccessException e) {
+ // TODO(bazel-team): Print a nice error message. Maybe the method exists
+ // and an argument is missing or has the wrong type.
+ throw new EvalException(getLocation(), "Method invocation failed: " + e);
+ } catch (InvocationTargetException e) {
+ if (e.getCause() instanceof FuncallException) {
+ throw new EvalException(getLocation(), e.getCause().getMessage());
+ } else if (e.getCause() != null) {
+ throw new EvalExceptionWithJavaCause(getLocation(), e.getCause());
+ } else {
+ // This is unlikely to happen
+ throw new EvalException(getLocation(), "Method invocation failed: " + e);
+ }
+ } catch (ExecutionException e) {
+ throw new EvalException(getLocation(), "Method invocation failed: " + e);
+ }
+ }
+
+ private String formatMethod(String methodName, List<Object> args) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(methodName).append("(");
+ boolean first = true;
+ for (Object obj : args) {
+ if (!first) {
+ sb.append(", ");
+ }
+ sb.append(EvalUtils.getDatatypeName(obj));
+ first = false;
+ }
+ return sb.append(")").toString();
+ }
+
+ /**
+ * Add one argument to the keyword map, raising an exception when names conflict.
+ */
+ private void addKeywordArg(Map<String, Object> kwargs, String name, Object value)
+ throws EvalException {
+ if (kwargs.put(name, value) != null) {
+ throw new EvalException(getLocation(),
+ "duplicate keyword '" + name + "' in call to '" + func + "'");
+ }
+ }
+
+ /**
+ * Add multiple arguments to the keyword map (**kwargs).
+ */
+ private void addKeywordArgs(Map<String, Object> kwargs, Object items)
+ throws EvalException {
+ if (!(items instanceof Map<?, ?>)) {
+ throw new EvalException(getLocation(),
+ "Argument after ** must be a dictionary, not " + EvalUtils.getDatatypeName(items));
+ }
+ for (Map.Entry<?, ?> entry : ((Map<?, ?>) items).entrySet()) {
+ if (!(entry.getKey() instanceof String)) {
+ throw new EvalException(getLocation(),
+ "Keywords must be strings, not " + EvalUtils.getDatatypeName(entry.getKey()));
+ }
+ addKeywordArg(kwargs, (String) entry.getKey(), entry.getValue());
+ }
+ }
+
+ private void evalArguments(List<Object> posargs, Map<String, Object> kwargs,
+ Environment env, Function function)
+ throws EvalException, InterruptedException {
+ ArgConversion conversion = getArgConversion(function);
+ for (Argument arg : args) {
+ Object value = arg.getValue().eval(env);
+ if (conversion == ArgConversion.FROM_SKYLARK) {
+ value = SkylarkType.convertFromSkylark(value);
+ } else if (conversion == ArgConversion.TO_SKYLARK) {
+ // We try to auto convert the type if we can.
+ value = SkylarkType.convertToSkylark(value, getLocation());
+ // We call into Skylark so we need to be sure that the caller uses the appropriate types.
+ SkylarkType.checkTypeAllowedInSkylark(value, getLocation());
+ }
+ if (arg.isPositional()) {
+ posargs.add(value);
+ } else if (arg.isKwargs()) { // expand the kwargs
+ addKeywordArgs(kwargs, value);
+ } else {
+ addKeywordArg(kwargs, arg.getArgName(), value);
+ }
+ }
+ if (function instanceof UserDefinedFunction) {
+ // Adding the default values for a UserDefinedFunction if needed.
+ UserDefinedFunction func = (UserDefinedFunction) function;
+ if (args.size() < func.getArgs().size()) {
+ for (Map.Entry<String, Object> entry : func.getDefaultValues().entrySet()) {
+ String key = entry.getKey();
+ if (func.getArgIndex(key) >= numPositionalArgs && !kwargs.containsKey(key)) {
+ kwargs.put(key, entry.getValue());
+ }
+ }
+ }
+ }
+ }
+
+ static boolean isNamespace(Class<?> classObject) {
+ return classObject.isAnnotationPresent(SkylarkModule.class)
+ && classObject.getAnnotation(SkylarkModule.class).namespace();
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ List<Object> posargs = new ArrayList<>();
+ Map<String, Object> kwargs = new LinkedHashMap<>();
+
+ if (obj != null) {
+ Object objValue = obj.eval(env);
+ // Strings, lists and dictionaries (maps) have functions that we want to use in MethodLibrary.
+ // For other classes, we can call the Java methods.
+ Function function =
+ env.getFunction(EvalUtils.getSkylarkType(objValue.getClass()), func.getName());
+ if (function != null) {
+ if (!isNamespace(objValue.getClass())) {
+ posargs.add(objValue);
+ }
+ evalArguments(posargs, kwargs, env, function);
+ return EvalUtils.checkNotNull(this, function.call(posargs, kwargs, this, env));
+ } else if (env.isSkylarkEnabled()) {
+
+ // When calling a Java method, the name is not in the Environment, so
+ // evaluating 'func' would fail. For arguments we don't need to consider the default
+ // arguments since the Java function doesn't have any.
+
+ evalArguments(posargs, kwargs, env, null);
+ if (!kwargs.isEmpty()) {
+ throw new EvalException(func.getLocation(),
+ "Keyword arguments are not allowed when calling a java method");
+ }
+ if (objValue instanceof Class<?>) {
+ // Static Java method call
+ return invokeJavaMethod(null, (Class<?>) objValue, func.getName(), posargs);
+ } else {
+ return invokeJavaMethod(objValue, objValue.getClass(), func.getName(), posargs);
+ }
+ } else {
+ throw new EvalException(getLocation(), String.format(
+ "function '%s' is not defined on '%s'", func.getName(),
+ EvalUtils.getDatatypeName(objValue)));
+ }
+ }
+
+ Object funcValue = func.eval(env);
+ if (!(funcValue instanceof Function)) {
+ throw new EvalException(getLocation(),
+ "'" + EvalUtils.getDatatypeName(funcValue)
+ + "' object is not callable");
+ }
+ Function function = (Function) funcValue;
+ evalArguments(posargs, kwargs, env, function);
+ return EvalUtils.checkNotNull(this, function.call(posargs, kwargs, this, env));
+ }
+
+ private ArgConversion getArgConversion(Function function) {
+ if (function == null) {
+ // It means we try to call a Java function.
+ return ArgConversion.FROM_SKYLARK;
+ }
+ // If we call a UserDefinedFunction we call into Skylark. If we call from Skylark
+ // the argument conversion is invariant, but if we call from the BUILD language
+ // we might need an auto conversion.
+ return function instanceof UserDefinedFunction
+ ? ArgConversion.TO_SKYLARK : ArgConversion.NO_CONVERSION;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ // TODO(bazel-team): implement semantical check.
+
+ if (obj != null) {
+ // TODO(bazel-team): validate function calls on objects too.
+ return env.getReturnType(obj.validate(env), func.getName(), getLocation());
+ } else {
+ // TODO(bazel-team): Imported functions are not validated properly.
+ if (!env.hasSymbolInEnvironment(func.getName())) {
+ throw new EvalException(getLocation(),
+ String.format("function '%s' does not exist", func.getName()));
+ }
+ return env.getReturnType(func.getName(), getLocation());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Function.java b/src/main/java/com/google/devtools/build/lib/syntax/Function.java
new file mode 100644
index 0000000..5636a95
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Function.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Function values in the BUILD language.
+ *
+ * <p>Each implementation of this interface defines a function in the BUILD language.
+ *
+ */
+public interface Function {
+
+ /**
+ * Implements the behavior of a call to a function with positional arguments
+ * "args" and keyword arguments "kwargs". The "ast" argument is provided to
+ * allow construction of EvalExceptions containing source information.
+ */
+ Object call(List<Object> args,
+ Map<String, Object> kwargs,
+ FuncallExpression ast,
+ Environment env)
+ throws EvalException, InterruptedException;
+
+ /**
+ * Returns the name of the function.
+ */
+ String getName();
+
+ // TODO(bazel-team): implement this for MethodLibrary functions as well.
+ /**
+ * Returns the type of the object on which this function is defined or null
+ * if this is a global function.
+ */
+ Class<?> getObjectType();
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/FunctionDefStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/FunctionDefStatement.java
new file mode 100644
index 0000000..29ed579
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/FunctionDefStatement.java
@@ -0,0 +1,97 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType;
+
+import java.util.Collection;
+
+/**
+ * Syntax node for a function definition.
+ */
+public class FunctionDefStatement extends Statement {
+
+ private final Ident ident;
+ private final ImmutableList<Argument> args;
+ private final ImmutableList<Statement> statements;
+
+ public FunctionDefStatement(Ident ident, Collection<Argument> args,
+ Collection<Statement> statements) {
+ for (Argument arg : args) {
+ Preconditions.checkArgument(arg.isNamed());
+ }
+ this.ident = ident;
+ this.args = ImmutableList.copyOf(args);
+ this.statements = ImmutableList.copyOf(statements);
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ ImmutableMap.Builder<String, Object> defaultValues = ImmutableMap.builder();
+ for (Argument arg : args) {
+ if (arg.hasValue()) {
+ defaultValues.put(arg.getArgName(), arg.getValue().eval(env));
+ }
+ }
+ env.update(ident.getName(), new UserDefinedFunction(
+ ident, args, defaultValues.build(), statements, (SkylarkEnvironment) env));
+ }
+
+ @Override
+ public String toString() {
+ return "def " + ident + "(" + args + "):\n";
+ }
+
+ public Ident getIdent() {
+ return ident;
+ }
+
+ public ImmutableList<Statement> getStatements() {
+ return statements;
+ }
+
+ public ImmutableList<Argument> getArgs() {
+ return args;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ SkylarkFunctionType type = SkylarkFunctionType.of(ident.getName());
+ ValidationEnvironment localEnv = new ValidationEnvironment(env, type);
+ for (Argument i : args) {
+ SkylarkType argType = SkylarkType.UNKNOWN;
+ if (i.hasValue()) {
+ argType = i.getValue().validate(env);
+ if (argType.equals(SkylarkType.NONE)) {
+ argType = SkylarkType.UNKNOWN;
+ }
+ }
+ localEnv.update(i.getArgName(), argType, getLocation());
+ }
+ for (Statement stmts : statements) {
+ stmts.validate(localEnv);
+ }
+ env.updateFunction(ident.getName(), type, getLocation());
+ // Register a dummy return value with an incompatible type if there was no return statement.
+ type.setReturnType(SkylarkType.NONE, getLocation());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/GlobCriteria.java b/src/main/java/com/google/devtools/build/lib/syntax/GlobCriteria.java
new file mode 100644
index 0000000..577bd4a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/GlobCriteria.java
@@ -0,0 +1,214 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Functions;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.Iterables;
+
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/**
+ * Either the arguments to a glob call (the include and exclude lists) or the
+ * contents of a fixed list that was appended to a list of glob results.
+ * (The latter need to be stored by {@link GlobList} in order to fully
+ * reproduce the inputs that created the output list.)
+ *
+ * <p>For example, the expression
+ * <code>glob(['*.java']) + ['x.properties']</code>
+ * will result in two GlobCriteria: one has include = ['*.java'], glob = true
+ * and the other, include = ['x.properties'], glob = false.
+ */
+public class GlobCriteria {
+
+ /**
+ * A list of names or patterns that are included by this glob. They should
+ * consist of characters that are valid in labels in the BUILD language.
+ */
+ private final ImmutableList<String> include;
+
+ /**
+ * A list of names or patterns that are excluded by this glob. They should
+ * consist of characters that are valid in labels in the BUILD language.
+ */
+ private final ImmutableList<String> exclude;
+
+ /** True if the includes list was passed to glob(), false if not. */
+ private final boolean glob;
+
+ /**
+ * Parses criteria from its {@link #toExpression} form.
+ * Package-private for use by tests and GlobList.
+ * @throws IllegalArgumentException if the expression cannot be parsed
+ */
+ public static GlobCriteria parse(String text) {
+ if (text.startsWith("glob([") && text.endsWith("])")) {
+ int excludeIndex = text.indexOf("], exclude=[");
+ if (excludeIndex == -1) {
+ String listText = text.substring(6, text.length() - 2);
+ return new GlobCriteria(parseList(listText), ImmutableList.<String>of(), true);
+ } else {
+ String listText = text.substring(6, excludeIndex);
+ String excludeText = text.substring(excludeIndex + 12, text.length() - 2);
+ return new GlobCriteria(parseList(listText), parseList(excludeText), true);
+ }
+ } else if (text.startsWith("[") && text.endsWith("]")) {
+ String listText = text.substring(1, text.length() - 1);
+ return new GlobCriteria(parseList(listText), ImmutableList.<String>of(), false);
+ } else {
+ throw new IllegalArgumentException(
+ "unrecognized format (not from toExpression?): " + text);
+ }
+ }
+
+ /**
+ * Constructs a copy of a given glob critera object, with additional exclude patterns added.
+ *
+ * @param base a glob criteria object to copy. Must be an actual glob
+ * @param excludes a list of pattern strings indicating new excludes to provide
+ * @return a new glob criteria object which contains the same parameters as {@code base}, with
+ * the additional patterns in {@code excludes} added.
+ * @throws IllegalArgumentException if {@code base} is not a glob
+ */
+ public static GlobCriteria createWithAdditionalExcludes(GlobCriteria base,
+ List<String> excludes) {
+ Preconditions.checkArgument(base.isGlob());
+ return fromGlobCall(base.include,
+ ImmutableList.copyOf(Iterables.concat(base.exclude, excludes)));
+ }
+
+ /**
+ * Constructs a copy of a fixed list, converted to Strings.
+ */
+ public static GlobCriteria fromList(Iterable<?> list) {
+ Iterable<String> strings = Iterables.transform(list, Functions.toStringFunction());
+ return new GlobCriteria(ImmutableList.copyOf(strings), ImmutableList.<String>of(), false);
+ }
+
+ /**
+ * Constructs a glob call with include and exclude list.
+ *
+ * @param include list of included patterns
+ * @param exclude list of excluded patterns
+ */
+ public static GlobCriteria fromGlobCall(
+ ImmutableList<String> include, ImmutableList<String> exclude) {
+ return new GlobCriteria(include, exclude, true);
+ }
+
+ /**
+ * Constructs a glob call with include and exclude list.
+ */
+ private GlobCriteria(ImmutableList<String> include, ImmutableList<String> exclude, boolean glob) {
+ this.include = include;
+ this.exclude = exclude;
+ this.glob = glob;
+ }
+
+ /**
+ * Returns the patterns that were included in this {@code glob()} call.
+ */
+ public ImmutableList<String> getIncludePatterns() {
+ return include;
+ }
+
+ /**
+ * Returns the patterns that were excluded in this {@code glob()} call.
+ */
+ public ImmutableList<String> getExcludePatterns() {
+ return exclude;
+ }
+
+ /**
+ * Returns true if the include list was passed to {@code glob()}, false
+ * if it was a fixed list. If this returns false, the exclude list will
+ * always be empty.
+ */
+ public boolean isGlob() {
+ return glob;
+ }
+
+ /**
+ * Returns a String that represents this glob as a BUILD expression.
+ * For example, <code>glob(['abc', 'def'], exclude=['uvw', 'xyz'])</code>
+ * or <code>['foo', 'bar', 'baz']</code>.
+ */
+ public String toExpression() {
+ StringBuilder sb = new StringBuilder();
+ if (glob) {
+ sb.append("glob(");
+ }
+ sb.append('[');
+ appendList(sb, include);
+ if (!exclude.isEmpty()) {
+ sb.append("], exclude=[");
+ appendList(sb, exclude);
+ }
+ sb.append(']');
+ if (glob) {
+ sb.append(')');
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public String toString() {
+ return toExpression();
+ }
+
+ /**
+ * Takes a list of Strings, quotes them in single quotes, and appends them to
+ * a StringBuilder separated by a comma and space. This can be parsed back
+ * out by {@link #parseList}.
+ */
+ private static void appendList(StringBuilder sb, List<String> list) {
+ boolean first = true;
+ for (String content : list) {
+ if (!first) {
+ sb.append(", ");
+ }
+ sb.append('\'').append(content).append('\'');
+ first = false;
+ }
+ }
+
+ /**
+ * Takes a String in the format created by {@link #appendList} and returns
+ * the original Strings. A null String (which may be returned when Pattern
+ * does not find a match) or the String "" (which will be captured in "[]")
+ * will result in an empty list.
+ */
+ private static ImmutableList<String> parseList(@Nullable String text) {
+ if (text == null) {
+ return ImmutableList.of();
+ }
+ Iterable<String> split = Splitter.on(", ").split(text);
+ Builder<String> listBuilder = ImmutableList.builder();
+ for (String element : split) {
+ if (!element.isEmpty()) {
+ if ((element.length() < 2) || !element.startsWith("'") || !element.endsWith("'")) {
+ throw new IllegalArgumentException("expected a filename or pattern in quotes: " + text);
+ }
+ listBuilder.add(element.substring(1, element.length() - 1));
+ }
+ }
+ return listBuilder.build();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/GlobList.java b/src/main/java/com/google/devtools/build/lib/syntax/GlobList.java
new file mode 100644
index 0000000..82afd01
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/GlobList.java
@@ -0,0 +1,122 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ForwardingList;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.Iterables;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Glob matches and information about glob patterns, which are useful to
+ * ide_build_info. Its implementation of the List interface is as an immutable
+ * list of the matching files. Glob criteria can be retrieved through
+ * {@link #getCriteria}.
+ *
+ * @param <E> the element this List contains (generally either String or Label)
+ */
+public class GlobList<E> extends ForwardingList<E> {
+
+ /** Include/exclude criteria. */
+ private final ImmutableList<GlobCriteria> criteria;
+
+ /** Matching files (usually either String or Label). */
+ private final ImmutableList<E> matches;
+
+ /**
+ * Constructs a list with {@code glob()} call results.
+ *
+ * @param includes the patterns that the glob includes
+ * @param excludes the patterns that the glob excludes
+ * @param matches the filenames that matched the includes/excludes criteria
+ */
+ public static <T> GlobList<T> captureResults(List<String> includes,
+ List<String> excludes, List<T> matches) {
+ GlobCriteria criteria = GlobCriteria.fromGlobCall(
+ ImmutableList.copyOf(includes), ImmutableList.copyOf(excludes));
+ return new GlobList<>(ImmutableList.of(criteria), matches);
+ }
+
+ /**
+ * Parses a GlobInfo from its {@link #toExpression} representation.
+ */
+ public static GlobList<String> parse(String text) {
+ List<GlobCriteria> criteria = new ArrayList<>();
+ Iterable<String> globs = Splitter.on(" + ").split(text);
+ for (String glob : globs) {
+ criteria.add(GlobCriteria.parse(glob));
+ }
+ return new GlobList<>(criteria, ImmutableList.<String>of());
+ }
+
+ /**
+ * Concatenates two lists into a new GlobList. If either of the lists is a
+ * GlobList, its GlobCriteria are preserved. Otherwise a simple GlobCriteria
+ * is created to represent the fixed list.
+ */
+ public static <T> GlobList<T> concat(
+ List<? extends T> list1, List<? extends T> list2) {
+ // we add the list to both includes and matches, preserving order
+ Builder<GlobCriteria> criteriaBuilder = ImmutableList.<GlobCriteria>builder();
+ if (list1 instanceof GlobList<?>) {
+ criteriaBuilder.addAll(((GlobList<?>) list1).criteria);
+ } else {
+ criteriaBuilder.add(GlobCriteria.fromList(list1));
+ }
+ if (list2 instanceof GlobList<?>) {
+ criteriaBuilder.addAll(((GlobList<?>) list2).criteria);
+ } else {
+ criteriaBuilder.add(GlobCriteria.fromList(list2));
+ }
+ List<T> matches = ImmutableList.copyOf(Iterables.concat(list1, list2));
+ return new GlobList<>(criteriaBuilder.build(), matches);
+ }
+
+ /**
+ * Constructs a list with given criteria and matches.
+ */
+ public GlobList(List<GlobCriteria> criteria, List<E> matches) {
+ Preconditions.checkNotNull(criteria);
+ Preconditions.checkNotNull(matches);
+ this.criteria = ImmutableList.copyOf(criteria);
+ this.matches = ImmutableList.copyOf(matches);
+ }
+
+ /**
+ * Returns the criteria used to create this list, from which the
+ * includes/excludes can be retrieved.
+ */
+ public ImmutableList<GlobCriteria> getCriteria() {
+ return criteria;
+ }
+
+ /**
+ * Returns a String that represents this glob list as a BUILD expression.
+ */
+ public String toExpression() {
+ return Joiner.on(" + ").join(criteria);
+ }
+
+ @Override
+ protected ImmutableList<E> delegate() {
+ return matches;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Ident.java b/src/main/java/com/google/devtools/build/lib/syntax/Ident.java
new file mode 100644
index 0000000..86bd458
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Ident.java
@@ -0,0 +1,66 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Syntax node for an identifier.
+ */
+public final class Ident extends Expression {
+
+ private final String name;
+
+ public Ident(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Returns the name of the Ident.
+ */
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException {
+ try {
+ return env.lookup(name);
+ } catch (Environment.NoSuchVariableException e) {
+ if (name.equals("$error$")) {
+ throw new EvalException(getLocation(), "contains syntax error(s)", true);
+ } else {
+ throw new EvalException(getLocation(), "name '" + name + "' is not defined");
+ }
+ }
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ if (env.hasSymbolInEnvironment(name)) {
+ return env.getVartype(name);
+ } else {
+ throw new EvalException(getLocation(), "name '" + name + "' is not defined");
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/IfStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/IfStatement.java
new file mode 100644
index 0000000..3877a9c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/IfStatement.java
@@ -0,0 +1,138 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+// TODO(bazel-team): maybe we should get rid of the ConditionalStatements and
+// create a chain of if-else statements for elif-s.
+/**
+ * Syntax node for an if/else statement.
+ */
+public final class IfStatement extends Statement {
+
+ /**
+ * Syntax node for an [el]if statement.
+ */
+ static final class ConditionalStatements extends Statement {
+
+ private final Expression condition;
+ private final ImmutableList<Statement> stmts;
+
+ public ConditionalStatements(Expression condition, List<Statement> stmts) {
+ this.condition = Preconditions.checkNotNull(condition);
+ this.stmts = ImmutableList.copyOf(stmts);
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ for (Statement stmt : stmts) {
+ stmt.exec(env);
+ }
+ }
+
+ @Override
+ public String toString() {
+ // TODO(bazel-team): see TODO in the outer class
+ return "[el]if " + condition + ": ...\n";
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ Expression getCondition() {
+ return condition;
+ }
+
+ ImmutableList<Statement> getStmts() {
+ return stmts;
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ // EvalUtils.toBoolean() evaluates everything so we don't need type check here.
+ condition.validate(env);
+ validateStmts(env, stmts);
+ }
+ }
+
+ private final ImmutableList<ConditionalStatements> thenBlocks;
+ private final ImmutableList<Statement> elseBlock;
+
+ /**
+ * Constructs a if-elif-else statement. The else part is mandatory, but the list may be empty.
+ * ThenBlocks has to have at least one element.
+ */
+ IfStatement(List<ConditionalStatements> thenBlocks, List<Statement> elseBlock) {
+ Preconditions.checkArgument(thenBlocks.size() > 0);
+ this.thenBlocks = ImmutableList.copyOf(thenBlocks);
+ this.elseBlock = ImmutableList.copyOf(elseBlock);
+ }
+
+ public ImmutableList<ConditionalStatements> getThenBlocks() {
+ return thenBlocks;
+ }
+
+ public ImmutableList<Statement> getElseBlock() {
+ return elseBlock;
+ }
+
+ @Override
+ public String toString() {
+ // TODO(bazel-team): if we want to print the complete statement, the function
+ // needs an extra argument to specify indentation level.
+ return "if : ...\n";
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ for (ConditionalStatements stmt : thenBlocks) {
+ if (EvalUtils.toBoolean(stmt.getCondition().eval(env))) {
+ stmt.exec(env);
+ return;
+ }
+ }
+ for (Statement stmt : elseBlock) {
+ stmt.exec(env);
+ }
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ env.startTemporarilyDisableReadonlyCheckSession();
+ for (ConditionalStatements stmts : thenBlocks) {
+ stmts.validate(env);
+ }
+ validateStmts(env, elseBlock);
+ env.finishTemporarilyDisableReadonlyCheckSession();
+ }
+
+ private static void validateStmts(ValidationEnvironment env, List<Statement> stmts)
+ throws EvalException {
+ for (Statement stmt : stmts) {
+ stmt.validate(env);
+ }
+ env.finishTemporarilyDisableReadonlyCheckBranch();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/IntegerLiteral.java b/src/main/java/com/google/devtools/build/lib/syntax/IntegerLiteral.java
new file mode 100644
index 0000000..e6852e6b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/IntegerLiteral.java
@@ -0,0 +1,34 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Syntax node for an integer literal.
+ */
+public final class IntegerLiteral extends Literal<Integer> {
+
+ public IntegerLiteral(Integer value) {
+ super(value);
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ return SkylarkType.INT;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Label.java b/src/main/java/com/google/devtools/build/lib/syntax/Label.java
new file mode 100644
index 0000000..89db4de
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Label.java
@@ -0,0 +1,412 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ComparisonChain;
+import com.google.devtools.build.lib.cmdline.LabelValidator;
+import com.google.devtools.build.lib.cmdline.LabelValidator.BadLabelException;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.packages.PackageIdentifier;
+import com.google.devtools.build.lib.util.StringCanonicalizer;
+import com.google.devtools.build.lib.util.StringUtilities;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.InvalidObjectException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+
+/**
+ * A class to identify a BUILD target. All targets belong to exactly one package.
+ * The name of a target is called its label. A typical label looks like this:
+ * //dir1/dir2:target_name where 'dir1/dir2' identifies the package containing a BUILD file,
+ * and 'target_name' identifies the target within the package.
+ *
+ * <p>Parsing is robust against bad input, for example, from the command line.
+ */
+@SkylarkModule(name = "Label", doc = "A BUILD target identifier.")
+@Immutable @ThreadSafe
+public final class Label implements Comparable<Label>, Serializable {
+
+ /**
+ * Thrown by the parsing methods to indicate a bad label.
+ */
+ public static class SyntaxException extends Exception {
+ public SyntaxException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Factory for Labels from absolute string form, possibly including a repository name prefix. For
+ * example:
+ * <pre>
+ * //foo/bar
+ * {@literal @}foo//bar
+ * {@literal @}foo//bar:baz
+ * </pre>
+ */
+ public static Label parseRepositoryLabel(String absName) throws SyntaxException {
+ String repo = PackageIdentifier.DEFAULT_REPOSITORY;
+ int packageStartPos = absName.indexOf("//");
+ if (packageStartPos > 0) {
+ repo = absName.substring(0, packageStartPos);
+ absName = absName.substring(packageStartPos);
+ }
+ try {
+ LabelValidator.PackageAndTarget labelParts = LabelValidator.parseAbsoluteLabel(absName);
+ return new Label(new PackageIdentifier(repo, new PathFragment(labelParts.getPackageName())),
+ labelParts.getTargetName());
+ } catch (BadLabelException e) {
+ throw new SyntaxException(e.getMessage());
+ }
+ }
+
+ /**
+ * Factory for Labels from absolute string form. e.g.
+ * <pre>
+ * //foo/bar
+ * //foo/bar:quux
+ * </pre>
+ */
+ public static Label parseAbsolute(String absName) throws SyntaxException {
+ try {
+ LabelValidator.PackageAndTarget labelParts = LabelValidator.parseAbsoluteLabel(absName);
+ return create(labelParts.getPackageName(), labelParts.getTargetName());
+ } catch (BadLabelException e) {
+ throw new SyntaxException(e.getMessage());
+ }
+ }
+
+ /**
+ * Alternate factory method for Labels from absolute strings. This is a convenience method for
+ * cases when a Label needs to be initialized statically, so the declared exception is
+ * inconvenient.
+ *
+ * <p>Do not use this when the argument is not hard-wired.
+ */
+ public static Label parseAbsoluteUnchecked(String absName) {
+ try {
+ return parseAbsolute(absName);
+ } catch (SyntaxException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ /**
+ * Factory for Labels from separate components.
+ *
+ * @param packageName The name of the package. The package name does
+ * <b>not</b> include {@code //}. Must be valid according to
+ * {@link LabelValidator#validatePackageName}.
+ * @param targetName The name of the target within the package. Must be
+ * valid according to {@link LabelValidator#validateTargetName}.
+ * @throws SyntaxException if either of the arguments was invalid.
+ */
+ public static Label create(String packageName, String targetName) throws SyntaxException {
+ return new Label(packageName, targetName);
+ }
+
+ /**
+ * Similar factory to above, but takes a package identifier to allow external repository labels
+ * to be created.
+ */
+ public static Label create(PackageIdentifier packageId, String targetName)
+ throws SyntaxException {
+ return new Label(packageId, targetName);
+ }
+
+ /**
+ * Resolves a relative label using a workspace-relative path to the current working directory. The
+ * method handles these cases:
+ * <ul>
+ * <li>The label is absolute.
+ * <li>The label starts with a colon.
+ * <li>The label consists of a relative path, a colon, and a local part.
+ * <li>The label consists only of a local part.
+ * </ul>
+ *
+ * <p>Note that this method does not support any of the special syntactic constructs otherwise
+ * supported on the command line, like ":all", "/...", and so on.
+ *
+ * <p>It would be cleaner to use the TargetPatternEvaluator for this resolution, but that is not
+ * possible, because it is sometimes necessary to resolve a relative label before the package path
+ * is setup; in particular, before the tools/defaults package is created.
+ *
+ * @throws SyntaxException if the resulting label is not valid
+ */
+ public static Label parseCommandLineLabel(String label, PathFragment workspaceRelativePath)
+ throws SyntaxException {
+ Preconditions.checkArgument(!workspaceRelativePath.isAbsolute());
+ if (label.startsWith("//")) {
+ return parseAbsolute(label);
+ }
+ int index = label.indexOf(':');
+ if (index < 0) {
+ index = 0;
+ label = ":" + label;
+ }
+ PathFragment path = workspaceRelativePath.getRelative(label.substring(0, index));
+ // Use the String, String constructor, to make sure that the package name goes through the
+ // validity check.
+ return new Label(path.getPathString(), label.substring(index + 1));
+ }
+
+ /**
+ * Validates the given target name and returns a canonical String instance if it is valid.
+ * Otherwise it throws a SyntaxException.
+ */
+ private static String canonicalizeTargetName(String name) throws SyntaxException {
+ String error = LabelValidator.validateTargetName(name);
+ if (error != null) {
+ error = "invalid target name '" + StringUtilities.sanitizeControlChars(name) + "': " + error;
+ throw new SyntaxException(error);
+ }
+
+ // TODO(bazel-team): This should be an error, but we can't make it one for legacy reasons.
+ if (name.endsWith("/.")) {
+ name = name.substring(0, name.length() - 2);
+ }
+
+ return StringCanonicalizer.intern(name);
+ }
+
+ /**
+ * Validates the given package name and returns a canonical PathFragment instance if it is valid.
+ * Otherwise it throws a SyntaxException.
+ */
+ private static PathFragment validate(String packageName, String name) throws SyntaxException {
+ String error = LabelValidator.validatePackageName(packageName);
+ if (error != null) {
+ error = "invalid package name '" + packageName + "': " + error;
+ // This check is just for a more helpful error message
+ // i.e. valid target name, invalid package name, colon-free label form
+ // used => probably they meant "//foo:bar.c" not "//foo/bar.c".
+ if (packageName.endsWith("/" + name)) {
+ error += " (perhaps you meant \":" + name + "\"?)";
+ }
+ throw new SyntaxException(error);
+ }
+ return new PathFragment(packageName);
+ }
+
+ /** The name and repository of the package. */
+ private final PackageIdentifier packageIdentifier;
+
+ /** The name of the target within the package. Canonical. */
+ private final String name;
+
+ /**
+ * Constructor from a package name, target name. Both are checked for validity
+ * and a SyntaxException is thrown if either is invalid.
+ * TODO(bazel-team): move the validation to {@link PackageIdentifier}. Unfortunately, there are a
+ * bazillion tests that use invalid package names (taking advantage of the fact that calling
+ * Label(PathFragment, String) doesn't validate the package name).
+ */
+ private Label(String packageName, String name) throws SyntaxException {
+ this(validate(packageName, name), name);
+ }
+
+ /**
+ * Constructor from canonical valid package name and a target name. The target
+ * name is checked for validity and a SyntaxException is throw if it isn't.
+ */
+ private Label(PathFragment packageName, String name) throws SyntaxException {
+ this(PackageIdentifier.createInDefaultRepo(packageName), name);
+ }
+
+ private Label(PackageIdentifier packageIdentifier, String name)
+ throws SyntaxException {
+ Preconditions.checkNotNull(packageIdentifier);
+ Preconditions.checkNotNull(name);
+
+ try {
+ this.packageIdentifier = packageIdentifier;
+ this.name = canonicalizeTargetName(name);
+ } catch (SyntaxException e) {
+ // This check is just for a more helpful error message
+ // i.e. valid target name, invalid package name, colon-free label form
+ // used => probably they meant "//foo:bar.c" not "//foo/bar.c".
+ if (packageIdentifier.getPackageFragment().getPathString().endsWith("/" + name)) {
+ throw new SyntaxException(e.getMessage() + " (perhaps you meant \":" + name + "\"?)");
+ }
+ throw e;
+ }
+ }
+
+ private Object writeReplace() {
+ return new LabelSerializationProxy(toString());
+ }
+
+ private void readObject(ObjectInputStream stream) throws InvalidObjectException {
+ throw new InvalidObjectException("Serialization is allowed only by proxy");
+ }
+
+ public PackageIdentifier getPackageIdentifier() {
+ return packageIdentifier;
+ }
+
+ /**
+ * Returns the name of the package in which this rule was declared (e.g. {@code
+ * //file/base:fileutils_test} returns {@code file/base}).
+ */
+ @SkylarkCallable(name = "package", structField = true,
+ doc = "The package part of this label. "
+ + "For instance:<br>"
+ + "<pre class=language-python>label(\"//pkg/foo:abc\").package == \"pkg/foo\"</pre>")
+ public String getPackageName() {
+ return packageIdentifier.getPackageFragment().getPathString();
+ }
+
+ /**
+ * Returns the path fragment of the package in which this rule was declared (e.g. {@code
+ * //file/base:fileutils_test} returns {@code file/base}).
+ */
+ public PathFragment getPackageFragment() {
+ return packageIdentifier.getPackageFragment();
+ }
+
+ public static final com.google.common.base.Function<Label, PathFragment> PACKAGE_FRAGMENT =
+ new com.google.common.base.Function<Label, PathFragment>() {
+ @Override
+ public PathFragment apply(Label label) {
+ return label.getPackageFragment();
+ }
+ };
+
+ /**
+ * Returns the label as a path fragment, using the package and the label name.
+ */
+ public PathFragment toPathFragment() {
+ return packageIdentifier.getPackageFragment().getRelative(name);
+ }
+
+ /**
+ * Returns the name by which this rule was declared (e.g. {@code //foo/bar:baz}
+ * returns {@code baz}).
+ */
+ @SkylarkCallable(name = "name", structField = true,
+ doc = "The name of this label within the package. "
+ + "For instance:<br>"
+ + "<pre class=language-python>label(\"//pkg/foo:abc\").name == \"abc\"</pre>")
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Renders this label in canonical form.
+ *
+ * <p>invariant: {@code parseAbsolute(x.toString()).equals(x)}
+ */
+ @Override
+ public String toString() {
+ return packageIdentifier.getRepository() + "//" + packageIdentifier.getPackageFragment()
+ + ":" + name;
+ }
+
+ /**
+ * Renders this label in shorthand form.
+ *
+ * <p>Labels with canonical form {@code //foo/bar:bar} have the shorthand form {@code //foo/bar}.
+ * All other labels have identical shorthand and canonical forms.
+ */
+ public String toShorthandString() {
+ return packageIdentifier.getRepository() + (getPackageFragment().getBaseName().equals(name)
+ ? "//" + getPackageFragment()
+ : toString());
+ }
+
+ /**
+ * Returns a label in the same package as this label with the given target name.
+ *
+ * @throws SyntaxException if {@code targetName} is not a valid target name
+ */
+ public Label getLocalTargetLabel(String targetName) throws SyntaxException {
+ return new Label(packageIdentifier, targetName);
+ }
+
+ /**
+ * Resolves a relative or absolute label name. If given name is absolute, then this method calls
+ * {@link #parseAbsolute}. Otherwise, it calls {@link #getLocalTargetLabel}.
+ *
+ * <p>For example:
+ * {@code :quux} relative to {@code //foo/bar:baz} is {@code //foo/bar:quux};
+ * {@code //wiz:quux} relative to {@code //foo/bar:baz} is {@code //wiz:quux}.
+ *
+ * @param relName the relative label name; must be non-empty.
+ */
+ @SkylarkCallable(name = "relative", doc =
+ "Resolves a relative or absolute label name.<br>"
+ + "For example:<br><ul>"
+ + "<li><code>:quux</code> relative to <code>//foo/bar:baz</code> is "
+ + "<code>//foo/bar:quux</code></li>"
+ + "<li><code>//wiz:quux</code> relative to <code>//foo/bar:baz</code> is "
+ + "<code>//wiz:quux</code></li></ul>")
+ public Label getRelative(String relName) throws SyntaxException {
+ if (relName.length() == 0) {
+ throw new SyntaxException("empty package-relative label");
+ }
+ if (relName.startsWith("//")) {
+ return parseAbsolute(relName);
+ } else if (relName.equals(":")) {
+ throw new SyntaxException("':' is not a valid package-relative label");
+ } else if (relName.charAt(0) == ':') {
+ return getLocalTargetLabel(relName.substring(1));
+ } else {
+ return getLocalTargetLabel(relName);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode() ^ packageIdentifier.hashCode();
+ }
+
+ /**
+ * Two labels are equal iff both their name and their package name are equal.
+ */
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof Label)) {
+ return false;
+ }
+ Label otherLabel = (Label) other;
+ return name.equals(otherLabel.name) // least likely one first
+ && packageIdentifier.equals(otherLabel.packageIdentifier);
+ }
+
+ /**
+ * Defines the order between labels.
+ *
+ * <p>Labels are ordered primarily by package name and secondarily by target name. Both components
+ * are ordered lexicographically. Thus {@code //a:b/c} comes before {@code //a/b:a}, i.e. the
+ * position of the colon is significant to the order.
+ */
+ @Override
+ public int compareTo(Label other) {
+ return ComparisonChain.start()
+ .compare(packageIdentifier, other.packageIdentifier)
+ .compare(name, other.name)
+ .result();
+ }
+
+ /**
+ * Returns a suitable string for the user-friendly representation of the Label. Works even if the
+ * argument is null.
+ */
+ public static String print(Label label) {
+ return label == null ? "(unknown)" : label.toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/LabelSerializationProxy.java b/src/main/java/com/google/devtools/build/lib/syntax/LabelSerializationProxy.java
new file mode 100644
index 0000000..5b4556a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/LabelSerializationProxy.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectOutput;
+
+/**
+ * A serialization proxy for {@code Label}.
+ */
+public class LabelSerializationProxy implements Externalizable {
+
+ private String labelString;
+
+ public LabelSerializationProxy(String labelString) {
+ this.labelString = labelString;
+ }
+
+ // For deserialization machinery.
+ public LabelSerializationProxy() {
+ }
+
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+ // Manual serialization gives us about a 30% reduction in size.
+ out.writeUTF(labelString);
+ }
+
+ @Override
+ public void readExternal(java.io.ObjectInput in) throws IOException {
+ this.labelString = in.readUTF();
+ }
+
+ private Object readResolve() {
+ return Label.parseAbsoluteUnchecked(labelString);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
new file mode 100644
index 0000000..fc12c66
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
@@ -0,0 +1,803 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+
+/**
+ * A tokenizer for the BUILD language.
+ * <p>
+ * See: <a href="https://docs.python.org/2/reference/lexical_analysis.html"/>
+ * for some details.
+ * <p>
+ * Since BUILD files are small, we just tokenize the entire file a-priori
+ * instead of interleaving scanning with parsing.
+ */
+public final class Lexer {
+
+ private final EventHandler eventHandler;
+
+ // Input buffer and position
+ private char[] buffer;
+ private int pos;
+
+ /**
+ * The part of the location information that is common to all LexerLocation
+ * instances created by this Lexer. Factored into a separate object so that
+ * many Locations instances can share the same information as compactly as
+ * possible, without closing over a Lexer instance.
+ */
+ private static class LocationInfo {
+ final LineNumberTable lineNumberTable;
+ final Path filename;
+ LocationInfo(Path filename, LineNumberTable lineNumberTable) {
+ this.filename = filename;
+ this.lineNumberTable = lineNumberTable;
+ }
+ }
+
+ private final LocationInfo locationInfo;
+
+ // The stack of enclosing indentation levels; always contains '0' at the
+ // bottom.
+ private final Stack<Integer> indentStack = new Stack<>();
+
+ private final List<Token> tokens = new ArrayList<>();
+
+ // The number of unclosed open-parens ("(", '{', '[') at the current point in
+ // the stream. Whitespace is handled differently when this is nonzero.
+ private int openParenStackDepth = 0;
+
+ private boolean containsErrors;
+
+ private boolean parsePython;
+
+ /**
+ * Constructs a lexer which tokenizes the contents of the specified
+ * InputBuffer. Any errors during lexing are reported on "handler".
+ */
+ public Lexer(ParserInputSource input, EventHandler eventHandler, boolean parsePython) {
+ this.buffer = input.getContent();
+ this.pos = 0;
+ this.parsePython = parsePython;
+ this.eventHandler = eventHandler;
+ this.locationInfo = new LocationInfo(input.getPath(),
+ LineNumberTable.create(buffer, input.getPath()));
+
+ indentStack.push(0);
+ tokenize();
+ }
+
+ public Lexer(ParserInputSource input, EventHandler eventHandler) {
+ this(input, eventHandler, false);
+ }
+
+ /**
+ * Returns the filename from which the lexer's input came. Returns a dummy
+ * value if the input came from a string.
+ */
+ public Path getFilename() {
+ return locationInfo.filename;
+ }
+
+ /**
+ * Returns true if there were errors during scanning of this input file or
+ * string. The Lexer may attempt to recover from errors, but clients should
+ * not rely on the results of scanning if this flag is set.
+ */
+ public boolean containsErrors() {
+ return containsErrors;
+ }
+
+ /**
+ * Returns the (mutable) list of tokens generated by the Lexer.
+ */
+ public List<Token> getTokens() {
+ return tokens;
+ }
+
+ private void popParen() {
+ if (openParenStackDepth == 0) {
+ error("indentation error");
+ } else {
+ openParenStackDepth--;
+ }
+ }
+
+ private void error(String message) {
+ error(message, pos - 1, pos - 1);
+ }
+
+ private void error(String message, int start, int end) {
+ this.containsErrors = true;
+ eventHandler.handle(Event.error(createLocation(start, end), message));
+ }
+
+ Location createLocation(int start, int end) {
+ return new LexerLocation(locationInfo, start, end);
+ }
+
+ // Don't use an inner class as we don't want to close over the Lexer, only
+ // the LocationInfo.
+ @Immutable
+ private static final class LexerLocation extends Location {
+
+ private final LineNumberTable lineNumberTable;
+
+ LexerLocation(LocationInfo locationInfo, int start, int end) {
+ super(start, end);
+ this.lineNumberTable = locationInfo.lineNumberTable;
+ }
+
+ @Override
+ public PathFragment getPath() {
+ return lineNumberTable.getPath(getStartOffset()).asFragment();
+ }
+
+ @Override
+ public LineAndColumn getStartLineAndColumn() {
+ return lineNumberTable.getLineAndColumn(getStartOffset());
+ }
+
+ @Override
+ public LineAndColumn getEndLineAndColumn() {
+ return lineNumberTable.getLineAndColumn(getEndOffset());
+ }
+ }
+
+ /** invariant: symbol positions are half-open intervals. */
+ private void addToken(Token s) {
+ tokens.add(s);
+ }
+
+ /**
+ * Parses an end-of-line sequence, handling statement indentation correctly.
+ *
+ * UNIX newlines are assumed (LF). Carriage returns are always ignored.
+ *
+ * ON ENTRY: 'pos' is the index of the char after '\n'.
+ * ON EXIT: 'pos' is the index of the next non-space char after '\n'.
+ */
+ private void newline() {
+ if (openParenStackDepth > 0) {
+ newlineInsideExpression(); // in an expression: ignore space
+ } else {
+ newlineOutsideExpression(); // generate NEWLINE/INDENT/OUTDENT tokens
+ }
+ }
+
+ private void newlineInsideExpression() {
+ while (pos < buffer.length) {
+ switch (buffer[pos]) {
+ case ' ': case '\t': case '\r':
+ pos++;
+ break;
+ default:
+ return;
+ }
+ }
+ }
+
+ private void newlineOutsideExpression() {
+ if (pos > 1) { // skip over newline at start of file
+ addToken(new Token(TokenKind.NEWLINE, pos - 1, pos));
+ }
+
+ // we're in a stmt: suck up space at beginning of next line
+ int indentLen = 0;
+ while (pos < buffer.length) {
+ char c = buffer[pos];
+ if (c == ' ') {
+ indentLen++;
+ pos++;
+ } else if (c == '\t') {
+ indentLen += 8 - indentLen % 8;
+ pos++;
+ } else if (c == '\n') { // entirely blank line: discard
+ indentLen = 0;
+ pos++;
+ } else if (c == '#') { // line containing only indented comment
+ int oldPos = pos;
+ while (pos < buffer.length && c != '\n') {
+ c = buffer[pos++];
+ }
+ addToken(new Token(TokenKind.COMMENT, oldPos, pos - 1, bufferSlice(oldPos, pos - 1)));
+ indentLen = 0;
+ } else { // printing character
+ break;
+ }
+ }
+
+ if (pos == buffer.length) {
+ indentLen = 0;
+ } // trailing space on last line
+
+ int peekedIndent = indentStack.peek();
+ if (peekedIndent < indentLen) { // push a level
+ indentStack.push(indentLen);
+ addToken(new Token(TokenKind.INDENT, pos - 1, pos));
+
+ } else if (peekedIndent > indentLen) { // pop one or more levels
+ while (peekedIndent > indentLen) {
+ indentStack.pop();
+ addToken(new Token(TokenKind.OUTDENT, pos - 1, pos));
+ peekedIndent = indentStack.peek();
+ }
+
+ if (peekedIndent < indentLen) {
+ error("indentation error");
+ }
+ }
+ }
+
+ /**
+ * Returns true if current position is in the middle of a triple quote
+ * delimiter (3 x quot), and advances 'pos' by two if so.
+ */
+ private boolean skipTripleQuote(char quot) {
+ if (pos + 1 < buffer.length && buffer[pos] == quot && buffer[pos + 1] == quot) {
+ pos += 2;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Scans a string literal delimited by 'quot', containing escape sequences.
+ *
+ * ON ENTRY: 'pos' is 1 + the index of the first delimiter
+ * ON EXIT: 'pos' is 1 + the index of the last delimiter.
+ *
+ * @return the string-literal token.
+ */
+ private Token escapedStringLiteral(char quot) {
+ boolean inTriplequote = skipTripleQuote(quot);
+
+ int oldPos = pos - 1;
+ // more expensive second choice that expands escaped into a buffer
+ StringBuilder literal = new StringBuilder();
+ while (pos < buffer.length) {
+ char c = buffer[pos];
+ pos++;
+ switch (c) {
+ case '\n':
+ if (inTriplequote) {
+ literal.append(c);
+ break;
+ } else {
+ error("unterminated string literal at eol", oldPos, pos);
+ newline();
+ return new Token(TokenKind.STRING, oldPos, pos, literal.toString());
+ }
+ case '\\':
+ if (pos == buffer.length) {
+ error("unterminated string literal at eof", oldPos, pos);
+ return new Token(TokenKind.STRING, oldPos, pos, literal.toString());
+ }
+ c = buffer[pos];
+ pos++;
+ switch (c) {
+ case '\n':
+ // ignore end of line character
+ break;
+ case 'n':
+ literal.append('\n');
+ break;
+ case 'r':
+ literal.append('\r');
+ break;
+ case 't':
+ literal.append('\t');
+ break;
+ case '\\':
+ literal.append('\\');
+ break;
+ case '\'':
+ literal.append('\'');
+ break;
+ case '"':
+ literal.append('"');
+ break;
+ case '0': case '1': case '2': case '3':
+ case '4': case '5': case '6': case '7': { // octal escape
+ int octal = c - '0';
+ if (pos < buffer.length) {
+ c = buffer[pos];
+ if (c >= '0' && c <= '7') {
+ pos++;
+ octal = (octal << 3) | (c - '0');
+ if (pos < buffer.length) {
+ c = buffer[pos];
+ if (c >= '0' && c <= '7') {
+ pos++;
+ octal = (octal << 3) | (c - '0');
+ }
+ }
+ }
+ }
+ literal.append((char) (octal & 0xff));
+ break;
+ }
+ case 'a': case 'b': case 'f': case 'N': case 'u': case 'U': case 'v': case 'x':
+ // exists in Python but not implemented in Blaze => error
+ error("escape sequence not implemented: \\" + c, oldPos, pos);
+ break;
+ default:
+ // unknown char escape => "\literal"
+ literal.append('\\');
+ literal.append(c);
+ break;
+ }
+ break;
+ case '\'':
+ case '"':
+ if (c != quot
+ || (inTriplequote && !skipTripleQuote(quot))) {
+ // Non-matching quote, treat it like a regular char.
+ literal.append(c);
+ } else {
+ // Matching close-delimiter, all done.
+ return new Token(TokenKind.STRING, oldPos, pos, literal.toString());
+ }
+ break;
+ default:
+ literal.append(c);
+ break;
+ }
+ }
+ error("unterminated string literal at eof", oldPos, pos);
+ return new Token(TokenKind.STRING, oldPos, pos, literal.toString());
+ }
+
+ /**
+ * Scans a string literal delimited by 'quot'.
+ *
+ * <ul>
+ * <li> ON ENTRY: 'pos' is 1 + the index of the first delimiter
+ * <li> ON EXIT: 'pos' is 1 + the index of the last delimiter.
+ * </ul>
+ *
+ * @param isRaw if true, do not escape the string.
+ * @return the string-literal token.
+ */
+ private Token stringLiteral(char quot, boolean isRaw) {
+ int oldPos = pos - 1;
+
+ // Don't even attempt to parse triple-quotes here.
+ if (skipTripleQuote(quot)) {
+ pos -= 2;
+ return escapedStringLiteral(quot);
+ }
+
+ // first quick optimistic scan for a simple non-escaped string
+ while (pos < buffer.length) {
+ char c = buffer[pos++];
+ switch (c) {
+ case '\n':
+ error("unterminated string literal at eol", oldPos, pos);
+ Token t = new Token(TokenKind.STRING, oldPos, pos,
+ bufferSlice(oldPos + 1, pos - 1));
+ newline();
+ return t;
+ case '\\':
+ if (isRaw) {
+ // skip the next character
+ pos++;
+ break;
+ } else {
+ // oops, hit an escape, need to start over & build a new string buffer
+ pos = oldPos + 1;
+ return escapedStringLiteral(quot);
+ }
+ case '\'':
+ case '"':
+ if (c == quot) {
+ // close-quote, all done.
+ return new Token(TokenKind.STRING, oldPos, pos,
+ bufferSlice(oldPos + 1, pos - 1));
+ }
+ }
+ }
+
+ error("unterminated string literal at eof", oldPos, pos);
+ return new Token(TokenKind.STRING, oldPos, pos,
+ bufferSlice(oldPos + 1, pos));
+ }
+
+ private static final Map<String, TokenKind> keywordMap = new HashMap<>();
+
+ static {
+ keywordMap.put("and", TokenKind.AND);
+ keywordMap.put("as", TokenKind.AS);
+ keywordMap.put("class", TokenKind.CLASS); // reserved for future expansion
+ keywordMap.put("def", TokenKind.DEF);
+ keywordMap.put("elif", TokenKind.ELIF);
+ keywordMap.put("else", TokenKind.ELSE);
+ keywordMap.put("except", TokenKind.EXCEPT);
+ keywordMap.put("finally", TokenKind.FINALLY);
+ keywordMap.put("for", TokenKind.FOR);
+ keywordMap.put("from", TokenKind.FROM);
+ keywordMap.put("if", TokenKind.IF);
+ keywordMap.put("import", TokenKind.IMPORT);
+ keywordMap.put("in", TokenKind.IN);
+ keywordMap.put("not", TokenKind.NOT);
+ keywordMap.put("or", TokenKind.OR);
+ keywordMap.put("return", TokenKind.RETURN);
+ keywordMap.put("try", TokenKind.TRY);
+ }
+
+ private TokenKind getTokenKindForIdentfier(String id) {
+ TokenKind kind = keywordMap.get(id);
+ return kind == null ? TokenKind.IDENTIFIER : kind;
+ }
+
+ private String scanIdentifier() {
+ int oldPos = pos - 1;
+ while (pos < buffer.length) {
+ switch (buffer[pos]) {
+ case '_':
+ case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
+ case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
+ case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
+ case 's': case 't': case 'u': case 'v': case 'w': case 'x':
+ case 'y': case 'z':
+ case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
+ case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
+ case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
+ case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
+ case 'Y': case 'Z':
+ case '0': case '1': case '2': case '3': case '4': case '5':
+ case '6': case '7': case '8': case '9':
+ pos++;
+ break;
+ default:
+ return bufferSlice(oldPos, pos);
+ }
+ }
+ return bufferSlice(oldPos, pos);
+ }
+
+ /**
+ * Scans an identifier or keyword.
+ *
+ * ON ENTRY: 'pos' is 1 + the index of the first char in the identifier.
+ * ON EXIT: 'pos' is 1 + the index of the last char in the identifier.
+ *
+ * @return the identifier or keyword token.
+ */
+ private Token identifierOrKeyword() {
+ int oldPos = pos - 1;
+ String id = scanIdentifier();
+ TokenKind kind = getTokenKindForIdentfier(id);
+ return new Token(kind, oldPos, pos,
+ (kind == TokenKind.IDENTIFIER) ? id : null);
+ }
+
+ private String scanInteger() {
+ int oldPos = pos - 1;
+ while (pos < buffer.length) {
+ char c = buffer[pos];
+ switch (c) {
+ case 'X': case 'x':
+ case 'a': case 'A':
+ case 'b': case 'B':
+ case 'c': case 'C':
+ case 'd': case 'D':
+ case 'e': case 'E':
+ case 'f': case 'F':
+ case '0': case '1':
+ case '2': case '3':
+ case '4': case '5':
+ case '6': case '7':
+ case '8': case '9':
+ pos++;
+ break;
+ default:
+ return bufferSlice(oldPos, pos);
+ }
+ }
+ // TODO(bazel-team): (2009) to do roundtripping when we evaluate the integer
+ // constants, we must save the actual text of the tokens, not just their
+ // integer value.
+
+ return bufferSlice(oldPos, pos);
+ }
+
+ /**
+ * Scans an integer literal.
+ *
+ * ON ENTRY: 'pos' is 1 + the index of the first char in the literal.
+ * ON EXIT: 'pos' is 1 + the index of the last char in the literal.
+ *
+ * @return the integer token.
+ */
+ private Token integer() {
+ int oldPos = pos - 1;
+ String literal = scanInteger();
+
+ final String substring;
+ final int radix;
+ if (literal.startsWith("0x") || literal.startsWith("0X")) {
+ radix = 16;
+ substring = literal.substring(2);
+ } else if (literal.startsWith("0") && literal.length() > 1) {
+ radix = 8;
+ substring = literal.substring(1);
+ } else {
+ radix = 10;
+ substring = literal;
+ }
+
+ int value = 0;
+ try {
+ value = Integer.parseInt(substring, radix);
+ } catch (NumberFormatException e) {
+ error("invalid base-" + radix + " integer constant: " + literal);
+ }
+
+ return new Token(TokenKind.INT, oldPos, pos, value);
+ }
+
+ /**
+ * Tokenizes a two-char operator.
+ * @return true if it tokenized an operator
+ */
+ private boolean tokenizeTwoChars() {
+ if (pos + 2 >= buffer.length) {
+ return false;
+ }
+ char c1 = buffer[pos];
+ char c2 = buffer[pos + 1];
+ if (c2 == '=') {
+ switch (c1) {
+ case '=': {
+ addToken(new Token(TokenKind.EQUALS_EQUALS, pos, pos + 2));
+ return true;
+ }
+ case '!': {
+ addToken(new Token(TokenKind.NOT_EQUALS, pos, pos + 2));
+ return true;
+ }
+ case '>': {
+ addToken(new Token(TokenKind.GREATER_EQUALS, pos, pos + 2));
+ return true;
+ }
+ case '<': {
+ addToken(new Token(TokenKind.LESS_EQUALS, pos, pos + 2));
+ return true;
+ }
+ case '+': {
+ addToken(new Token(TokenKind.PLUS_EQUALS, pos, pos + 2));
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Performs tokenization of the character buffer of file contents provided to
+ * the constructor.
+ */
+ private void tokenize() {
+ while (pos < buffer.length) {
+ if (tokenizeTwoChars()) {
+ pos += 2;
+ continue;
+ }
+ char c = buffer[pos];
+ pos++;
+ switch (c) {
+ case '{': {
+ addToken(new Token(TokenKind.LBRACE, pos - 1, pos));
+ openParenStackDepth++;
+ break;
+ }
+ case '}': {
+ addToken(new Token(TokenKind.RBRACE, pos - 1, pos));
+ popParen();
+ break;
+ }
+ case '(': {
+ addToken(new Token(TokenKind.LPAREN, pos - 1, pos));
+ openParenStackDepth++;
+ break;
+ }
+ case ')': {
+ addToken(new Token(TokenKind.RPAREN, pos - 1, pos));
+ popParen();
+ break;
+ }
+ case '[': {
+ addToken(new Token(TokenKind.LBRACKET, pos - 1, pos));
+ openParenStackDepth++;
+ break;
+ }
+ case ']': {
+ addToken(new Token(TokenKind.RBRACKET, pos - 1, pos));
+ popParen();
+ break;
+ }
+ case '>': {
+ addToken(new Token(TokenKind.GREATER, pos - 1, pos));
+ break;
+ }
+ case '<': {
+ addToken(new Token(TokenKind.LESS, pos - 1, pos));
+ break;
+ }
+ case ':': {
+ addToken(new Token(TokenKind.COLON, pos - 1, pos));
+ break;
+ }
+ case ',': {
+ addToken(new Token(TokenKind.COMMA, pos - 1, pos));
+ break;
+ }
+ case '+': {
+ addToken(new Token(TokenKind.PLUS, pos - 1, pos));
+ break;
+ }
+ case '-': {
+ addToken(new Token(TokenKind.MINUS, pos - 1, pos));
+ break;
+ }
+ case '=': {
+ addToken(new Token(TokenKind.EQUALS, pos - 1, pos));
+ break;
+ }
+ case '%': {
+ addToken(new Token(TokenKind.PERCENT, pos - 1, pos));
+ break;
+ }
+ case ';': {
+ addToken(new Token(TokenKind.SEMI, pos - 1, pos));
+ break;
+ }
+ case '.': {
+ addToken(new Token(TokenKind.DOT, pos - 1, pos));
+ break;
+ }
+ case '*': {
+ addToken(new Token(TokenKind.STAR, pos - 1, pos));
+ break;
+ }
+ case ' ':
+ case '\t':
+ case '\r': {
+ /* ignore */
+ break;
+ }
+ case '\\': {
+ // Backslash character is valid only at the end of a line (or in a string)
+ if (pos + 1 < buffer.length && buffer[pos] == '\n') {
+ pos++; // skip the end of line character
+ } else {
+ addToken(new Token(TokenKind.ILLEGAL, pos - 1, pos, Character.toString(c)));
+ }
+ break;
+ }
+ case '\n': {
+ newline();
+ break;
+ }
+ case '#': {
+ int oldPos = pos - 1;
+ while (pos < buffer.length) {
+ c = buffer[pos];
+ if (c == '\n') {
+ break;
+ } else {
+ pos++;
+ }
+ }
+ addToken(new Token(TokenKind.COMMENT, oldPos, pos, bufferSlice(oldPos, pos)));
+ break;
+ }
+ case '\'':
+ case '\"': {
+ addToken(stringLiteral(c, false));
+ break;
+ }
+ default: {
+ // detect raw strings, e.g. r"str"
+ if (c == 'r' && pos < buffer.length
+ && (buffer[pos] == '\'' || buffer[pos] == '\"')) {
+ c = buffer[pos];
+ pos++;
+ addToken(stringLiteral(c, true));
+ break;
+ }
+
+ if (Character.isDigit(c)) {
+ addToken(integer());
+ } else if (Character.isJavaIdentifierStart(c) && c != '$') {
+ addToken(identifierOrKeyword());
+ } else {
+ // Some characters in Python are not recognized in Blaze syntax (e.g. '!')
+ if (parsePython) {
+ addToken(new Token(TokenKind.ILLEGAL, pos - 1, pos, Character.toString(c)));
+ } else {
+ error("invalid character: '" + c + "'");
+ }
+ }
+ break;
+ } // default
+ } // switch
+ } // while
+
+ if (indentStack.size() > 1) { // top of stack is always zero
+ addToken(new Token(TokenKind.NEWLINE, pos - 1, pos));
+ while (indentStack.size() > 1) {
+ indentStack.pop();
+ addToken(new Token(TokenKind.OUTDENT, pos - 1, pos));
+ }
+ }
+
+ // Like Python, always end with a NEWLINE token, even if no '\n' in input:
+ if (tokens.size() == 0
+ || tokens.get(tokens.size() - 1).kind != TokenKind.NEWLINE) {
+ addToken(new Token(TokenKind.NEWLINE, pos - 1, pos));
+ }
+
+ addToken(new Token(TokenKind.EOF, pos, pos));
+ }
+
+ /**
+ * Returns the character in the input buffer at the given position.
+ *
+ * @param at the position to get the character at.
+ * @return the character at the given position.
+ */
+ public char charAt(int at) {
+ return buffer[at];
+ }
+
+ /**
+ * Returns the string at the current line, minus the new line.
+ *
+ * @param line the line from which to retrieve the String, 1-based
+ * @return the text of the line
+ */
+ public String stringAtLine(int line) {
+ Pair<Integer, Integer> offsets = locationInfo.lineNumberTable.getOffsetsForLine(line);
+ return bufferSlice(offsets.first, offsets.second);
+ }
+
+ /**
+ * Returns parts of the source buffer based on offsets
+ *
+ * @param start the beginning offset for the slice
+ * @param end the offset immediately following the slice
+ * @return the text at offset start with length end - start
+ */
+ private String bufferSlice(int start, int end) {
+ return new String(this.buffer, start, end - start);
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/LineNumberTable.java b/src/main/java/com/google/devtools/build/lib/syntax/LineNumberTable.java
new file mode 100644
index 0000000..4842a16
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/LineNumberTable.java
@@ -0,0 +1,235 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Location.LineAndColumn;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.StringUtilities;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A table to keep track of line numbers in source files. The client creates a LineNumberTable for
+ * their buffer using {@link #create}. The client can then ask for the line and column given a
+ * position using ({@link #getLineAndColumn(int)}).
+ */
+abstract class LineNumberTable implements Serializable {
+
+ /**
+ * Returns the (line, column) pair for the specified offset.
+ */
+ abstract LineAndColumn getLineAndColumn(int offset);
+
+ /**
+ * Returns the (start, end) offset pair for a specified line, not including
+ * newline chars.
+ */
+ abstract Pair<Integer, Integer> getOffsetsForLine(int line);
+
+ /**
+ * Returns the path corresponding to the given offset.
+ */
+ abstract Path getPath(int offset);
+
+ static LineNumberTable create(char[] buffer, Path path) {
+ // If #line appears within a BUILD file, we assume it has been preprocessed
+ // by gconfig2blaze. We ignore all actual newlines and compute the logical
+ // LNT based only on the presence of #line markers.
+ return StringUtilities.containsSubarray(buffer, "\n#line ".toCharArray())
+ ? new HashLine(buffer, path)
+ : new Regular(buffer, path);
+ }
+
+ /**
+ * Line number table implementation for regular source files. Records
+ * offsets of newlines.
+ */
+ @Immutable
+ private static class Regular extends LineNumberTable {
+
+ /**
+ * A mapping from line number (line >= 1) to character offset into the file.
+ */
+ private final int[] linestart;
+ private final Path path;
+ private final int bufferLength;
+
+ private Regular(char[] buffer, Path path) {
+ // Compute the size.
+ int size = 2;
+ for (int i = 0; i < buffer.length; i++) {
+ if (buffer[i] == '\n') {
+ size++;
+ }
+ }
+ linestart = new int[size];
+
+ int index = 0;
+ linestart[index++] = 0; // The 0th line does not exist - so we fill something in
+ // to make sure the start pos for the 1st line ends up at
+ // linestart[1]. Using 0 is useful for tables that are
+ // completely empty.
+ linestart[index++] = 0; // The first line ("line 1") starts at offset 0.
+
+ // Scan the buffer and record the offset of each line start. Doing this
+ // once upfront is faster than checking each char as it is pulled from
+ // the buffer.
+ for (int i = 0; i < buffer.length; i++) {
+ if (buffer[i] == '\n') {
+ linestart[index++] = i + 1;
+ }
+ }
+ this.bufferLength = buffer.length;
+ this.path = path;
+ }
+
+ private int getLineAt(int pos) {
+ if (pos < 0) {
+ throw new IllegalArgumentException("Illegal position: " + pos);
+ }
+ int lowBoundary = 1, highBoundary = linestart.length - 1;
+ while (true) {
+ if ((highBoundary - lowBoundary) <= 1) {
+ if (linestart[highBoundary] > pos) {
+ return lowBoundary;
+ } else {
+ return highBoundary;
+ }
+ }
+ int medium = lowBoundary + ((highBoundary - lowBoundary) >> 1);
+ if (linestart[medium] > pos) {
+ highBoundary = medium;
+ } else {
+ lowBoundary = medium;
+ }
+ }
+ }
+
+ @Override
+ LineAndColumn getLineAndColumn(int offset) {
+ int line = getLineAt(offset);
+ int column = offset - linestart[line] + 1;
+ return new LineAndColumn(line, column);
+ }
+
+ @Override
+ Path getPath(int offset) {
+ return path;
+ }
+
+ @Override
+ Pair<Integer, Integer> getOffsetsForLine(int line) {
+ if (line <= 0 || line >= linestart.length) {
+ throw new IllegalArgumentException("Illegal line: " + line);
+ }
+ return Pair.of(linestart[line], line < linestart.length - 1
+ ? linestart[line + 1]
+ : bufferLength);
+ }
+ }
+
+ /**
+ * Line number table implementation for source files that have been
+ * preprocessed. Ignores newlines and uses only #line directives.
+ */
+ // TODO(bazel-team): Use binary search instead of linear search.
+ @Immutable
+ private static class HashLine extends LineNumberTable {
+
+ /**
+ * Represents a "#line" directive
+ */
+ private static class SingleHashLine implements Serializable {
+ final private int offset;
+ final private int line;
+ final private Path path;
+
+ SingleHashLine(int offset, int line, Path path) {
+ this.offset = offset;
+ this.line = line;
+ this.path = path;
+ }
+ }
+
+ private static final Pattern pattern = Pattern.compile("\n#line ([0-9]+) \"([^\"\\n]+)\"");
+
+ private final List<SingleHashLine> table;
+ private final Path defaultPath;
+ private final int bufferLength;
+
+ private HashLine(char[] buffer, Path defaultPath) {
+ // Not especially efficient, but that's fine: we just exec'd Python.
+ String bufString = new String(buffer);
+ Matcher m = pattern.matcher(bufString);
+ ImmutableList.Builder<SingleHashLine> tableBuilder = ImmutableList.builder();
+ while (m.find()) {
+ tableBuilder.add(new SingleHashLine(
+ m.start(0) + 1, //offset (+1 to skip \n in pattern)
+ Integer.valueOf(m.group(1)), // line number
+ defaultPath.getRelative(m.group(2)))); // filename is an absolute path
+ }
+ this.table = tableBuilder.build();
+ this.bufferLength = buffer.length;
+ this.defaultPath = defaultPath;
+ }
+
+ @Override
+ LineAndColumn getLineAndColumn(int offset) {
+ int line = -1;
+ for (int ii = 0, len = table.size(); ii < len; ii++) {
+ SingleHashLine hash = table.get(ii);
+ if (hash.offset > offset) {
+ break;
+ }
+ line = hash.line;
+ }
+ return new LineAndColumn(line, 1);
+ }
+
+ @Override
+ Path getPath(int offset) {
+ Path path = this.defaultPath;
+ for (int ii = 0, len = table.size(); ii < len; ii++) {
+ SingleHashLine hash = table.get(ii);
+ if (hash.offset > offset) {
+ break;
+ }
+ path = hash.path;
+ }
+ return path;
+ }
+
+ /**
+ * Returns 0, 0 for an unknown line
+ */
+ @Override
+ Pair<Integer, Integer> getOffsetsForLine(int line) {
+ for (int ii = 0, len = table.size(); ii < len; ii++) {
+ if (table.get(ii).line == line) {
+ return Pair.of(table.get(ii).offset, ii < len - 1
+ ? table.get(ii + 1).offset
+ : bufferLength);
+ }
+ }
+ return Pair.of(0, 0);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ListComprehension.java b/src/main/java/com/google/devtools/build/lib/syntax/ListComprehension.java
new file mode 100644
index 0000000..6a13ba8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ListComprehension.java
@@ -0,0 +1,133 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Syntax node for lists comprehension expressions.
+ */
+public final class ListComprehension extends Expression {
+
+ private final Expression elementExpression;
+ // This cannot be a map, because we need to both preserve order _and_ allow duplicate identifiers.
+ private final List<Map.Entry<Ident, Expression>> lists;
+
+ /**
+ * [elementExpr (for var in listExpr)+]
+ */
+ public ListComprehension(Expression elementExpression) {
+ this.elementExpression = elementExpression;
+ lists = new ArrayList<Map.Entry<Ident, Expression>>();
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ if (lists.size() == 0) {
+ return convert(new ArrayList<>(), env);
+ }
+
+ List<Map.Entry<Ident, Iterable<?>>> listValues = Lists.newArrayListWithCapacity(lists.size());
+ int size = 1;
+ for (Map.Entry<Ident, Expression> list : lists) {
+ Object listValueObject = list.getValue().eval(env);
+ final Iterable<?> listValue = EvalUtils.toIterable(listValueObject, getLocation());
+ int listSize = EvalUtils.size(listValue);
+ if (listSize == 0) {
+ return convert(new ArrayList<>(), env);
+ }
+ size *= listSize;
+ listValues.add(Maps.<Ident, Iterable<?>>immutableEntry(list.getKey(), listValue));
+ }
+ List<Object> resultList = Lists.newArrayListWithCapacity(size);
+ evalLists(env, listValues, resultList);
+ return convert(resultList, env);
+ }
+
+ private Object convert(List<Object> list, Environment env) throws EvalException {
+ if (env.isSkylarkEnabled()) {
+ return SkylarkList.list(list, getLocation());
+ } else {
+ return list;
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append('[').append(elementExpression);
+ for (Map.Entry<Ident, Expression> list : lists) {
+ sb.append(" for ").append(list.getKey()).append(" in ").append(list.getValue());
+ }
+ sb.append(']');
+ return sb.toString();
+ }
+
+ public Expression getElementExpression() {
+ return elementExpression;
+ }
+
+ public void add(Ident ident, Expression listExpression) {
+ lists.add(Maps.immutableEntry(ident, listExpression));
+ }
+
+ public List<Map.Entry<Ident, Expression>> getLists() {
+ return lists;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ /**
+ * Evaluates element expression over all combinations of list element values.
+ *
+ * <p>Iterates over all elements in outermost list (list at index 0) and
+ * updates the value of the list variable in the environment on each
+ * iteration. If there are no other lists to iterate over added evaluation
+ * of the element expression to the result. Otherwise calls itself recursively
+ * with all the lists except the outermost.
+ */
+ private void evalLists(Environment env, List<Map.Entry<Ident, Iterable<?>>> listValues,
+ List<Object> result) throws EvalException, InterruptedException {
+ Map.Entry<Ident, Iterable<?>> listValue = listValues.get(0);
+ for (Object listElement : listValue.getValue()) {
+ env.update(listValue.getKey().getName(), listElement);
+ if (listValues.size() == 1) {
+ result.add(elementExpression.eval(env));
+ } else {
+ evalLists(env, listValues.subList(1, listValues.size()), result);
+ }
+ }
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ for (Map.Entry<Ident, Expression> list : lists) {
+ // TODO(bazel-team): Get the type of elements
+ SkylarkType type = list.getValue().validate(env);
+ env.checkIterable(type, getLocation());
+ env.update(list.getKey().getName(), SkylarkType.UNKNOWN, getLocation());
+ }
+ elementExpression.validate(env);
+ return SkylarkType.of(SkylarkList.class);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ListLiteral.java b/src/main/java/com/google/devtools/build/lib/syntax/ListLiteral.java
new file mode 100644
index 0000000..9437135
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ListLiteral.java
@@ -0,0 +1,128 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Syntax node for list and tuple literals.
+ *
+ * (Note that during evaluation, both list and tuple values are represented by
+ * java.util.List objects, the only difference between them being whether or not
+ * they are mutable.)
+ */
+public final class ListLiteral extends Expression {
+
+ /**
+ * Types of the ListLiteral.
+ */
+ public static enum Kind {LIST, TUPLE}
+
+ private final Kind kind;
+
+ private final List<Expression> exprs;
+
+ private ListLiteral(Kind kind, List<Expression> exprs) {
+ this.kind = kind;
+ this.exprs = exprs;
+ }
+
+ public static ListLiteral makeList(List<Expression> exprs) {
+ return new ListLiteral(Kind.LIST, exprs);
+ }
+
+ public static ListLiteral makeTuple(List<Expression> exprs) {
+ return new ListLiteral(Kind.TUPLE, exprs);
+ }
+
+ /**
+ * Returns the list of expressions for each element of the tuple.
+ */
+ public List<Expression> getElements() {
+ return exprs;
+ }
+
+ /**
+ * Returns true if this list is a tuple (a hash table, immutable list).
+ */
+ public boolean isTuple() {
+ return kind == Kind.TUPLE;
+ }
+
+ private static char startChar(Kind kind) {
+ switch(kind) {
+ case LIST: return '[';
+ case TUPLE: return '(';
+ }
+ return '[';
+ }
+
+ private static char endChar(Kind kind) {
+ switch(kind) {
+ case LIST: return ']';
+ case TUPLE: return ')';
+ }
+ return ']';
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(startChar(kind));
+ String sep = "";
+ for (Expression e : exprs) {
+ sb.append(sep);
+ sb.append(e);
+ sep = ", ";
+ }
+ sb.append(endChar(kind));
+ return sb.toString();
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ List<Object> result = new ArrayList<>();
+ for (Expression expr : exprs) {
+ // Convert NPEs to EvalExceptions.
+ if (expr == null) {
+ throw new EvalException(getLocation(), "null expression in " + this);
+ }
+ result.add(expr.eval(env));
+ }
+ if (env.isSkylarkEnabled()) {
+ return isTuple()
+ ? SkylarkList.tuple(result) : SkylarkList.list(result, getLocation());
+ } else {
+ return EvalUtils.makeSequence(result, isTuple());
+ }
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ SkylarkType type = SkylarkType.UNKNOWN;
+ if (!isTuple()) {
+ for (Expression expr : exprs) {
+ SkylarkType nextType = expr.validate(env);
+ type = type.infer(nextType, "list literal", expr.getLocation(), getLocation());
+ }
+ }
+ return SkylarkType.of(SkylarkList.class, type.getType());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Literal.java b/src/main/java/com/google/devtools/build/lib/syntax/Literal.java
new file mode 100644
index 0000000..9289081
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Literal.java
@@ -0,0 +1,44 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Generic base class for primitive literals.
+ */
+public abstract class Literal<T> extends Expression {
+
+ protected final T value;
+
+ protected Literal(T value) {
+ this.value = value;
+ }
+
+ /**
+ * Returns the value of this literal.
+ */
+ public T getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ return value.toString();
+ }
+
+ @Override
+ Object eval(Environment env) {
+ return value;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/LoadStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/LoadStatement.java
new file mode 100644
index 0000000..6873995
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/LoadStatement.java
@@ -0,0 +1,78 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.List;
+
+/**
+ * Syntax node for an import statement.
+ */
+public final class LoadStatement extends Statement {
+
+ private final ImmutableList<Ident> symbols;
+ private final PathFragment importPath;
+
+ /**
+ * Constructs an import statement.
+ */
+ LoadStatement(String path, List<Ident> symbols) {
+ this.symbols = ImmutableList.copyOf(symbols);
+ this.importPath = new PathFragment(path + ".bzl");
+ }
+
+ public ImmutableList<Ident> getSymbols() {
+ return symbols;
+ }
+
+ public PathFragment getImportPath() {
+ return importPath;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("load(\"%s\", %s)", importPath, Joiner.on(", ").join(symbols));
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ for (Ident i : symbols) {
+ try {
+ if (i.getName().startsWith("_")) {
+ throw new EvalException(getLocation(), "symbol '" + i + "' is private and cannot "
+ + "be imported");
+ }
+ env.importSymbol(getImportPath(), i.getName());
+ } catch (Environment.NoSuchVariableException | Environment.LoadFailedException e) {
+ throw new EvalException(getLocation(), e.getMessage());
+ }
+ }
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ // TODO(bazel-team): implement semantical check.
+ for (Ident symbol : symbols) {
+ env.update(symbol.getName(), SkylarkType.UNKNOWN, getLocation());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/MixedModeFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/MixedModeFunction.java
new file mode 100644
index 0000000..0427157
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/MixedModeFunction.java
@@ -0,0 +1,187 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.Type.ConversionException;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Abstract implementation of Function for functions that accept a mixture of
+ * positional and keyword parameters, as in Python.
+ */
+public abstract class MixedModeFunction extends AbstractFunction {
+
+ // Nomenclature:
+ // "Parameters" are formal parameters of a function definition.
+ // "Arguments" are actual parameters supplied at the call site.
+
+ // Number of regular named parameters (excluding *p and **p) in the
+ // equivalent Python function definition).
+ private final List<String> parameters;
+
+ // Number of leading "parameters" which are mandatory
+ private final int numMandatoryParameters;
+
+ // True if this function requires all arguments to be named
+ // TODO(bazel-team): replace this by a count of arguments before the * with optional arg,
+ // in the style Python 3 or PEP 3102.
+ private final boolean onlyNamedArguments;
+
+ // Location of the function definition, or null for builtin functions.
+ protected final Location location;
+
+ /**
+ * Constructs an instance of Function that supports Python-style mixed-mode
+ * parameter passing.
+ *
+ * @param parameters a list of named parameters
+ * @param numMandatoryParameters the number of leading parameters which are
+ * considered mandatory; the remaining ones may be omitted, in which
+ * case they will have the default value of null.
+ */
+ public MixedModeFunction(String name,
+ Iterable<String> parameters,
+ int numMandatoryParameters,
+ boolean onlyNamedArguments) {
+ this(name, parameters, numMandatoryParameters, onlyNamedArguments, null);
+ }
+
+ protected MixedModeFunction(String name,
+ Iterable<String> parameters,
+ int numMandatoryParameters,
+ boolean onlyNamedArguments,
+ Location location) {
+ super(name);
+ this.parameters = ImmutableList.copyOf(parameters);
+ this.numMandatoryParameters = numMandatoryParameters;
+ this.onlyNamedArguments = onlyNamedArguments;
+ this.location = location;
+ }
+
+ @Override
+ public Object call(List<Object> args,
+ Map<String, Object> kwargs,
+ FuncallExpression ast,
+ Environment env)
+ throws EvalException, InterruptedException {
+
+ // ast is null when called from Java (as there's no Skylark call site).
+ Location loc = ast == null ? location : ast.getLocation();
+ if (onlyNamedArguments && args.size() > 0) {
+ throw new EvalException(loc,
+ getSignature() + " does not accept positional arguments");
+ }
+
+ if (kwargs == null) {
+ kwargs = ImmutableMap.<String, Object>of();
+ }
+
+ int numParams = parameters.size();
+ int numArgs = args.size();
+ Object[] namedArguments = new Object[numParams];
+
+ // first, positional arguments:
+ if (numArgs > numParams) {
+ throw new EvalException(loc,
+ "too many positional arguments in call to " + getSignature());
+ }
+ for (int ii = 0; ii < numArgs; ++ii) {
+ namedArguments[ii] = args.get(ii);
+ }
+
+ // TODO(bazel-team): here, support *varargs splicing
+
+ // second, keyword arguments:
+ for (Map.Entry<String, Object> entry : kwargs.entrySet()) {
+ String keyword = entry.getKey();
+ int pos = parameters.indexOf(keyword);
+ if (pos == -1) {
+ throw new EvalException(loc,
+ "unexpected keyword '" + keyword
+ + "' in call to " + getSignature());
+ } else {
+ if (namedArguments[pos] != null) {
+ throw new EvalException(loc, getSignature()
+ + " got multiple values for keyword argument '" + keyword + "'");
+ }
+ namedArguments[pos] = kwargs.get(keyword);
+ }
+ }
+
+ // third, defaults:
+ for (int ii = 0; ii < numMandatoryParameters; ++ii) {
+ if (namedArguments[ii] == null) {
+ throw new EvalException(loc,
+ getSignature() + " received insufficient arguments");
+ }
+ }
+ // (defaults are always null so nothing extra to do here.)
+
+ try {
+ return call(namedArguments, ast, env);
+ } catch (ConversionException | IllegalArgumentException | IllegalStateException
+ | ClassCastException e) {
+ throw new EvalException(loc, e.getMessage());
+ }
+ }
+
+ /**
+ * Like Function.call, but generalised to support Python-style mixed-mode
+ * keyword and positional parameter passing.
+ *
+ * @param args an array of argument values corresponding to the list
+ * of named parameters passed to the constructor.
+ */
+ protected Object call(Object[] args, FuncallExpression ast)
+ throws EvalException, ConversionException, InterruptedException {
+ throw new UnsupportedOperationException("Method not overridden");
+ }
+
+ /**
+ * Override this method instead of the one above, if you need to access
+ * the environment.
+ */
+ protected Object call(Object[] args, FuncallExpression ast, Environment env)
+ throws EvalException, ConversionException, InterruptedException {
+ return call(args, ast);
+ }
+
+ /**
+ * Render this object in the form of an equivalent Python function signature.
+ */
+ public String getSignature() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(getName()).append('(');
+ int ii = 0;
+ int len = parameters.size();
+ for (; ii < len; ++ii) {
+ String parameter = parameters.get(ii);
+ if (ii > 0) {
+ sb.append(", ");
+ }
+ sb.append(parameter);
+ if (ii >= numMandatoryParameters) {
+ sb.append(" = null");
+ }
+ }
+ sb.append(')');
+ return sb.toString();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/NotExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/NotExpression.java
new file mode 100644
index 0000000..5a13e79
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/NotExpression.java
@@ -0,0 +1,52 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * As syntax node for the not boolean operation.
+ */
+public class NotExpression extends Expression {
+
+ private final Expression expression;
+
+ public NotExpression(Expression expression) {
+ this.expression = expression;
+ }
+
+ Expression getExpression() {
+ return expression;
+ }
+
+ @Override
+ Object eval(Environment env) throws EvalException, InterruptedException {
+ return !EvalUtils.toBoolean(expression.eval(env));
+ }
+
+ @Override
+ public String toString() {
+ return "not " + expression;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ // Don't need type check here since EvalUtils.toBoolean() converts everything.
+ expression.validate(env);
+ return SkylarkType.BOOL;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Operator.java b/src/main/java/com/google/devtools/build/lib/syntax/Operator.java
new file mode 100644
index 0000000..628570e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Operator.java
@@ -0,0 +1,47 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Infix operators supported by the build language.
+ */
+public enum Operator {
+
+ AND("and"),
+ EQUALS_EQUALS("=="),
+ GREATER(">"),
+ GREATER_EQUALS(">="),
+ IN("in"),
+ LESS("<"),
+ LESS_EQUALS("<="),
+ MINUS("-"),
+ MULT("*"),
+ NOT("not"),
+ NOT_EQUALS("!="),
+ OR("or"),
+ PERCENT("%"),
+ PLUS("+");
+
+ private final String name;
+
+ private Operator(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Parser.java b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
new file mode 100644
index 0000000..66c3c67
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
@@ -0,0 +1,1274 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.CachingPackageLocator;
+import com.google.devtools.build.lib.syntax.DictionaryLiteral.DictionaryEntryLiteral;
+import com.google.devtools.build.lib.syntax.IfStatement.ConditionalStatements;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Recursive descent parser for LL(2) BUILD language.
+ * Loosely based on Python 2 grammar.
+ * See https://docs.python.org/2/reference/grammar.html
+ *
+ */
+class Parser {
+
+ /**
+ * Combines the parser result into a single value object.
+ */
+ public static final class ParseResult {
+ /** The statements (rules, basically) from the parsed file. */
+ public final List<Statement> statements;
+
+ /** The comments from the parsed file. */
+ public final List<Comment> comments;
+
+ /** Whether the file contained any errors. */
+ public final boolean containsErrors;
+
+ public ParseResult(List<Statement> statements, List<Comment> comments, boolean containsErrors) {
+ // No need to copy here; when the object is created, the parser instance is just about to go
+ // out of scope and be garbage collected.
+ this.statements = Preconditions.checkNotNull(statements);
+ this.comments = Preconditions.checkNotNull(comments);
+ this.containsErrors = containsErrors;
+ }
+ }
+
+ private static final EnumSet<TokenKind> STATEMENT_TERMINATOR_SET =
+ EnumSet.of(TokenKind.EOF, TokenKind.NEWLINE);
+
+ private static final EnumSet<TokenKind> LIST_TERMINATOR_SET =
+ EnumSet.of(TokenKind.EOF, TokenKind.RBRACKET, TokenKind.SEMI);
+
+ private static final EnumSet<TokenKind> DICT_TERMINATOR_SET =
+ EnumSet.of(TokenKind.EOF, TokenKind.RBRACE, TokenKind.SEMI);
+
+ private static final EnumSet<TokenKind> EXPR_TERMINATOR_SET = EnumSet.of(
+ TokenKind.EOF,
+ TokenKind.COMMA,
+ TokenKind.COLON,
+ TokenKind.FOR,
+ TokenKind.PLUS,
+ TokenKind.MINUS,
+ TokenKind.PERCENT,
+ TokenKind.RPAREN,
+ TokenKind.RBRACKET);
+
+ private Token token; // current lookahead token
+ private Token pushedToken = null; // used to implement LL(2)
+
+ private static final boolean DEBUGGING = false;
+
+ private final Lexer lexer;
+ private final EventHandler eventHandler;
+ private final List<Comment> comments;
+ private final boolean parsePython;
+ /** Whether advanced language constructs are allowed */
+ private boolean skylarkMode = false;
+
+ private static final Map<TokenKind, Operator> binaryOperators =
+ new ImmutableMap.Builder<TokenKind, Operator>()
+ .put(TokenKind.AND, Operator.AND)
+ .put(TokenKind.EQUALS_EQUALS, Operator.EQUALS_EQUALS)
+ .put(TokenKind.GREATER, Operator.GREATER)
+ .put(TokenKind.GREATER_EQUALS, Operator.GREATER_EQUALS)
+ .put(TokenKind.IN, Operator.IN)
+ .put(TokenKind.LESS, Operator.LESS)
+ .put(TokenKind.LESS_EQUALS, Operator.LESS_EQUALS)
+ .put(TokenKind.MINUS, Operator.MINUS)
+ .put(TokenKind.NOT_EQUALS, Operator.NOT_EQUALS)
+ .put(TokenKind.OR, Operator.OR)
+ .put(TokenKind.PERCENT, Operator.PERCENT)
+ .put(TokenKind.PLUS, Operator.PLUS)
+ .put(TokenKind.STAR, Operator.MULT)
+ .build();
+
+ private static final Map<TokenKind, Operator> augmentedAssignmentMethods =
+ new ImmutableMap.Builder<TokenKind, Operator>()
+ .put(TokenKind.PLUS_EQUALS, Operator.PLUS) // += // TODO(bazel-team): other similar operators
+ .build();
+
+ /** Highest precedence goes last.
+ * Based on: http://docs.python.org/2/reference/expressions.html#operator-precedence
+ **/
+ private static final List<EnumSet<Operator>> operatorPrecedence = ImmutableList.of(
+ EnumSet.of(Operator.OR),
+ EnumSet.of(Operator.AND),
+ EnumSet.of(Operator.NOT),
+ EnumSet.of(Operator.EQUALS_EQUALS, Operator.NOT_EQUALS, Operator.LESS, Operator.LESS_EQUALS,
+ Operator.GREATER, Operator.GREATER_EQUALS, Operator.IN),
+ EnumSet.of(Operator.MINUS, Operator.PLUS),
+ EnumSet.of(Operator.MULT, Operator.PERCENT));
+
+ private Iterator<Token> tokens = null;
+ private int errorsCount;
+ private boolean recoveryMode; // stop reporting errors until next statement
+
+ private CachingPackageLocator locator;
+
+ private List<Path> includedFiles;
+
+ private static final String PREPROCESSING_NEEDED =
+ "Add \"# PYTHON-PREPROCESSING-REQUIRED\" on the first line of the file";
+
+ private Parser(Lexer lexer, EventHandler eventHandler, CachingPackageLocator locator,
+ boolean parsePython) {
+ this.lexer = lexer;
+ this.eventHandler = eventHandler;
+ this.parsePython = parsePython;
+ this.tokens = lexer.getTokens().iterator();
+ this.comments = new ArrayList<Comment>();
+ this.locator = locator;
+ this.includedFiles = new ArrayList<Path>();
+ this.includedFiles.add(lexer.getFilename());
+ nextToken();
+ }
+
+ private Parser(Lexer lexer, EventHandler eventHandler, CachingPackageLocator locator) {
+ this(lexer, eventHandler, locator, false /* parsePython */);
+ }
+
+ public Parser setSkylarkMode(boolean skylarkMode) {
+ this.skylarkMode = skylarkMode;
+ return this;
+ }
+
+ /**
+ * Entry-point to parser that parses a build file with comments. All errors
+ * encountered during parsing are reported via "reporter".
+ */
+ public static ParseResult parseFile(
+ Lexer lexer, EventHandler eventHandler, CachingPackageLocator locator,
+ boolean parsePython) {
+ Parser parser = new Parser(lexer, eventHandler, locator, parsePython);
+ List<Statement> statements = parser.parseFileInput();
+ return new ParseResult(statements, parser.comments,
+ parser.errorsCount > 0 || lexer.containsErrors());
+ }
+
+ /**
+ * Entry-point to parser that parses a build file with comments. All errors
+ * encountered during parsing are reported via "reporter". Enable Skylark extensions
+ * that are not part of the core BUILD language.
+ */
+ public static ParseResult parseFileForSkylark(
+ Lexer lexer, EventHandler eventHandler, CachingPackageLocator locator,
+ ValidationEnvironment validationEnvironment) {
+ Parser parser = new Parser(lexer, eventHandler, locator).setSkylarkMode(true);
+ List<Statement> statements = parser.parseFileInput();
+ boolean hasSemanticalErrors = false;
+ try {
+ for (Statement statement : statements) {
+ statement.validate(validationEnvironment);
+ }
+ } catch (EvalException e) {
+ eventHandler.handle(Event.error(e.getLocation(), e.getMessage()));
+ hasSemanticalErrors = true;
+ }
+ return new ParseResult(statements, parser.comments,
+ parser.errorsCount > 0 || lexer.containsErrors() || hasSemanticalErrors);
+ }
+
+ /**
+ * Entry-point to parser that parses a statement. All errors encountered
+ * during parsing are reported via "reporter".
+ */
+ @VisibleForTesting
+ public static Statement parseStatement(
+ Lexer lexer, EventHandler eventHandler) {
+ return new Parser(lexer, eventHandler, null).parseSmallStatement();
+ }
+
+ /**
+ * Entry-point to parser that parses an expression. All errors encountered
+ * during parsing are reported via "reporter". The expression may be followed
+ * by newline tokens.
+ */
+ @VisibleForTesting
+ public static Expression parseExpression(Lexer lexer, EventHandler eventHandler) {
+ Parser parser = new Parser(lexer, eventHandler, null);
+ Expression result = parser.parseExpression();
+ while (parser.token.kind == TokenKind.NEWLINE) {
+ parser.nextToken();
+ }
+ parser.expect(TokenKind.EOF);
+ return result;
+ }
+
+ private void addIncludedFiles(List<Path> files) {
+ this.includedFiles.addAll(files);
+ }
+
+ private void reportError(Location location, String message) {
+ errorsCount++;
+ // Limit the number of reported errors to avoid spamming output.
+ if (errorsCount <= 5) {
+ eventHandler.handle(Event.error(location, message));
+ }
+ }
+
+ private void syntaxError(Token token) {
+ if (!recoveryMode) {
+ String msg = token.kind == TokenKind.INDENT
+ ? "indentation error"
+ : "syntax error at '" + token + "'";
+ reportError(lexer.createLocation(token.left, token.right), msg);
+ recoveryMode = true;
+ }
+ }
+
+ // Consumes the current token. If it is not of the specified (expected)
+ // kind, reports a syntax error.
+ private boolean expect(TokenKind kind) {
+ boolean expected = token.kind == kind;
+ if (!expected) {
+ syntaxError(token);
+ }
+ nextToken();
+ return expected;
+ }
+
+ /**
+ * Consume tokens past the first token that has a kind that is in the set of
+ * teminatingTokens.
+ * @param terminatingTokens
+ * @return the end offset of the terminating token.
+ */
+ private int syncPast(EnumSet<TokenKind> terminatingTokens) {
+ Preconditions.checkState(terminatingTokens.contains(TokenKind.EOF));
+ while (!terminatingTokens.contains(token.kind)) {
+ nextToken();
+ }
+ int end = token.right;
+ // read past the synchronization token
+ nextToken();
+ return end;
+ }
+
+ /**
+ * Consume tokens until we reach the first token that has a kind that is in
+ * the set of teminatingTokens.
+ * @param terminatingTokens
+ * @return the end offset of the terminating token.
+ */
+ private int syncTo(EnumSet<TokenKind> terminatingTokens) {
+ // EOF must be in the set to prevent an infinite loop
+ Preconditions.checkState(terminatingTokens.contains(TokenKind.EOF));
+ // read past the problematic token
+ int previous = token.right;
+ nextToken();
+ int current = previous;
+ while (!terminatingTokens.contains(token.kind)) {
+ nextToken();
+ previous = current;
+ current = token.right;
+ }
+ return previous;
+ }
+
+ private void nextToken() {
+ if (pushedToken != null) {
+ token = pushedToken;
+ pushedToken = null;
+ } else {
+ if (token == null || token.kind != TokenKind.EOF) {
+ token = tokens.next();
+ // transparently handle comment tokens
+ while (token.kind == TokenKind.COMMENT) {
+ makeComment(token);
+ token = tokens.next();
+ }
+ }
+ }
+ if (DEBUGGING) {
+ System.err.print(token);
+ }
+ }
+
+ private void pushToken(Token tokenToPush) {
+ if (pushedToken != null) {
+ throw new IllegalStateException("Exceeded LL(2) lookahead!");
+ }
+ pushedToken = token;
+ token = tokenToPush;
+ }
+
+ // create an error expression
+ private Ident makeErrorExpression(int start, int end) {
+ return setLocation(new Ident("$error$"), start, end);
+ }
+
+ // Convenience wrapper around ASTNode.setLocation that returns the node.
+ private <NODE extends ASTNode> NODE
+ setLocation(NODE node, int startOffset, int endOffset) {
+ node.setLocation(lexer.createLocation(startOffset, endOffset));
+ return node;
+ }
+
+ // Another convenience wrapper method around ASTNode.setLocation
+ private <NODE extends ASTNode> NODE setLocation(NODE node, Location location) {
+ node.setLocation(location);
+ return node;
+ }
+
+ // Convenience method that uses end offset from the last node.
+ private <NODE extends ASTNode> NODE setLocation(NODE node, int startOffset, ASTNode lastNode) {
+ return setLocation(node, startOffset, lastNode.getLocation().getEndOffset());
+ }
+
+ // create a funcall expression
+ private Expression makeFuncallExpression(Expression receiver, Ident function,
+ List<Argument> args,
+ int start, int end) {
+ if (function.getLocation() == null) {
+ function = setLocation(function, start, end);
+ }
+ boolean seenKeywordArg = false;
+ boolean seenKwargs = false;
+ for (Argument arg : args) {
+ if (arg.isPositional()) {
+ if (seenKeywordArg || seenKwargs) {
+ reportError(arg.getLocation(), "syntax error: non-keyword arg after keyword arg");
+ return makeErrorExpression(start, end);
+ }
+ } else if (arg.isKwargs()) {
+ if (seenKwargs) {
+ reportError(arg.getLocation(), "there can be only one **kwargs argument");
+ return makeErrorExpression(start, end);
+ }
+ seenKwargs = true;
+ } else {
+ seenKeywordArg = true;
+ }
+ }
+
+ return setLocation(new FuncallExpression(receiver, function, args), start, end);
+ }
+
+ // arg ::= IDENTIFIER '=' expr
+ // | expr
+ private Argument parseFunctionCallArgument() {
+ int start = token.left;
+ if (token.kind == TokenKind.IDENTIFIER) {
+ Token identToken = token;
+ String name = (String) token.value;
+ Ident ident = setLocation(new Ident(name), start, token.right);
+ nextToken();
+ if (token.kind == TokenKind.EQUALS) { // it's a named argument
+ nextToken();
+ Expression expr = parseExpression();
+ return setLocation(new Argument(ident, expr), start, expr);
+ } else { // oops, back up!
+ pushToken(identToken);
+ }
+ }
+ // parse **expr
+ if (token.kind == TokenKind.STAR) {
+ expect(TokenKind.STAR);
+ expect(TokenKind.STAR);
+ Expression expr = parseExpression();
+ return setLocation(new Argument(null, expr, true), start, expr);
+ }
+ // parse a positional argument
+ Expression expr = parseExpression();
+ return setLocation(new Argument(expr), start, expr);
+ }
+
+ // arg ::= IDENTIFIER '=' expr
+ // | IDENTIFIER
+ private Argument parseFunctionDefArgument(boolean onlyOptional) {
+ int start = token.left;
+ Ident ident = parseIdent();
+ if (token.kind == TokenKind.EQUALS) { // there's a default value
+ nextToken();
+ Expression expr = parseExpression();
+ return setLocation(new Argument(ident, expr), start, expr);
+ } else if (onlyOptional) {
+ reportError(ident.getLocation(),
+ "Optional arguments are only allowed at the end of the argument list.");
+ }
+ return setLocation(new Argument(ident), start, ident);
+ }
+
+ // funcall_suffix ::= '(' arg_list? ')'
+ private Expression parseFuncallSuffix(int start, Expression receiver,
+ Ident function) {
+ List<Argument> args = Collections.emptyList();
+ expect(TokenKind.LPAREN);
+ int end;
+ if (token.kind == TokenKind.RPAREN) {
+ end = token.right;
+ nextToken(); // RPAREN
+ } else {
+ args = parseFunctionCallArguments(); // (includes optional trailing comma)
+ end = token.right;
+ expect(TokenKind.RPAREN);
+ }
+ return makeFuncallExpression(receiver, function, args, start, end);
+ }
+
+ // selector_suffix ::= '.' IDENTIFIER
+ // |'.' IDENTIFIER funcall_suffix
+ private Expression parseSelectorSuffix(int start, Expression receiver) {
+ expect(TokenKind.DOT);
+ if (token.kind == TokenKind.IDENTIFIER) {
+ Ident ident = parseIdent();
+ if (token.kind == TokenKind.LPAREN) {
+ return parseFuncallSuffix(start, receiver, ident);
+ } else {
+ return setLocation(new DotExpression(receiver, ident), start, token.right);
+ }
+ } else {
+ syntaxError(token);
+ int end = syncTo(EXPR_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+ }
+
+ // arg_list ::= ( (arg ',')* arg ','? )?
+ private List<Argument> parseFunctionCallArguments() {
+ List<Argument> args = new ArrayList<>();
+ // terminating tokens for an arg list
+ while (token.kind != TokenKind.RPAREN) {
+ if (token.kind == TokenKind.EOF) {
+ syntaxError(token);
+ break;
+ }
+ args.add(parseFunctionCallArgument());
+ if (token.kind == TokenKind.COMMA) {
+ nextToken();
+ } else {
+ break;
+ }
+ }
+ return args;
+ }
+
+ // expr_list ::= ( (expr ',')* expr ','? )?
+ private List<Expression> parseExprList() {
+ List<Expression> list = new ArrayList<>();
+ // terminating tokens for an expression list
+ while (token.kind != TokenKind.RPAREN && token.kind != TokenKind.RBRACKET) {
+ list.add(parseExpression());
+ if (token.kind == TokenKind.COMMA) {
+ nextToken();
+ } else {
+ break;
+ }
+ }
+ return list;
+ }
+
+ // dict_entry_list ::= ( (dict_entry ',')* dict_entry ','? )?
+ private List<DictionaryEntryLiteral> parseDictEntryList() {
+ List<DictionaryEntryLiteral> list = new ArrayList<>();
+ // the terminating token for a dict entry list
+ while (token.kind != TokenKind.RBRACE) {
+ list.add(parseDictEntry());
+ if (token.kind == TokenKind.COMMA) {
+ nextToken();
+ } else {
+ break;
+ }
+ }
+ return list;
+ }
+
+ // dict_entry ::= expression ':' expression
+ private DictionaryEntryLiteral parseDictEntry() {
+ int start = token.left;
+ Expression key = parseExpression();
+ expect(TokenKind.COLON);
+ Expression value = parseExpression();
+ return setLocation(new DictionaryEntryLiteral(key, value), start, value);
+ }
+
+ private ExpressionStatement mocksubincludeExpression(
+ String labelName, String file, Location location) {
+ List<Argument> args = new ArrayList<>();
+ args.add(setLocation(new Argument(new StringLiteral(labelName, '"')), location));
+ args.add(setLocation(new Argument(new StringLiteral(file, '"')), location));
+ Ident mockIdent = setLocation(new Ident("mocksubinclude"), location);
+ Expression funCall = new FuncallExpression(null, mockIdent, args);
+ return setLocation(new ExpressionStatement(funCall), location);
+ }
+
+ // parse a file from an include call
+ private void include(String labelName, List<Statement> list, Location location) {
+ if (locator == null) {
+ return;
+ }
+
+ try {
+ Label label = Label.parseAbsolute(labelName);
+ String packageName = label.getPackageFragment().getPathString();
+ Path packagePath = locator.getBuildFileForPackage(packageName);
+ if (packagePath == null) {
+ reportError(location, "Package '" + packageName + "' not found");
+ list.add(mocksubincludeExpression(labelName, "", location));
+ return;
+ }
+ Path path = packagePath.getParentDirectory();
+ Path file = path.getRelative(label.getName());
+
+ if (this.includedFiles.contains(file)) {
+ reportError(location, "Recursive inclusion of file '" + path + "'");
+ return;
+ }
+ ParserInputSource inputSource = ParserInputSource.create(file);
+
+ // Insert call to the mocksubinclude function to get the dependencies right.
+ list.add(mocksubincludeExpression(labelName, file.toString(), location));
+
+ Lexer lexer = new Lexer(inputSource, eventHandler, parsePython);
+ Parser parser = new Parser(lexer, eventHandler, locator, parsePython);
+ parser.addIncludedFiles(this.includedFiles);
+ list.addAll(parser.parseFileInput());
+ } catch (Label.SyntaxException e) {
+ reportError(location, "Invalid label '" + labelName + "'");
+ } catch (IOException e) {
+ reportError(location, "Include of '" + labelName + "' failed: " + e.getMessage());
+ list.add(mocksubincludeExpression(labelName, "", location));
+ }
+ }
+
+ // primary ::= INTEGER
+ // | STRING
+ // | STRING '.' IDENTIFIER funcall_suffix
+ // | IDENTIFIER
+ // | IDENTIFIER funcall_suffix
+ // | IDENTIFIER '.' selector_suffix
+ // | list_expression
+ // | '(' ')' // a tuple with zero elements
+ // | '(' expr ')' // a parenthesized expression
+ // | '(' expr ',' expr_list ')' // a tuple with n elements
+ // | dict_expression
+ // | '-' primary_with_suffix
+ private Expression parsePrimary() {
+ int start = token.left;
+ switch (token.kind) {
+ case INT: {
+ IntegerLiteral literal = new IntegerLiteral((Integer) token.value);
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ case STRING: {
+ String value = (String) token.value;
+ int end = token.right;
+ char quoteChar = lexer.charAt(start);
+ nextToken();
+ if (token.kind == TokenKind.STRING) {
+ reportError(lexer.createLocation(end, token.left),
+ "Implicit string concatenation is forbidden, use the + operator");
+ }
+ StringLiteral literal = new StringLiteral(value, quoteChar);
+ setLocation(literal, start, end);
+ return literal;
+ }
+ case IDENTIFIER: {
+ Ident ident = parseIdent();
+ if (token.kind == TokenKind.LPAREN) { // it's a function application
+ return parseFuncallSuffix(start, null, ident);
+ } else {
+ return ident;
+ }
+ }
+ case LBRACKET: { // it's a list
+ return parseListExpression();
+ }
+ case LBRACE: { // it's a dictionary
+ return parseDictExpression();
+ }
+ case LPAREN: {
+ nextToken();
+ // check for the empty tuple literal
+ if (token.kind == TokenKind.RPAREN) {
+ ListLiteral literal =
+ ListLiteral.makeTuple(Collections.<Expression>emptyList());
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ // parse the first expression
+ Expression expression = parseExpression();
+ if (token.kind == TokenKind.COMMA) { // it's a tuple
+ nextToken();
+ // parse the rest of the expression tuple
+ List<Expression> tuple = parseExprList();
+ // add the first expression to the front of the tuple
+ tuple.add(0, expression);
+ expect(TokenKind.RPAREN);
+ return setLocation(
+ ListLiteral.makeTuple(tuple), start, token.right);
+ }
+ setLocation(expression, start, token.right);
+ if (token.kind == TokenKind.RPAREN) {
+ nextToken();
+ return expression;
+ }
+ syntaxError(token);
+ int end = syncTo(EXPR_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+ case MINUS: {
+ nextToken();
+
+ List<Argument> args = new ArrayList<>();
+ Expression expr = parsePrimaryWithSuffix();
+ args.add(setLocation(new Argument(expr), start, expr));
+ return makeFuncallExpression(null, new Ident("-"), args,
+ start, token.right);
+ }
+ default: {
+ syntaxError(token);
+ int end = syncTo(EXPR_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+ }
+ }
+
+ // primary_with_suffix ::= primary selector_suffix*
+ // | primary substring_suffix
+ private Expression parsePrimaryWithSuffix() {
+ int start = token.left;
+ Expression receiver = parsePrimary();
+ while (true) {
+ if (token.kind == TokenKind.DOT) {
+ receiver = parseSelectorSuffix(start, receiver);
+ } else if (token.kind == TokenKind.LBRACKET) {
+ receiver = parseSubstringSuffix(start, receiver);
+ } else {
+ break;
+ }
+ }
+ return receiver;
+ }
+
+ // substring_suffix ::= '[' expression? ':' expression? ']'
+ private Expression parseSubstringSuffix(int start, Expression receiver) {
+ List<Argument> args = new ArrayList<>();
+ Expression startExpr;
+ Expression endExpr;
+
+ expect(TokenKind.LBRACKET);
+ int loc1 = token.left;
+ if (token.kind == TokenKind.COLON) {
+ startExpr = setLocation(new IntegerLiteral(0), token.left, token.right);
+ } else {
+ startExpr = parseExpression();
+ }
+ args.add(setLocation(new Argument(startExpr), loc1, startExpr));
+ // This is a dictionary access
+ if (token.kind == TokenKind.RBRACKET) {
+ expect(TokenKind.RBRACKET);
+ return makeFuncallExpression(receiver, new Ident("$index"), args,
+ start, token.right);
+ }
+ // This is a substring
+ expect(TokenKind.COLON);
+ int loc2 = token.left;
+ if (token.kind == TokenKind.RBRACKET) {
+ endExpr = setLocation(new IntegerLiteral(Integer.MAX_VALUE), token.left, token.right);
+ } else {
+ endExpr = parseExpression();
+ }
+ expect(TokenKind.RBRACKET);
+
+ args.add(setLocation(new Argument(endExpr), loc2, endExpr));
+ return makeFuncallExpression(receiver, new Ident("$substring"), args,
+ start, token.right);
+ }
+
+ // loop_variables ::= '(' variables ')'
+ // | variables
+ // variables ::= ident (',' ident)*
+ private Ident parseForLoopVariables() {
+ int start = token.left;
+ boolean hasParen = false;
+ if (token.kind == TokenKind.LPAREN) {
+ hasParen = true;
+ nextToken();
+ }
+
+ // TODO(bazel-team): allow multiple variables in the core Blaze language too.
+ Ident firstIdent = parseIdent();
+ boolean multipleVariables = false;
+
+ while (token.kind == TokenKind.COMMA) {
+ multipleVariables = true;
+ nextToken();
+ parseIdent();
+ }
+
+ if (hasParen) {
+ expect(TokenKind.RPAREN);
+ }
+
+ int end = token.right;
+ if (multipleVariables && !parsePython) {
+ reportError(lexer.createLocation(start, end),
+ "For loops with multiple variables are not yet supported. "
+ + PREPROCESSING_NEEDED);
+ }
+ return multipleVariables ? makeErrorExpression(start, end) : firstIdent;
+ }
+
+ // list_expression ::= '[' ']'
+ // |'[' expr ']'
+ // |'[' expr ',' expr_list ']'
+ // |'[' expr ('FOR' loop_variables 'IN' expr)+ ']'
+ private Expression parseListExpression() {
+ int start = token.left;
+ expect(TokenKind.LBRACKET);
+ if (token.kind == TokenKind.RBRACKET) { // empty List
+ ListLiteral literal =
+ ListLiteral.makeList(Collections.<Expression>emptyList());
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ Expression expression = parseExpression();
+ Preconditions.checkNotNull(expression,
+ "null element in list in AST at %s:%s", token.left, token.right);
+ switch (token.kind) {
+ case RBRACKET: { // singleton List
+ ListLiteral literal =
+ ListLiteral.makeList(Collections.singletonList(expression));
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ case FOR: { // list comprehension
+ ListComprehension listComprehension =
+ new ListComprehension(expression);
+ do {
+ nextToken();
+ Ident ident = parseForLoopVariables();
+ if (token.kind == TokenKind.IN) {
+ nextToken();
+ Expression listExpression = parseExpression();
+ listComprehension.add(ident, listExpression);
+ } else {
+ break;
+ }
+ if (token.kind == TokenKind.RBRACKET) {
+ setLocation(listComprehension, start, token.right);
+ nextToken();
+ return listComprehension;
+ }
+ } while (token.kind == TokenKind.FOR);
+
+ syntaxError(token);
+ int end = syncPast(LIST_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+ case COMMA: {
+ nextToken();
+ List<Expression> list = parseExprList();
+ Preconditions.checkState(!list.contains(null),
+ "null element in list in AST at %s:%s", token.left, token.right);
+ list.add(0, expression);
+ if (token.kind == TokenKind.RBRACKET) {
+ ListLiteral literal = ListLiteral.makeList(list);
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ syntaxError(token);
+ int end = syncPast(LIST_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+ default: {
+ syntaxError(token);
+ int end = syncPast(LIST_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+ }
+ }
+
+ // dict_expression ::= '{' '}'
+ // |'{' dict_entry_list '}'
+ // |'{' dict_entry 'FOR' loop_variables 'IN' expr '}'
+ private Expression parseDictExpression() {
+ int start = token.left;
+ expect(TokenKind.LBRACE);
+ if (token.kind == TokenKind.RBRACE) { // empty List
+ DictionaryLiteral literal =
+ new DictionaryLiteral(ImmutableList.<DictionaryEntryLiteral>of());
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ DictionaryEntryLiteral entry = parseDictEntry();
+ if (token.kind == TokenKind.FOR) {
+ // Dict comprehension
+ nextToken();
+ Ident loopVar = parseForLoopVariables();
+ expect(TokenKind.IN);
+ Expression listExpression = parseExpression();
+ expect(TokenKind.RBRACE);
+ return setLocation(new DictComprehension(
+ entry.getKey(), entry.getValue(), loopVar, listExpression), start, token.right);
+ }
+ List<DictionaryEntryLiteral> entries = new ArrayList<>();
+ entries.add(entry);
+ if (token.kind == TokenKind.COMMA) {
+ expect(TokenKind.COMMA);
+ entries.addAll(parseDictEntryList());
+ }
+ if (token.kind == TokenKind.RBRACE) {
+ DictionaryLiteral literal = new DictionaryLiteral(entries);
+ setLocation(literal, start, token.right);
+ nextToken();
+ return literal;
+ }
+ syntaxError(token);
+ int end = syncPast(DICT_TERMINATOR_SET);
+ return makeErrorExpression(start, end);
+ }
+
+ private Ident parseIdent() {
+ if (token.kind != TokenKind.IDENTIFIER) {
+ syntaxError(token);
+ return makeErrorExpression(token.left, token.right);
+ }
+ Ident ident = new Ident(((String) token.value));
+ setLocation(ident, token.left, token.right);
+ nextToken();
+ return ident;
+ }
+
+ // binop_expression ::= binop_expression OP binop_expression
+ // | parsePrimaryWithSuffix
+ // This function takes care of precedence between operators (see operatorPrecedence for
+ // the order), and it assumes left-to-right associativity.
+ private Expression parseBinOpExpression(int prec) {
+ int start = token.left;
+ Expression expr = parseExpression(prec + 1);
+ // The loop is not strictly needed, but it prevents risks of stack overflow. Depth is
+ // limited to number of different precedence levels (operatorPrecedence.size()).
+ for (;;) {
+ if (!binaryOperators.containsKey(token.kind)) {
+ return expr;
+ }
+ Operator operator = binaryOperators.get(token.kind);
+ if (!operatorPrecedence.get(prec).contains(operator)) {
+ return expr;
+ }
+ nextToken();
+ Expression secondary = parseExpression(prec + 1);
+ expr = optimizeBinOpExpression(operator, expr, secondary);
+ setLocation(expr, start, secondary);
+ }
+ }
+
+ // Optimize binary expressions.
+ // string literal + string literal can be concatenated into one string literal
+ // so we don't have to do the expensive string concatenation at runtime.
+ private Expression optimizeBinOpExpression(
+ Operator operator, Expression expr, Expression secondary) {
+ if (operator == Operator.PLUS) {
+ if (expr instanceof StringLiteral && secondary instanceof StringLiteral) {
+ StringLiteral left = (StringLiteral) expr;
+ StringLiteral right = (StringLiteral) secondary;
+ if (left.getQuoteChar() == right.getQuoteChar()) {
+ return new StringLiteral(left.getValue() + right.getValue(), left.getQuoteChar());
+ }
+ }
+ }
+ return new BinaryOperatorExpression(operator, expr, secondary);
+ }
+
+ private Expression parseExpression() {
+ return parseExpression(0);
+ }
+
+ private Expression parseExpression(int prec) {
+ if (prec >= operatorPrecedence.size()) {
+ return parsePrimaryWithSuffix();
+ }
+ if (token.kind == TokenKind.NOT && operatorPrecedence.get(prec).contains(Operator.NOT)) {
+ return parseNotExpression(prec);
+ }
+ return parseBinOpExpression(prec);
+ }
+
+ // not_expr :== 'not' expr
+ private Expression parseNotExpression(int prec) {
+ int start = token.left;
+ expect(TokenKind.NOT);
+ Expression expression = parseExpression(prec + 1);
+ NotExpression notExpression = new NotExpression(expression);
+ return setLocation(notExpression, start, token.right);
+ }
+
+ // file_input ::= ('\n' | stmt)* EOF
+ private List<Statement> parseFileInput() {
+ List<Statement> list = new ArrayList<>();
+ while (token.kind != TokenKind.EOF) {
+ if (token.kind == TokenKind.NEWLINE) {
+ expect(TokenKind.NEWLINE);
+ } else {
+ parseTopLevelStatement(list);
+ }
+ }
+ return list;
+ }
+
+ // load(STRING (COMMA STRING)*)
+ private void parseLoad(List<Statement> list) {
+ int start = token.left;
+ if (token.kind != TokenKind.STRING) {
+ expect(TokenKind.STRING);
+ return;
+ }
+ String path = (String) token.value;
+ nextToken();
+ expect(TokenKind.COMMA);
+
+ List<Ident> symbols = new ArrayList<>();
+ if (token.kind == TokenKind.STRING) {
+ symbols.add(new Ident((String) token.value));
+ }
+ expect(TokenKind.STRING);
+ while (token.kind == TokenKind.COMMA) {
+ expect(TokenKind.COMMA);
+ if (token.kind == TokenKind.STRING) {
+ symbols.add(new Ident((String) token.value));
+ }
+ expect(TokenKind.STRING);
+ }
+ expect(TokenKind.RPAREN);
+ list.add(setLocation(new LoadStatement(path, symbols), start, token.left));
+ }
+
+ private void parseTopLevelStatement(List<Statement> list) {
+ // In Python grammar, there is no "top-level statement" and imports are
+ // considered as "small statements". We are a bit stricter than Python here.
+ int start = token.left;
+
+ // Check if there is an include
+ if (token.kind == TokenKind.IDENTIFIER) {
+ Token identToken = token;
+ Ident ident = parseIdent();
+
+ if (ident.getName().equals("include") && token.kind == TokenKind.LPAREN && !skylarkMode) {
+ expect(TokenKind.LPAREN);
+ if (token.kind == TokenKind.STRING) {
+ include((String) token.value, list, lexer.createLocation(start, token.right));
+ }
+ expect(TokenKind.STRING);
+ expect(TokenKind.RPAREN);
+ return;
+ } else if (ident.getName().equals("load") && token.kind == TokenKind.LPAREN) {
+ expect(TokenKind.LPAREN);
+ parseLoad(list);
+ return;
+ }
+ pushToken(identToken); // push the ident back to parse it as a statement
+ }
+ parseStatement(list, true);
+ }
+
+ // simple_stmt ::= small_stmt (';' small_stmt)* ';'? NEWLINE
+ private void parseSimpleStatement(List<Statement> list) {
+ list.add(parseSmallStatement());
+
+ while (token.kind == TokenKind.SEMI) {
+ nextToken();
+ if (token.kind == TokenKind.NEWLINE) {
+ break;
+ }
+ list.add(parseSmallStatement());
+ }
+ expect(TokenKind.NEWLINE);
+ // This is a safe place to recover: There is a new line at top-level
+ // and the parser is at the end of a statement.
+ recoveryMode = false;
+ }
+
+ // small_stmt ::= assign_stmt
+ // | expr
+ // | RETURN expr
+ // assign_stmt ::= expr ('=' | augassign) expr
+ // augassign ::= ('+=' )
+ // Note that these are in Python, but not implemented here (at least for now):
+ // '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |'<<=' | '>>=' | '**=' | '//='
+ // Semantic difference from Python:
+ // In Skylark, x += y is simple syntactic sugar for x = x + y.
+ // In Python, x += y is more or less equivalent to x = x + y, but if a method is defined
+ // on x.__iadd__(y), then it takes precedence, and in the case of lists it side-effects
+ // the original list (it doesn't do that on tuples); if no such method is defined it falls back
+ // to the x.__add__(y) method that backs x + y. In Skylark, we don't support this side-effect.
+ // Note also that there is a special casing to translate 'ident[key] = value'
+ // to 'ident = ident + {key: value}'. This is needed to support the pure version of Python-like
+ // dictionary assignment syntax.
+ private Statement parseSmallStatement() {
+ int start = token.left;
+ if (token.kind == TokenKind.RETURN) {
+ return parseReturnStatement();
+ }
+ Expression expression = parseExpression();
+ if (token.kind == TokenKind.EQUALS) {
+ nextToken();
+ Expression rvalue = parseExpression();
+ if (expression instanceof FuncallExpression) {
+ FuncallExpression func = (FuncallExpression) expression;
+ if (func.getFunction().getName().equals("$index") && func.getObject() instanceof Ident) {
+ // Special casing to translate 'ident[key] = value' to 'ident = ident + {key: value}'
+ // Note that the locations of these extra expressions are fake.
+ Preconditions.checkArgument(func.getArguments().size() == 1);
+ DictionaryLiteral dictRValue = setLocation(new DictionaryLiteral(ImmutableList.of(
+ setLocation(new DictionaryEntryLiteral(func.getArguments().get(0).getValue(), rvalue),
+ start, token.right))), start, token.right);
+ BinaryOperatorExpression binExp = setLocation(new BinaryOperatorExpression(
+ Operator.PLUS, func.getObject(), dictRValue), start, token.right);
+ return setLocation(new AssignmentStatement(func.getObject(), binExp), start, token.right);
+ }
+ }
+ return setLocation(new AssignmentStatement(expression, rvalue), start, rvalue);
+ } else if (augmentedAssignmentMethods.containsKey(token.kind)) {
+ Operator operator = augmentedAssignmentMethods.get(token.kind);
+ nextToken();
+ Expression operand = parseExpression();
+ int end = operand.getLocation().getEndOffset();
+ return setLocation(new AssignmentStatement(expression,
+ setLocation(new BinaryOperatorExpression(
+ operator, expression, operand), start, end)),
+ start, end);
+ } else {
+ return setLocation(new ExpressionStatement(expression), start, expression);
+ }
+ }
+
+ // if_stmt ::= IF expr ':' suite [ELIF expr ':' suite]* [ELSE ':' suite]?
+ private void parseIfStatement(List<Statement> list) {
+ int start = token.left;
+ List<ConditionalStatements> thenBlocks = new ArrayList<>();
+ thenBlocks.add(parseConditionalStatements(TokenKind.IF));
+ while (token.kind == TokenKind.ELIF) {
+ thenBlocks.add(parseConditionalStatements(TokenKind.ELIF));
+ }
+ List<Statement> elseBlock = new ArrayList<>();
+ if (token.kind == TokenKind.ELSE) {
+ expect(TokenKind.ELSE);
+ expect(TokenKind.COLON);
+ parseSuite(elseBlock);
+ }
+ Statement stmt = new IfStatement(thenBlocks, elseBlock);
+ list.add(setLocation(stmt, start, token.right));
+ }
+
+ // cond_stmts ::= [EL]IF expr ':' suite
+ private ConditionalStatements parseConditionalStatements(TokenKind tokenKind) {
+ int start = token.left;
+ expect(tokenKind);
+ Expression expr = parseExpression();
+ expect(TokenKind.COLON);
+ List<Statement> thenBlock = new ArrayList<>();
+ parseSuite(thenBlock);
+ ConditionalStatements stmt = new ConditionalStatements(expr, thenBlock);
+ return setLocation(stmt, start, token.right);
+ }
+
+ // for_stmt ::= FOR IDENTIFIER IN expr ':' suite
+ private void parseForStatement(List<Statement> list) {
+ int start = token.left;
+ expect(TokenKind.FOR);
+ Ident ident = parseIdent();
+ expect(TokenKind.IN);
+ Expression collection = parseExpression();
+ expect(TokenKind.COLON);
+ List<Statement> block = new ArrayList<>();
+ parseSuite(block);
+ Statement stmt = new ForStatement(ident, collection, block);
+ list.add(setLocation(stmt, start, token.right));
+ }
+
+ // def foo(bar1, bar2):
+ private void parseFunctionDefStatement(List<Statement> list) {
+ int start = token.left;
+ expect(TokenKind.DEF);
+ Ident ident = parseIdent();
+ expect(TokenKind.LPAREN);
+ // parsing the function arguments, at this point only identifiers
+ // TODO(bazel-team): support proper arguments with default values and kwargs
+ List<Argument> args = parseFunctionDefArguments();
+ expect(TokenKind.RPAREN);
+ expect(TokenKind.COLON);
+ List<Statement> block = new ArrayList<>();
+ parseSuite(block);
+ FunctionDefStatement stmt = new FunctionDefStatement(ident, args, block);
+ list.add(setLocation(stmt, start, token.right));
+ }
+
+ private List<Argument> parseFunctionDefArguments() {
+ List<Argument> args = new ArrayList<>();
+ Set<String> argNames = new HashSet<>();
+ boolean onlyOptional = false;
+ while (token.kind != TokenKind.RPAREN) {
+ Argument arg = parseFunctionDefArgument(onlyOptional);
+ if (arg.hasValue()) {
+ onlyOptional = true;
+ }
+ args.add(arg);
+ if (argNames.contains(arg.getArgName())) {
+ reportError(lexer.createLocation(token.left, token.right),
+ "duplicate argument name in function definition");
+ }
+ argNames.add(arg.getArgName());
+ if (token.kind == TokenKind.COMMA) {
+ nextToken();
+ } else {
+ break;
+ }
+ }
+ return args;
+ }
+
+ // suite ::= simple_stmt
+ // | NEWLINE INDENT stmt+ OUTDENT
+ private void parseSuite(List<Statement> list) {
+ if (token.kind == TokenKind.NEWLINE) {
+ expect(TokenKind.NEWLINE);
+ if (token.kind != TokenKind.INDENT) {
+ reportError(lexer.createLocation(token.left, token.right),
+ "expected an indented block");
+ return;
+ }
+ expect(TokenKind.INDENT);
+ while (token.kind != TokenKind.OUTDENT && token.kind != TokenKind.EOF) {
+ parseStatement(list, false);
+ }
+ expect(TokenKind.OUTDENT);
+ } else {
+ Statement stmt = parseSmallStatement();
+ list.add(stmt);
+ expect(TokenKind.NEWLINE);
+ }
+ }
+
+ // skipSuite does not check that the code is syntactically correct, it
+ // just skips based on indentation levels.
+ private void skipSuite() {
+ if (token.kind == TokenKind.NEWLINE) {
+ expect(TokenKind.NEWLINE);
+ if (token.kind != TokenKind.INDENT) {
+ reportError(lexer.createLocation(token.left, token.right),
+ "expected an indented block");
+ return;
+ }
+ expect(TokenKind.INDENT);
+
+ // Don't try to parse all the Python syntax, just skip the block
+ // until the corresponding outdent token.
+ int depth = 1;
+ while (depth > 0) {
+ // Because of the way the lexer works, this should never happen
+ Preconditions.checkState(token.kind != TokenKind.EOF);
+
+ if (token.kind == TokenKind.INDENT) {
+ depth++;
+ }
+ if (token.kind == TokenKind.OUTDENT) {
+ depth--;
+ }
+ nextToken();
+ }
+
+ } else {
+ // the block ends at the newline token
+ // e.g. if x == 3: print "three"
+ syncTo(STATEMENT_TERMINATOR_SET);
+ }
+ }
+
+ // stmt ::= simple_stmt
+ // | compound_stmt
+ private void parseStatement(List<Statement> list, boolean isTopLevel) {
+ if (token.kind == TokenKind.DEF && skylarkMode) {
+ if (!isTopLevel) {
+ reportError(lexer.createLocation(token.left, token.right),
+ "nested functions are not allowed. Move the function to top-level");
+ }
+ parseFunctionDefStatement(list);
+ } else if (token.kind == TokenKind.IF && skylarkMode) {
+ parseIfStatement(list);
+ } else if (token.kind == TokenKind.FOR && skylarkMode) {
+ if (isTopLevel) {
+ reportError(lexer.createLocation(token.left, token.right),
+ "for loops are not allowed on top-level. Put it into a function");
+ }
+ parseForStatement(list);
+ } else if (token.kind == TokenKind.IF
+ || token.kind == TokenKind.ELSE
+ || token.kind == TokenKind.FOR
+ || token.kind == TokenKind.CLASS
+ || token.kind == TokenKind.DEF
+ || token.kind == TokenKind.TRY) {
+ skipBlock();
+ } else {
+ parseSimpleStatement(list);
+ }
+ }
+
+ // return_stmt ::= RETURN expr
+ private ReturnStatement parseReturnStatement() {
+ int start = token.left;
+ expect(TokenKind.RETURN);
+ Expression expression = parseExpression();
+ return setLocation(new ReturnStatement(expression), start, expression);
+ }
+
+ // block ::= ('if' | 'for' | 'class') expr ':' suite
+ private void skipBlock() {
+ int start = token.left;
+ Token blockToken = token;
+ syncTo(EnumSet.of(TokenKind.COLON, TokenKind.EOF)); // skip over expression or name
+ if (!parsePython) {
+ reportError(lexer.createLocation(start, token.right), "syntax error at '"
+ + blockToken + "': This Python-style construct is not supported. "
+ + PREPROCESSING_NEEDED);
+ }
+ expect(TokenKind.COLON);
+ skipSuite();
+ }
+
+ // create a comment node
+ private void makeComment(Token token) {
+ comments.add(setLocation(new Comment((String) token.value), token.left, token.right));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ParserInputSource.java b/src/main/java/com/google/devtools/build/lib/syntax/ParserInputSource.java
new file mode 100644
index 0000000..488c762
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ParserInputSource.java
@@ -0,0 +1,112 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.hash.HashCode;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * An abstraction for reading input from a file or taking it as a pre-cooked
+ * char[] or String.
+ */
+public abstract class ParserInputSource {
+
+ protected ParserInputSource() {}
+
+ /**
+ * Returns the content of the input source.
+ */
+ public abstract char [] getContent();
+
+ /**
+ * Returns the path of the input source. Note: Once constructed, this object
+ * will never re-read the content from path.
+ */
+ public abstract Path getPath();
+
+ /**
+ * Create an input source instance by (eagerly) reading from the file at
+ * path. The file is assumed to be ISO-8859-1 encoded and smaller than
+ * 2 Gigs - these assumptions are reasonable for BUILD files, which is
+ * all we care about here.
+ */
+ public static ParserInputSource create(Path path) throws IOException {
+ char[] content = FileSystemUtils.readContentAsLatin1(path);
+ if (path.getFileSize() > content.length) {
+ // This assertion is to help diagnose problems arising from the
+ // filesystem; see bugs and #859334 and #920195.
+ throw new IOException("Unexpected short read from file '" + path
+ + "' (expected " + path.getFileSize() + ", got " + content.length + " bytes)");
+ }
+ return create(content, path);
+ }
+
+ /**
+ * Create an input source from the given content, and associate path with
+ * this source. Path will be used in error messages etc. but we will *never*
+ * attempt to read the content from path.
+ */
+ public static ParserInputSource create(String content, Path path) {
+ return create(content.toCharArray(), path);
+ }
+
+ /**
+ * Create an input source from the given content, and associate path with
+ * this source. Path will be used in error messages etc. but we will *never*
+ * attempt to read the content from path.
+ */
+ public static ParserInputSource create(final char[] content, final Path path) {
+ return new ParserInputSource() {
+
+ @Override
+ public char[] getContent() {
+ return content;
+ }
+
+ @Override
+ public Path getPath() {
+ return path;
+ }
+ };
+ }
+
+ /**
+ * Create an input source from the given input stream, and associate path
+ * with this source. 'path' will be used in error messages, etc, but will
+ * not (in general) be used to to read the content from path.
+ *
+ * (The exception is the case in which Python pre-processing is required; the
+ * path will be used to provide the input to the Python pre-processor.
+ * Arguably, we should just send the content as input to the subprocess
+ * instead of using the path, but it's not clear it's worth the effort.)
+ */
+ public static ParserInputSource create(InputStream in, Path path) throws IOException {
+ try {
+ return create(new String(FileSystemUtils.readContentAsLatin1(in)), path);
+ } finally {
+ in.close();
+ }
+ }
+
+ /**
+ * Returns a hash code calculated from the string content of this file.
+ */
+ public String contentHashCode() throws IOException {
+ return HashCode.fromBytes(getPath().getMD5Digest()).toString();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ReturnStatement.java b/src/main/java/com/google/devtools/build/lib/syntax/ReturnStatement.java
new file mode 100644
index 0000000..07032c2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ReturnStatement.java
@@ -0,0 +1,75 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType;
+
+/**
+ * A wrapper Statement class for return expressions.
+ */
+public class ReturnStatement extends Statement {
+
+ /**
+ * Exception sent by the return statement, to be caught by the function body.
+ */
+ public class ReturnException extends EvalException {
+ Object value;
+
+ public ReturnException(Location location, Object value) {
+ super(location, "Return statements must be inside a function");
+ this.value = value;
+ }
+
+ public Object getValue() {
+ return value;
+ }
+ }
+
+ private final Expression returnExpression;
+
+ public ReturnStatement(Expression returnExpression) {
+ this.returnExpression = returnExpression;
+ }
+
+ @Override
+ void exec(Environment env) throws EvalException, InterruptedException {
+ throw new ReturnException(returnExpression.getLocation(), returnExpression.eval(env));
+ }
+
+ Expression getReturnExpression() {
+ return returnExpression;
+ }
+
+ @Override
+ public String toString() {
+ return "return " + returnExpression;
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ void validate(ValidationEnvironment env) throws EvalException {
+ // TODO(bazel-team): save the return type in the environment, to type-check functions.
+ SkylarkFunctionType fct = env.getCurrentFunction();
+ if (fct == null) {
+ throw new EvalException(getLocation(), "Return statements must be inside a function");
+ }
+ SkylarkType resultType = returnExpression.validate(env);
+ fct.setReturnType(resultType, getLocation());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SelectorValue.java b/src/main/java/com/google/devtools/build/lib/syntax/SelectorValue.java
new file mode 100644
index 0000000..4fb3bdb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SelectorValue.java
@@ -0,0 +1,45 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.util.Map;
+
+/**
+ * The value passed to a select({...}) statement, e.g.:
+ *
+ * <pre>
+ * rule(
+ * name = 'myrule',
+ * deps = select({
+ * 'a': [':adep'],
+ * 'b': [':bdep'],
+ * })
+ * </pre>
+ */
+public final class SelectorValue {
+ Map<?, ?> dictionary;
+
+ public SelectorValue(Map<?, ?> dictionary) {
+ this.dictionary = dictionary;
+ }
+
+ public Map<?, ?> getDictionary() {
+ return dictionary;
+ }
+
+ @Override
+ public String toString() {
+ return "selector({...})";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkBuiltin.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkBuiltin.java
new file mode 100644
index 0000000..a2f0d1b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkBuiltin.java
@@ -0,0 +1,61 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+
+/**
+ * An annotation to mark built-in keyword argument methods accessible from Skylark.
+ */
+@Target({ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SkylarkBuiltin {
+
+ String name();
+
+ String doc();
+
+ Param[] mandatoryParams() default {};
+
+ Param[] optionalParams() default {};
+
+ boolean hidden() default false;
+
+ Class<?> objectType() default Object.class;
+
+ Class<?> returnType() default Object.class;
+
+ boolean onlyLoadingPhase() default false;
+
+ /**
+ * An annotation for parameters of Skylark built-in functions.
+ */
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Param {
+
+ String name();
+
+ String doc();
+
+ Class<?> type() default Object.class;
+
+ Class<?> generic1() default Object.class;
+
+ boolean callbackEnabled() default false;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallable.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallable.java
new file mode 100644
index 0000000..ae6987f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallable.java
@@ -0,0 +1,36 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * A marker interface for Java methods which can be called from Skylark.
+ */
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SkylarkCallable {
+ String name() default "";
+
+ String doc();
+
+ boolean hidden() default false;
+
+ boolean structField() default false;
+
+ boolean allowReturnNones() default false;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallbackFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallbackFunction.java
new file mode 100644
index 0000000..2e94be8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkCallbackFunction.java
@@ -0,0 +1,44 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.collect.ImmutableList;
+
+
+/**
+ * A helper class for calling Skylark functions from Java.
+ */
+public class SkylarkCallbackFunction {
+
+ private final UserDefinedFunction callback;
+ private final FuncallExpression ast;
+ private final SkylarkEnvironment funcallEnv;
+
+ public SkylarkCallbackFunction(UserDefinedFunction callback, FuncallExpression ast,
+ SkylarkEnvironment funcallEnv) {
+ this.callback = callback;
+ this.ast = ast;
+ this.funcallEnv = funcallEnv;
+ }
+
+ public Object call(ClassObject ctx, Object... arguments) throws EvalException {
+ try {
+ return callback.call(
+ ImmutableList.<Object>builder().add(ctx).add(arguments).build(), null, ast, funcallEnv);
+ } catch (InterruptedException | ClassCastException
+ | IllegalArgumentException e) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkEnvironment.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkEnvironment.java
new file mode 100644
index 0000000..7e6f414
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkEnvironment.java
@@ -0,0 +1,253 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.Fingerprint;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * The environment for Skylark.
+ */
+public class SkylarkEnvironment extends Environment {
+
+ /**
+ * This set contains the variable names of all the successful lookups from the global
+ * environment. This is necessary because if in a function definition something
+ * reads a global variable after which a local variable with the same name is assigned an
+ * Exception needs to be thrown.
+ */
+ private final Set<String> readGlobalVariables = new HashSet<>();
+
+ private ImmutableList<String> stackTrace;
+
+ @Nullable private String fileContentHashCode;
+
+ /**
+ * Creates a Skylark Environment for function calling, from the global Environment of the
+ * caller Environment (which must be a Skylark Environment).
+ */
+ public static SkylarkEnvironment createEnvironmentForFunctionCalling(
+ Environment callerEnv, SkylarkEnvironment definitionEnv,
+ UserDefinedFunction function) throws EvalException {
+ if (callerEnv.getStackTrace().contains(function.getName())) {
+ throw new EvalException(function.getLocation(), "Recursion was detected when calling '"
+ + function.getName() + "' from '" + Iterables.getLast(callerEnv.getStackTrace()) + "'");
+ }
+ ImmutableList<String> stackTrace = new ImmutableList.Builder<String>()
+ .addAll(callerEnv.getStackTrace())
+ .add(function.getName())
+ .build();
+ SkylarkEnvironment childEnv =
+ // Always use the caller Environment's EventHandler. We cannot assume that the
+ // definition Environment's EventHandler is still working properly.
+ new SkylarkEnvironment(definitionEnv, stackTrace, callerEnv.eventHandler);
+ try {
+ for (String varname : callerEnv.propagatingVariables) {
+ childEnv.updateAndPropagate(varname, callerEnv.lookup(varname));
+ }
+ } catch (NoSuchVariableException e) {
+ // This should never happen.
+ throw new IllegalStateException(e);
+ }
+ childEnv.disabledVariables = callerEnv.disabledVariables;
+ childEnv.disabledNameSpaces = callerEnv.disabledNameSpaces;
+ return childEnv;
+ }
+
+ private SkylarkEnvironment(SkylarkEnvironment definitionEnv, ImmutableList<String> stackTrace,
+ EventHandler eventHandler) {
+ super(definitionEnv.getGlobalEnvironment());
+ this.stackTrace = stackTrace;
+ this.eventHandler = Preconditions.checkNotNull(eventHandler,
+ "EventHandler cannot be null in an Environment which calls into Skylark");
+ }
+
+ /**
+ * Creates a global SkylarkEnvironment.
+ */
+ public SkylarkEnvironment(EventHandler eventHandler, String astFileContentHashCode) {
+ super();
+ stackTrace = ImmutableList.of();
+ this.eventHandler = eventHandler;
+ this.fileContentHashCode = astFileContentHashCode;
+ }
+
+ @VisibleForTesting
+ public SkylarkEnvironment(EventHandler eventHandler) {
+ this(eventHandler, null);
+ }
+
+ public SkylarkEnvironment(SkylarkEnvironment globalEnv) {
+ super(globalEnv);
+ stackTrace = ImmutableList.of();
+ this.eventHandler = globalEnv.eventHandler;
+ }
+
+ @Override
+ public ImmutableList<String> getStackTrace() {
+ return stackTrace;
+ }
+
+ /**
+ * Clones this Skylark global environment.
+ */
+ public SkylarkEnvironment cloneEnv(EventHandler eventHandler) {
+ Preconditions.checkArgument(isGlobalEnvironment());
+ SkylarkEnvironment newEnv = new SkylarkEnvironment(eventHandler, this.fileContentHashCode);
+ for (Entry<String, Object> entry : env.entrySet()) {
+ newEnv.env.put(entry.getKey(), entry.getValue());
+ }
+ for (Map.Entry<Class<?>, Map<String, Function>> functionMap : functions.entrySet()) {
+ newEnv.functions.put(functionMap.getKey(), functionMap.getValue());
+ }
+ return newEnv;
+ }
+
+ /**
+ * Returns the global environment. Only works for Skylark environments. For the global Skylark
+ * environment this method returns this Environment.
+ */
+ public SkylarkEnvironment getGlobalEnvironment() {
+ // If there's a parent that's the global environment, otherwise this is.
+ return parent != null ? (SkylarkEnvironment) parent : this;
+ }
+
+ /**
+ * Returns true if this is a Skylark global environment.
+ */
+ public boolean isGlobalEnvironment() {
+ return parent == null;
+ }
+
+ /**
+ * Returns true if varname has been read as a global variable.
+ */
+ public boolean hasBeenReadGlobalVariable(String varname) {
+ return readGlobalVariables.contains(varname);
+ }
+
+ @Override
+ public boolean isSkylarkEnabled() {
+ return true;
+ }
+
+ /**
+ * @return the value from the environment whose name is "varname".
+ * @throws NoSuchVariableException if the variable is not defined in the environment.
+ */
+ @Override
+ public Object lookup(String varname) throws NoSuchVariableException {
+ if (disabledVariables.contains(varname)) {
+ throw new NoSuchVariableException(varname);
+ }
+ Object value = env.get(varname);
+ if (value == null) {
+ if (parent != null && parent.hasVariable(varname)) {
+ readGlobalVariables.add(varname);
+ return parent.lookup(varname);
+ }
+ throw new NoSuchVariableException(varname);
+ }
+ return value;
+ }
+
+ /**
+ * Like <code>lookup(String)</code>, but instead of throwing an exception in
+ * the case where "varname" is not defined, "defaultValue" is returned instead.
+ */
+ @Override
+ public Object lookup(String varname, Object defaultValue) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Updates the value of variable "varname" in the environment, corresponding
+ * to an AssignmentStatement.
+ */
+ @Override
+ public void update(String varname, Object value) {
+ Preconditions.checkNotNull(value, "update(value == null)");
+ env.put(varname, value);
+ }
+
+ /**
+ * Returns the class of the variable or null if the variable does not exist. This function
+ * works only in the local Environment, it doesn't check the global Environment.
+ */
+ public Class<?> getVariableType(String varname) {
+ Object variable = env.get(varname);
+ return variable != null ? EvalUtils.getSkylarkType(variable.getClass()) : null;
+ }
+
+ /**
+ * Removes the functions and the modules (i.e. the symbol of the module from the top level
+ * Environment and the functions attached to it) from the Environment which should be present
+ * only during the loading phase.
+ */
+ public void disableOnlyLoadingPhaseObjects() {
+ List<String> objectsToRemove = new ArrayList<>();
+ List<Class<?>> modulesToRemove = new ArrayList<>();
+ for (Map.Entry<String, Object> entry : env.entrySet()) {
+ Object object = entry.getValue();
+ if (object instanceof SkylarkFunction) {
+ if (((SkylarkFunction) object).isOnlyLoadingPhase()) {
+ objectsToRemove.add(entry.getKey());
+ }
+ } else if (object.getClass().isAnnotationPresent(SkylarkModule.class)) {
+ if (object.getClass().getAnnotation(SkylarkModule.class).onlyLoadingPhase()) {
+ objectsToRemove.add(entry.getKey());
+ modulesToRemove.add(entry.getValue().getClass());
+ }
+ }
+ }
+ for (String symbol : objectsToRemove) {
+ disabledVariables.add(symbol);
+ }
+ for (Class<?> moduleClass : modulesToRemove) {
+ disabledNameSpaces.add(moduleClass);
+ }
+ }
+
+ public void handleEvent(Event event) {
+ eventHandler.handle(event);
+ }
+
+ /**
+ * Returns a hash code calculated from the hash code of this Environment and the
+ * transitive closure of other Environments it loads.
+ */
+ public String getTransitiveFileContentHashCode() {
+ Fingerprint fingerprint = new Fingerprint();
+ fingerprint.addString(Preconditions.checkNotNull(fileContentHashCode));
+ // Calculate a new hash from the hash of the loaded Environments.
+ for (SkylarkEnvironment env : importedExtensions.values()) {
+ fingerprint.addString(env.getTransitiveFileContentHashCode());
+ }
+ return fingerprint.hexDigestAndReset();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkFunction.java
new file mode 100644
index 0000000..bd2cc83
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkFunction.java
@@ -0,0 +1,317 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.packages.Type.ConversionException;
+import com.google.devtools.build.lib.syntax.EvalException.EvalExceptionWithJavaCause;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * A function class for Skylark built in functions. Supports mandatory and optional arguments.
+ * All usable arguments have to be specified. In case of ambiguous arguments (a parameter is
+ * specified as positional and keyword arguments in the function call) an exception is thrown.
+ */
+public abstract class SkylarkFunction extends AbstractFunction {
+
+ private ImmutableList<String> parameters;
+ private ImmutableMap<String, SkylarkBuiltin.Param> parameterTypes;
+ private int mandatoryParamNum;
+ private boolean configured = false;
+ private Class<?> objectType;
+ private boolean onlyLoadingPhase;
+
+ /**
+ * Creates a SkylarkFunction with the given name.
+ */
+ public SkylarkFunction(String name) {
+ super(name);
+ }
+
+ /**
+ * Configures the parameter of this Skylark function using the annotation.
+ */
+ @VisibleForTesting
+ public void configure(SkylarkBuiltin annotation) {
+ Preconditions.checkState(!configured);
+ Preconditions.checkArgument(getName().equals(annotation.name()),
+ getName() + " != " + annotation.name());
+ mandatoryParamNum = 0;
+ ImmutableList.Builder<String> paramListBuilder = ImmutableList.builder();
+ ImmutableMap.Builder<String, SkylarkBuiltin.Param> paramTypeBuilder = ImmutableMap.builder();
+ for (SkylarkBuiltin.Param param : annotation.mandatoryParams()) {
+ paramListBuilder.add(param.name());
+ paramTypeBuilder.put(param.name(), param);
+ mandatoryParamNum++;
+ }
+ for (SkylarkBuiltin.Param param : annotation.optionalParams()) {
+ paramListBuilder.add(param.name());
+ paramTypeBuilder.put(param.name(), param);
+ }
+ parameters = paramListBuilder.build();
+ parameterTypes = paramTypeBuilder.build();
+ this.objectType = annotation.objectType().equals(Object.class) ? null : annotation.objectType();
+ this.onlyLoadingPhase = annotation.onlyLoadingPhase();
+ configured = true;
+ }
+
+ /**
+ * Returns true if the SkylarkFunction is configured.
+ */
+ public boolean isConfigured() {
+ return configured;
+ }
+
+ @Override
+ public Class<?> getObjectType() {
+ return objectType;
+ }
+
+ public boolean isOnlyLoadingPhase() {
+ return onlyLoadingPhase;
+ }
+
+ @Override
+ public Object call(List<Object> args,
+ Map<String, Object> kwargs,
+ FuncallExpression ast,
+ Environment env)
+ throws EvalException, InterruptedException {
+
+ Preconditions.checkState(configured, "Function " + getName() + " was not configured");
+ try {
+ ImmutableMap.Builder<String, Object> arguments = new ImmutableMap.Builder<>();
+ if (objectType != null && !FuncallExpression.isNamespace(objectType)) {
+ arguments.put("self", args.remove(0));
+ }
+
+ int maxParamNum = parameters.size();
+ int paramNum = args.size() + kwargs.size();
+
+ if (paramNum < mandatoryParamNum) {
+ throw new EvalException(ast.getLocation(),
+ String.format("incorrect number of arguments (got %s, expected at least %s)",
+ paramNum, mandatoryParamNum));
+ } else if (paramNum > maxParamNum) {
+ throw new EvalException(ast.getLocation(),
+ String.format("incorrect number of arguments (got %s, expected at most %s)",
+ paramNum, maxParamNum));
+ }
+
+ for (int i = 0; i < mandatoryParamNum; i++) {
+ Preconditions.checkState(i < args.size() || kwargs.containsKey(parameters.get(i)),
+ String.format("missing mandatory parameter: %s", parameters.get(i)));
+ }
+
+ for (int i = 0; i < args.size(); i++) {
+ checkTypeAndAddArg(parameters.get(i), args.get(i), arguments, ast.getLocation());
+ }
+
+ for (Entry<String, Object> kwarg : kwargs.entrySet()) {
+ int idx = parameters.indexOf(kwarg.getKey());
+ if (idx < 0) {
+ throw new EvalException(ast.getLocation(),
+ String.format("unknown keyword argument: %s", kwarg.getKey()));
+ }
+ if (idx < args.size()) {
+ throw new EvalException(ast.getLocation(),
+ String.format("ambiguous argument: %s", kwarg.getKey()));
+ }
+ checkTypeAndAddArg(kwarg.getKey(), kwarg.getValue(), arguments, ast.getLocation());
+ }
+
+ return call(arguments.build(), ast, env);
+ } catch (ConversionException | IllegalArgumentException | IllegalStateException
+ | ClassCastException | ClassNotFoundException | ExecutionException e) {
+ if (e.getMessage() != null) {
+ throw new EvalException(ast.getLocation(), e.getMessage());
+ } else {
+ // TODO(bazel-team): ideally this shouldn't happen, however we need this for debugging
+ throw new EvalExceptionWithJavaCause(ast.getLocation(), e);
+ }
+ }
+ }
+
+ private void checkTypeAndAddArg(String paramName, Object value,
+ ImmutableMap.Builder<String, Object> arguments, Location loc) throws EvalException {
+ SkylarkBuiltin.Param param = parameterTypes.get(paramName);
+ if (param.callbackEnabled() && Function.class.isAssignableFrom(value.getClass())) {
+ // If we pass a function as an argument we trust the Function implementation with the type
+ // check. It's OK since the function needs to be called manually anyway.
+ arguments.put(paramName, value);
+ return;
+ }
+ if (!(param.type().isAssignableFrom(value.getClass()))) {
+ throw new EvalException(loc, String.format("expected %s for '%s' but got %s instead\n"
+ + "%s.%s: %s",
+ EvalUtils.getDataTypeNameFromClass(param.type()), paramName,
+ EvalUtils.getDatatypeName(value), getName(), paramName, param.doc()));
+ }
+ if (param.type().equals(SkylarkList.class)) {
+ checkGeneric(paramName, param, value, ((SkylarkList) value).getGenericType(), loc);
+ } else if (param.type().equals(SkylarkNestedSet.class)) {
+ checkGeneric(paramName, param, value, ((SkylarkNestedSet) value).getGenericType(), loc);
+ }
+ arguments.put(paramName, value);
+ }
+
+ private void checkGeneric(String paramName, SkylarkBuiltin.Param param, Object value,
+ Class<?> genericType, Location loc) throws EvalException {
+ if (!genericType.equals(Object.class) && !param.generic1().isAssignableFrom(genericType)) {
+ String mainType = EvalUtils.getDataTypeNameFromClass(param.type());
+ throw new EvalException(loc, String.format(
+ "expected %s of %ss for '%s' but got %s of %ss instead\n%s.%s: %s",
+ mainType, EvalUtils.getDataTypeNameFromClass(param.generic1()),
+ paramName,
+ EvalUtils.getDatatypeName(value), EvalUtils.getDataTypeNameFromClass(genericType),
+ getName(), paramName, param.doc()));
+ }
+ }
+
+ /**
+ * The actual function call. All positional and keyword arguments are put in the
+ * arguments map.
+ */
+ protected abstract Object call(
+ Map<String, Object> arguments, FuncallExpression ast, Environment env) throws EvalException,
+ ConversionException,
+ IllegalArgumentException,
+ IllegalStateException,
+ ClassCastException,
+ ClassNotFoundException,
+ ExecutionException;
+
+ /**
+ * An intermediate class to provide a simpler interface for Skylark functions.
+ */
+ public abstract static class SimpleSkylarkFunction extends SkylarkFunction {
+
+ public SimpleSkylarkFunction(String name) {
+ super(name);
+ }
+
+ @Override
+ protected final Object call(
+ Map<String, Object> arguments, FuncallExpression ast, Environment env) throws EvalException,
+ ConversionException,
+ IllegalArgumentException,
+ IllegalStateException,
+ ClassCastException,
+ ExecutionException {
+ return call(arguments, ast.getLocation());
+ }
+
+ /**
+ * The actual function call. All positional and keyword arguments are put in the
+ * arguments map.
+ */
+ protected abstract Object call(Map<String, Object> arguments, Location loc)
+ throws EvalException,
+ ConversionException,
+ IllegalArgumentException,
+ IllegalStateException,
+ ClassCastException,
+ ExecutionException;
+ }
+
+ public static <TYPE> Iterable<TYPE> castList(Object obj, final Class<TYPE> type) {
+ if (obj == null) {
+ return ImmutableList.of();
+ }
+ return ((SkylarkList) obj).to(type);
+ }
+
+ public static <TYPE> Iterable<TYPE> castList(
+ Object obj, final Class<TYPE> type, final String what) throws ConversionException {
+ if (obj == null) {
+ return ImmutableList.of();
+ }
+ return Iterables.transform(Type.LIST.convert(obj, what),
+ new com.google.common.base.Function<Object, TYPE>() {
+ @Override
+ public TYPE apply(Object input) {
+ try {
+ return type.cast(input);
+ } catch (ClassCastException e) {
+ throw new IllegalArgumentException(String.format(
+ "expected %s type for '%s' but got %s instead",
+ EvalUtils.getDataTypeNameFromClass(type), what,
+ EvalUtils.getDatatypeName(input)));
+ }
+ }
+ });
+ }
+
+ public static <KEY_TYPE, VALUE_TYPE> ImmutableMap<KEY_TYPE, VALUE_TYPE> toMap(
+ Iterable<Map.Entry<KEY_TYPE, VALUE_TYPE>> obj) {
+ ImmutableMap.Builder<KEY_TYPE, VALUE_TYPE> builder = ImmutableMap.builder();
+ for (Map.Entry<KEY_TYPE, VALUE_TYPE> entry : obj) {
+ builder.put(entry.getKey(), entry.getValue());
+ }
+ return builder.build();
+ }
+
+ public static <KEY_TYPE, VALUE_TYPE> Iterable<Map.Entry<KEY_TYPE, VALUE_TYPE>> castMap(Object obj,
+ final Class<KEY_TYPE> keyType, final Class<VALUE_TYPE> valueType, final String what) {
+ if (obj == null) {
+ return ImmutableList.of();
+ }
+ if (!(obj instanceof Map<?, ?>)) {
+ throw new IllegalArgumentException(String.format(
+ "expected a dictionary for %s but got %s instead",
+ what, EvalUtils.getDatatypeName(obj)));
+ }
+ return Iterables.transform(((Map<?, ?>) obj).entrySet(),
+ new com.google.common.base.Function<Map.Entry<?, ?>, Map.Entry<KEY_TYPE, VALUE_TYPE>>() {
+ // This is safe. We check the type of the key-value pairs for every entry in the Map.
+ // In Map.Entry the key always has the type of the first generic parameter, the
+ // value has the second.
+ @SuppressWarnings("unchecked")
+ @Override
+ public Map.Entry<KEY_TYPE, VALUE_TYPE> apply(Map.Entry<?, ?> input) {
+ if (keyType.isAssignableFrom(input.getKey().getClass())
+ && valueType.isAssignableFrom(input.getValue().getClass())) {
+ return (Map.Entry<KEY_TYPE, VALUE_TYPE>) input;
+ }
+ throw new IllegalArgumentException(String.format(
+ "expected <%s, %s> type for '%s' but got <%s, %s> instead",
+ keyType.getSimpleName(), valueType.getSimpleName(), what,
+ EvalUtils.getDatatypeName(input.getKey()),
+ EvalUtils.getDatatypeName(input.getValue())));
+ }
+ });
+ }
+
+ // TODO(bazel-team): this is only used in SkylarkRuleConfgiuredTargetBuilder, fix typing for
+ // structs then remove this.
+ public static <TYPE> TYPE cast(Object elem, Class<TYPE> type, String what, Location loc)
+ throws EvalException {
+ try {
+ return type.cast(elem);
+ } catch (ClassCastException e) {
+ throw new EvalException(loc, String.format("expected %s for '%s' but got %s instead",
+ type.getSimpleName(), what, EvalUtils.getDatatypeName(elem)));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkList.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkList.java
new file mode 100644
index 0000000..ef9fe10
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkList.java
@@ -0,0 +1,373 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.events.Location;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * A class to handle lists and tuples in Skylark.
+ */
+@SkylarkModule(name = "list",
+ doc = "A language built-in type to support lists. Example of list literal:<br>"
+ + "<pre class=language-python>l = [1, 2, 3]</pre>"
+ + "Accessing elements is possible using indexing (starts from <code>0</code>):<br>"
+ + "<pre class=language-python>e = l[1] # e == 2</pre>"
+ + "Lists support the <code>+</code> operator to concatenate two lists. Example:<br>"
+ + "<pre class=language-python>l = [1, 2] + [3, 4] # l == [1, 2, 3, 4]\n"
+ + "l = [\"a\", \"b\"]\n"
+ + "l += [\"c\"] # l == [\"a\", \"b\", \"c\"]</pre>"
+ + "List elements have to be of the same type, <code>[1, 2, \"c\"]</code> results in an "
+ + "error. Lists - just like everything - are immutable, therefore <code>l[1] = \"a\""
+ + "</code> is not supported.")
+public abstract class SkylarkList implements Iterable<Object> {
+
+ private final boolean tuple;
+ private final Class<?> genericType;
+
+ private SkylarkList(boolean tuple, Class<?> genericType) {
+ this.tuple = tuple;
+ this.genericType = genericType;
+ }
+
+ /**
+ * The size of the list.
+ */
+ public abstract int size();
+
+ /**
+ * Returns true if the list is empty.
+ */
+ public abstract boolean isEmpty();
+
+ /**
+ * Returns the i-th element of the list.
+ */
+ public abstract Object get(int i);
+
+ /**
+ * Returns true if this list is a tuple.
+ */
+ public boolean isTuple() {
+ return tuple;
+ }
+
+ @VisibleForTesting
+ public Class<?> getGenericType() {
+ return genericType;
+ }
+
+ @Override
+ public String toString() {
+ return toList().toString();
+ }
+
+ // TODO(bazel-team): we should be very careful using this method. Check and remove
+ // auto conversions on the Java-Skylark interface if possible.
+ /**
+ * Converts this Skylark list to a Java list.
+ */
+ public abstract List<?> toList();
+
+ @SuppressWarnings("unchecked")
+ public <T> Iterable<T> to(Class<T> type) {
+ Preconditions.checkArgument(this == EMPTY_LIST || type.isAssignableFrom(genericType));
+ return (Iterable<T>) this;
+ }
+
+ private static final class EmptySkylarkList extends SkylarkList {
+ private EmptySkylarkList(boolean tuple) {
+ super(tuple, Object.class);
+ }
+
+ @Override
+ public Iterator<Object> iterator() {
+ return ImmutableList.of().iterator();
+ }
+
+ @Override
+ public int size() {
+ return 0;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return true;
+ }
+
+ @Override
+ public Object get(int i) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<?> toList() {
+ return isTuple() ? ImmutableList.of() : Lists.newArrayList();
+ }
+
+ @Override
+ public String toString() {
+ return "[]";
+ }
+ }
+
+ /**
+ * An empty Skylark list.
+ */
+ public static final SkylarkList EMPTY_LIST = new EmptySkylarkList(false);
+
+ private static final class SimpleSkylarkList extends SkylarkList {
+ private final ImmutableList<Object> list;
+
+ private SimpleSkylarkList(ImmutableList<Object> list, boolean tuple, Class<?> genericType) {
+ super(tuple, genericType);
+ this.list = Preconditions.checkNotNull(list);
+ }
+
+ @Override
+ public Iterator<Object> iterator() {
+ return list.iterator();
+ }
+
+ @Override
+ public int size() {
+ return list.size();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return list.isEmpty();
+ }
+
+ @Override
+ public Object get(int i) {
+ return list.get(i);
+ }
+
+ @Override
+ public List<?> toList() {
+ return isTuple() ? list : Lists.newArrayList(list);
+ }
+
+ @Override
+ public String toString() {
+ return list.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return list.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof SimpleSkylarkList)) {
+ return false;
+ }
+ SimpleSkylarkList other = (SimpleSkylarkList) obj;
+ return other.list.equals(this.list);
+ }
+ }
+
+ /**
+ * A Skylark list to support lazy iteration (i.e. we only call iterator on the object this
+ * list masks when it's absolutely necessary). This is useful if iteration is expensive
+ * (e.g. NestedSet-s). Size(), get() and isEmpty() are expensive operations but
+ * concatenation is quick.
+ */
+ private static final class LazySkylarkList extends SkylarkList {
+ private final Iterable<Object> iterable;
+ private ImmutableList<Object> list = null;
+
+ private LazySkylarkList(Iterable<Object> iterable, boolean tuple, Class<?> genericType) {
+ super(tuple, genericType);
+ this.iterable = Preconditions.checkNotNull(iterable);
+ }
+
+ @Override
+ public Iterator<Object> iterator() {
+ return iterable.iterator();
+ }
+
+ @Override
+ public int size() {
+ return Iterables.size(iterable);
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return Iterables.isEmpty(iterable);
+ }
+
+ @Override
+ public Object get(int i) {
+ return getList().get(i);
+ }
+
+ @Override
+ public List<?> toList() {
+ return getList();
+ }
+
+ private ImmutableList<Object> getList() {
+ if (list == null) {
+ list = ImmutableList.copyOf(iterable);
+ }
+ return list;
+ }
+ }
+
+ /**
+ * A Skylark list to support quick concatenation of lists. Concatenation is O(1),
+ * size(), isEmpty() is O(n), get() is O(h).
+ */
+ private static final class ConcatenatedSkylarkList extends SkylarkList {
+ private final SkylarkList left;
+ private final SkylarkList right;
+
+ private ConcatenatedSkylarkList(
+ SkylarkList left, SkylarkList right, boolean tuple, Class<?> genericType) {
+ super(tuple, genericType);
+ this.left = Preconditions.checkNotNull(left);
+ this.right = Preconditions.checkNotNull(right);
+ }
+
+ @Override
+ public Iterator<Object> iterator() {
+ return Iterables.concat(left, right).iterator();
+ }
+
+ @Override
+ public int size() {
+ // We shouldn't evaluate the size function until it's necessary, because it can be expensive
+ // for lazy lists (e.g. lists containing a NestedSet).
+ // TODO(bazel-team): make this class more clever to store the size and empty parameters
+ // for every non-LazySkylarkList member.
+ return left.size() + right.size();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return left.isEmpty() && right.isEmpty();
+ }
+
+ @Override
+ public Object get(int i) {
+ int leftSize = left.size();
+ if (i < leftSize) {
+ return left.get(i);
+ } else {
+ return right.get(i - leftSize);
+ }
+ }
+
+ @Override
+ public List<?> toList() {
+ return ImmutableList.<Object>builder().addAll(left).addAll(right).build();
+ }
+ }
+
+ /**
+ * Returns a Skylark list containing elements without a type check. Only use if all elements
+ * are of the same type.
+ */
+ public static SkylarkList list(Collection<?> elements, Class<?> genericType) {
+ if (elements.isEmpty()) {
+ return EMPTY_LIST;
+ }
+ return new SimpleSkylarkList(ImmutableList.copyOf(elements), false, genericType);
+ }
+
+ /**
+ * Returns a Skylark list containing elements without a type check and without creating
+ * an immutable copy. Therefore the iterable containing elements must be immutable
+ * (which is not checked here so callers must be extra careful). This way
+ * it's possibly to create a SkylarkList without requesting the original iterator. This
+ * can be useful for nested set - list conversions.
+ */
+ @SuppressWarnings("unchecked")
+ public static SkylarkList lazyList(Iterable<?> elements, Class<?> genericType) {
+ return new LazySkylarkList((Iterable<Object>) elements, false, genericType);
+ }
+
+ /**
+ * Returns a Skylark list containing elements. Performs type check and throws an exception
+ * in case the list contains elements of different type.
+ */
+ public static SkylarkList list(Collection<?> elements, Location loc) throws EvalException {
+ if (elements.isEmpty()) {
+ return EMPTY_LIST;
+ }
+ return new SimpleSkylarkList(
+ ImmutableList.copyOf(elements), false, getGenericType(elements, loc));
+ }
+
+ private static Class<?> getGenericType(Collection<?> elements, Location loc)
+ throws EvalException {
+ Class<?> genericType = elements.iterator().next().getClass();
+ for (Object element : elements) {
+ Class<?> type = element.getClass();
+ if (!EvalUtils.getSkylarkType(genericType).equals(EvalUtils.getSkylarkType(type))) {
+ throw new EvalException(loc, String.format(
+ "Incompatible types in list: found a %s but the first element is a %s",
+ EvalUtils.getDataTypeNameFromClass(type),
+ EvalUtils.getDataTypeNameFromClass(genericType)));
+ }
+ }
+ return genericType;
+ }
+
+ /**
+ * Returns a Skylark list created from Skylark lists left and right. Throws an exception
+ * if they are not of the same generic type.
+ */
+ public static SkylarkList concat(SkylarkList left, SkylarkList right, Location loc)
+ throws EvalException {
+ if (left.isTuple() != right.isTuple()) {
+ throw new EvalException(loc, "cannot concatenate lists and tuples");
+ }
+ if (left == EMPTY_LIST) {
+ return right;
+ }
+ if (right == EMPTY_LIST) {
+ return left;
+ }
+ if (!left.genericType.equals(right.genericType)) {
+ throw new EvalException(loc, String.format("cannot concatenate list of %s with list of %s",
+ EvalUtils.getDataTypeNameFromClass(left.genericType),
+ EvalUtils.getDataTypeNameFromClass(right.genericType)));
+ }
+ return new ConcatenatedSkylarkList(left, right, left.isTuple(), left.genericType);
+ }
+
+ /**
+ * Returns a Skylark tuple containing elements.
+ */
+ public static SkylarkList tuple(List<?> elements) {
+ // Tuple elements do not have to have the same type.
+ return new SimpleSkylarkList(ImmutableList.copyOf(elements), true, Object.class);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkModule.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkModule.java
new file mode 100644
index 0000000..96421b2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkModule.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation to mark Skylark modules or Skylark accessible Java data types.
+ * A Skylark modules always corresponds to exactly one Java class.
+ */
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SkylarkModule {
+
+ String name();
+
+ String doc();
+
+ boolean hidden() default false;
+
+ boolean namespace() default false;
+
+ boolean onlyLoadingPhase() default false;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkNestedSet.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkNestedSet.java
new file mode 100644
index 0000000..17fc55f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkNestedSet.java
@@ -0,0 +1,193 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
+import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.Location;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * A generic type safe NestedSet wrapper for Skylark.
+ */
+@SkylarkModule(name = "set",
+ doc = "A language built-in type to supports (nested) sets. "
+ + "Sets can be created using the global <code>set</code> function, and they "
+ + "support the <code>+</code> operator to extends and nest sets. Examples:<br>"
+ + "<pre class=language-python>s = set([1, 2])\n"
+ + "s += [3] # s == {1, 2, 3}\n"
+ + "s += set([4, 5]) # s == {1, 2, 3, {4, 5}}</pre>"
+ + "Note that in these examples <code>{..}</code> is not a valid literal to create sets. "
+ + "Sets have a fixed generic type, so <code>set([1]) + [\"a\"]</code> or "
+ + "<code>set([1]) + set([\"a\"])</code> results in an error.")
+@Immutable
+public final class SkylarkNestedSet implements Iterable<Object> {
+
+ private final Class<?> genericType;
+ @Nullable private final List<Object> items;
+ @Nullable private final List<NestedSet<Object>> transitiveItems;
+ private final NestedSet<?> set;
+
+ public SkylarkNestedSet(Order order, Object item, Location loc) throws EvalException {
+ this(order, Object.class, item, loc, new ArrayList<Object>(),
+ new ArrayList<NestedSet<Object>>());
+ }
+
+ public SkylarkNestedSet(SkylarkNestedSet left, Object right, Location loc) throws EvalException {
+ this(left.set.getOrder(), left.genericType, right, loc,
+ new ArrayList<Object>(checkItems(left.items, loc)),
+ new ArrayList<NestedSet<Object>>(checkItems(left.transitiveItems, loc)));
+ }
+
+ private static <T> T checkItems(T items, Location loc) throws EvalException {
+ // SkylarkNestedSets created directly from ordinary NestedSets (those were created in a
+ // native rule) don't have directly accessible items and transitiveItems, so we cannot
+ // add more elements to them.
+ if (items == null) {
+ throw new EvalException(loc, "Cannot add more elements to this set. Sets created in "
+ + "native rules cannot be left side operands of the + operator.");
+ }
+ return items;
+ }
+
+ // This is safe because of the type checking
+ @SuppressWarnings("unchecked")
+ private SkylarkNestedSet(Order order, Class<?> genericType, Object item, Location loc,
+ List<Object> items, List<NestedSet<Object>> transitiveItems) throws EvalException {
+
+ // Adding the item
+ if (item instanceof SkylarkNestedSet) {
+ SkylarkNestedSet nestedSet = (SkylarkNestedSet) item;
+ if (!nestedSet.isEmpty()) {
+ genericType = checkType(genericType, nestedSet.genericType, loc);
+ transitiveItems.add((NestedSet<Object>) nestedSet.set);
+ }
+ } else if (item instanceof SkylarkList) {
+ // TODO(bazel-team): we should check ImmutableList here but it screws up genrule at line 43
+ for (Object object : (SkylarkList) item) {
+ genericType = checkType(genericType, object.getClass(), loc);
+ items.add(object);
+ }
+ } else {
+ throw new EvalException(loc,
+ String.format("cannot add '%s'-s to nested sets", EvalUtils.getDatatypeName(item)));
+ }
+ this.genericType = Preconditions.checkNotNull(genericType, "type cannot be null");
+
+ // Initializing the real nested set
+ NestedSetBuilder<Object> builder = new NestedSetBuilder<Object>(order);
+ builder.addAll(items);
+ try {
+ for (NestedSet<Object> nestedSet : transitiveItems) {
+ builder.addTransitive(nestedSet);
+ }
+ } catch (IllegalStateException e) {
+ throw new EvalException(loc, e.getMessage());
+ }
+ this.set = builder.build();
+ this.items = ImmutableList.copyOf(items);
+ this.transitiveItems = ImmutableList.copyOf(transitiveItems);
+ }
+
+ /**
+ * Returns a type safe SkylarkNestedSet. Use this instead of the constructor if possible.
+ */
+ public static <T> SkylarkNestedSet of(Class<T> genericType, NestedSet<T> set) {
+ return new SkylarkNestedSet(genericType, set);
+ }
+
+ /**
+ * A not type safe constructor for SkylarkNestedSet. It's discouraged to use it unless type
+ * generic safety is guaranteed from the caller side.
+ */
+ SkylarkNestedSet(Class<?> genericType, NestedSet<?> set) {
+ // This is here for the sake of FuncallExpression.
+ this.genericType = Preconditions.checkNotNull(genericType, "type cannot be null");
+ this.set = Preconditions.checkNotNull(set, "set cannot be null");
+ this.items = null;
+ this.transitiveItems = null;
+ }
+
+ private static Class<?> checkType(Class<?> builderType, Class<?> itemType, Location loc)
+ throws EvalException {
+ if (Map.class.isAssignableFrom(itemType) || SkylarkList.class.isAssignableFrom(itemType)
+ || ClassObject.class.isAssignableFrom(itemType)) {
+ throw new EvalException(loc, String.format("nested set item is composite (type of %s)",
+ EvalUtils.getDataTypeNameFromClass(itemType)));
+ }
+ if (!EvalUtils.isSkylarkImmutable(itemType)) {
+ throw new EvalException(loc, String.format("nested set item is not immutable (type of %s)",
+ EvalUtils.getDataTypeNameFromClass(itemType)));
+ }
+ if (builderType.equals(Object.class)) {
+ return itemType;
+ }
+ if (!EvalUtils.getSkylarkType(builderType).equals(EvalUtils.getSkylarkType(itemType))) {
+ throw new EvalException(loc, String.format(
+ "nested set item is type of %s but the nested set accepts only %s-s",
+ EvalUtils.getDataTypeNameFromClass(itemType),
+ EvalUtils.getDataTypeNameFromClass(builderType)));
+ }
+ return builderType;
+ }
+
+ /**
+ * Returns the NestedSet embedded in this SkylarkNestedSet if it is of the parameter type.
+ */
+ // The precondition ensures generic type safety
+ @SuppressWarnings("unchecked")
+ public <T> NestedSet<T> getSet(Class<T> type) {
+ // Empty sets don't need have to have a type since they don't have items
+ if (set.isEmpty()) {
+ return (NestedSet<T>) set;
+ }
+ Preconditions.checkArgument(type.isAssignableFrom(genericType),
+ String.format("Expected %s as a type but got %s",
+ EvalUtils.getDataTypeNameFromClass(type),
+ EvalUtils.getDataTypeNameFromClass(genericType)));
+ return (NestedSet<T>) set;
+ }
+
+ // For some reason this cast is unsafe in Java
+ @SuppressWarnings("unchecked")
+ @Override
+ public Iterator<Object> iterator() {
+ return (Iterator<Object>) set.iterator();
+ }
+
+ public Collection<Object> toCollection() {
+ return ImmutableList.copyOf(set.toCollection());
+ }
+
+ public boolean isEmpty() {
+ return set.isEmpty();
+ }
+
+ @VisibleForTesting
+ public Class<?> getGenericType() {
+ return genericType;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkType.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkType.java
new file mode 100644
index 0000000..04c345f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkType.java
@@ -0,0 +1,307 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.events.Location;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.lang.reflect.WildcardType;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+/**
+ * A class representing types available in Skylark.
+ */
+public class SkylarkType {
+
+ private static final class Global {}
+
+ public static final SkylarkType UNKNOWN = new SkylarkType(Object.class);
+ public static final SkylarkType NONE = new SkylarkType(Environment.NoneType.class);
+ public static final SkylarkType GLOBAL = new SkylarkType(Global.class);
+
+ public static final SkylarkType STRING = new SkylarkType(String.class);
+ public static final SkylarkType INT = new SkylarkType(Integer.class);
+ public static final SkylarkType BOOL = new SkylarkType(Boolean.class);
+
+ private final Class<?> type;
+
+ // TODO(bazel-team): Change this to SkylarkType and check generics of generics etc.
+ // Object.class is used for UNKNOWN.
+ private Class<?> generic1;
+
+ public static SkylarkType of(Class<?> type, Class<?> generic1) {
+ return new SkylarkType(type, generic1);
+ }
+
+ public static SkylarkType of(Class<?> type) {
+ if (type.equals(Object.class)) {
+ return SkylarkType.UNKNOWN;
+ } else if (type.equals(String.class)) {
+ return SkylarkType.STRING;
+ } else if (type.equals(Integer.class)) {
+ return SkylarkType.INT;
+ } else if (type.equals(Boolean.class)) {
+ return SkylarkType.BOOL;
+ }
+ return new SkylarkType(type);
+ }
+
+ private SkylarkType(Class<?> type, Class<?> generic1) {
+ this.type = Preconditions.checkNotNull(type);
+ this.generic1 = Preconditions.checkNotNull(generic1);
+ }
+
+ private SkylarkType(Class<?> type) {
+ this.type = Preconditions.checkNotNull(type);
+ this.generic1 = Object.class;
+ }
+
+ public Class<?> getType() {
+ return type;
+ }
+
+ Class<?> getGenericType1() {
+ return generic1;
+ }
+
+ /**
+ * Returns the stronger type of this and o if they are compatible. Stronger means that
+ * the more information is available, e.g. STRING is stronger than UNKNOWN and
+ * LIST<STRING> is stronger than LIST<UNKNOWN>. Note than there's no type
+ * hierarchy in Skylark.
+ * <p>If they are not compatible an EvalException is thrown.
+ */
+ SkylarkType infer(SkylarkType o, String name, Location thisLoc, Location originalLoc)
+ throws EvalException {
+ if (this == o) {
+ return this;
+ }
+ if (this == UNKNOWN || this.equals(SkylarkType.NONE)) {
+ return o;
+ }
+ if (o == UNKNOWN || o.equals(SkylarkType.NONE)) {
+ return this;
+ }
+ if (!type.equals(o.type)) {
+ throw new EvalException(thisLoc, String.format("bad %s: %s is incompatible with %s at %s",
+ name,
+ EvalUtils.getDataTypeNameFromClass(o.getType()),
+ EvalUtils.getDataTypeNameFromClass(this.getType()),
+ originalLoc));
+ }
+ if (generic1.equals(Object.class)) {
+ return o;
+ }
+ if (o.generic1.equals(Object.class)) {
+ return this;
+ }
+ if (!generic1.equals(o.generic1)) {
+ throw new EvalException(thisLoc, String.format("bad %s: incompatible generic variable types "
+ + "%s with %s",
+ name,
+ EvalUtils.getDataTypeNameFromClass(o.generic1),
+ EvalUtils.getDataTypeNameFromClass(this.generic1)));
+ }
+ return this;
+ }
+
+ boolean isStruct() {
+ return type.equals(ClassObject.class);
+ }
+
+ boolean isList() {
+ return SkylarkList.class.isAssignableFrom(type);
+ }
+
+ boolean isDict() {
+ return Map.class.isAssignableFrom(type);
+ }
+
+ boolean isSet() {
+ return Set.class.isAssignableFrom(type);
+ }
+
+ boolean isNset() {
+ // TODO(bazel-team): NestedSets are going to be a bit strange with 2 type info (validation
+ // and execution time). That can be cleaned up once we have complete type inference.
+ return SkylarkNestedSet.class.isAssignableFrom(type);
+ }
+
+ boolean isSimple() {
+ return !isStruct() && !isDict() && !isList() && !isNset() && !isSet();
+ }
+
+ @Override
+ public String toString() {
+ return this == UNKNOWN ? "Unknown" : EvalUtils.getDataTypeNameFromClass(type);
+ }
+
+ // hashCode() and equals() only uses the type field
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof SkylarkType)) {
+ return false;
+ }
+ SkylarkType o = (SkylarkType) other;
+ return this.type.equals(o.type);
+ }
+
+ @Override
+ public int hashCode() {
+ return type.hashCode();
+ }
+
+ /**
+ * A class representing the type of a Skylark function.
+ */
+ public static final class SkylarkFunctionType extends SkylarkType {
+
+ private final String name;
+ @Nullable private SkylarkType returnType;
+ @Nullable private Location returnTypeLoc;
+
+ public static SkylarkFunctionType of(String name) {
+ return new SkylarkFunctionType(name, null);
+ }
+
+ public static SkylarkFunctionType of(String name, SkylarkType returnType) {
+ return new SkylarkFunctionType(name, returnType);
+ }
+
+ private SkylarkFunctionType(String name, SkylarkType returnType) {
+ super(Function.class);
+ this.name = name;
+ this.returnType = returnType;
+ }
+
+ public SkylarkType getReturnType() {
+ return returnType;
+ }
+
+ /**
+ * Sets the return type of the function type if it's compatible with the existing return type.
+ * Note that setting NONE only has an effect if the return type hasn't been set previously.
+ */
+ public void setReturnType(SkylarkType newReturnType, Location newLoc) throws EvalException {
+ if (returnType == null) {
+ returnType = newReturnType;
+ returnTypeLoc = newLoc;
+ } else if (newReturnType != SkylarkType.NONE) {
+ returnType =
+ returnType.infer(newReturnType, "return type of " + name, newLoc, returnTypeLoc);
+ if (returnType == newReturnType) {
+ returnTypeLoc = newLoc;
+ }
+ }
+ }
+ }
+
+ private static boolean isTypeAllowedInSkylark(Object object) {
+ if (object instanceof NestedSet<?>) {
+ return false;
+ } else if (object instanceof List<?>) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Throws EvalException if the type of the object is not allowed to be present in Skylark.
+ */
+ static void checkTypeAllowedInSkylark(Object object, Location loc) throws EvalException {
+ if (!isTypeAllowedInSkylark(object)) {
+ throw new EvalException(loc,
+ "Type is not allowed in Skylark: "
+ + object.getClass().getSimpleName());
+ }
+ }
+
+ private static Class<?> getGenericTypeFromMethod(Method method) {
+ // This is where we can infer generic type information, so SkylarkNestedSets can be
+ // created in a safe way. Eventually we should probably do something with Lists and Maps too.
+ ParameterizedType t = (ParameterizedType) method.getGenericReturnType();
+ Type type = t.getActualTypeArguments()[0];
+ if (type instanceof Class) {
+ return (Class<?>) type;
+ }
+ if (type instanceof WildcardType) {
+ WildcardType wildcard = (WildcardType) type;
+ Type upperBound = wildcard.getUpperBounds()[0];
+ if (upperBound instanceof Class) {
+ // i.e. List<? extends SuperClass>
+ return (Class<?>) upperBound;
+ }
+ }
+ // It means someone annotated a method with @SkylarkCallable with no specific generic type info.
+ // We shouldn't annotate methods which return List<?> or List<T>.
+ throw new IllegalStateException("Cannot infer type from method signature " + method);
+ }
+
+ /**
+ * Converts an object retrieved from a Java method to a Skylark-compatible type.
+ */
+ static Object convertToSkylark(Object object, Method method) {
+ if (object instanceof NestedSet<?>) {
+ return new SkylarkNestedSet(getGenericTypeFromMethod(method), (NestedSet<?>) object);
+ } else if (object instanceof List<?>) {
+ return SkylarkList.list((List<?>) object, getGenericTypeFromMethod(method));
+ }
+ return object;
+ }
+
+ /**
+ * Converts an object to a Skylark-compatible type if possible.
+ */
+ public static Object convertToSkylark(Object object, Location loc) throws EvalException {
+ if (object instanceof List<?>) {
+ return SkylarkList.list((List<?>) object, loc);
+ }
+ return object;
+ }
+
+ /**
+ * Converts object from a Skylark-compatible wrapper type to its original type.
+ */
+ public static Object convertFromSkylark(Object value) {
+ if (value instanceof SkylarkList) {
+ return ((SkylarkList) value).toList();
+ }
+ return value;
+ }
+
+ /**
+ * Creates a SkylarkType from the SkylarkBuiltin annotation.
+ */
+ public static SkylarkType getReturnType(SkylarkBuiltin annotation) {
+ if (annotation.returnType().equals(Object.class)) {
+ return SkylarkType.UNKNOWN;
+ }
+ if (Function.class.isAssignableFrom(annotation.returnType())) {
+ return SkylarkFunctionType.of(annotation.name());
+ }
+ return SkylarkType.of(annotation.returnType());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Statement.java b/src/main/java/com/google/devtools/build/lib/syntax/Statement.java
new file mode 100644
index 0000000..ca89b1a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Statement.java
@@ -0,0 +1,44 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Base class for all statements nodes in the AST.
+ */
+public abstract class Statement extends ASTNode {
+
+ /**
+ * Executes the statement in the specified build environment, which may be
+ * modified.
+ *
+ * @throws EvalException if execution of the statement could not be completed.
+ */
+ abstract void exec(Environment env) throws EvalException, InterruptedException;
+
+ /**
+ * Checks the semantics of the Statement using the SkylarkEnvironment according to
+ * the rules of the Skylark language. The SkylarkEnvironment can be used e.g. to check
+ * variable type collision, read only variables, detecting recursion, existence of
+ * built-in variables, functions, etc.
+ *
+ * <p>The semantical check should be performed after the Skylark extension is loaded
+ * (i.e. is syntactically correct) and before is executed. The point of the semantical check
+ * is to make sure (as much as possible) that no error can occur during execution (Skylark
+ * programmers get a "compile time" error). It should also check execution branches (e.g. in
+ * if statements) that otherwise might never get executed.
+ *
+ * @throws EvalException if the Statement has a semantical error.
+ */
+ abstract void validate(ValidationEnvironment env) throws EvalException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/StringLiteral.java b/src/main/java/com/google/devtools/build/lib/syntax/StringLiteral.java
new file mode 100644
index 0000000..98d5045
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/StringLiteral.java
@@ -0,0 +1,56 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * Syntax node for a string literal.
+ */
+public final class StringLiteral extends Literal<String> {
+
+ private final char quoteChar;
+
+ public StringLiteral(String value, char quoteChar) {
+ super(value);
+ this.quoteChar = quoteChar;
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder()
+ .append(quoteChar)
+ .append(value.replace(Character.toString(quoteChar), "\\" + quoteChar))
+ .append(quoteChar)
+ .toString();
+ }
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ /**
+ * Gets the quote character that was used for this string. For example, if
+ * the string was 'hello, world!', then this method returns '\''.
+ *
+ * @return the character used to quote the string.
+ */
+ public char getQuoteChar() {
+ return quoteChar;
+ }
+
+ @Override
+ SkylarkType validate(ValidationEnvironment env) throws EvalException {
+ return SkylarkType.STRING;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SyntaxTreeVisitor.java b/src/main/java/com/google/devtools/build/lib/syntax/SyntaxTreeVisitor.java
new file mode 100644
index 0000000..5a95026
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SyntaxTreeVisitor.java
@@ -0,0 +1,145 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.devtools.build.lib.syntax.DictionaryLiteral.DictionaryEntryLiteral;
+import com.google.devtools.build.lib.syntax.IfStatement.ConditionalStatements;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A visitor for visiting the nodes in the syntax tree left to right, top to
+ * bottom.
+ */
+public class SyntaxTreeVisitor {
+
+ public void visit(ASTNode node) {
+ // dispatch to the node specific method
+ node.accept(this);
+ }
+
+ public void visitAll(List<? extends ASTNode> nodes) {
+ for (ASTNode node : nodes) {
+ visit(node);
+ }
+ }
+
+ // node specific visit methods
+ public void visit(Argument node) {
+ if (node.isNamed()) {
+ visit(node.getName());
+ }
+ if (node.hasValue()) {
+ visit(node.getValue());
+ }
+ }
+
+ public void visit(BuildFileAST node) {
+ visitAll(node.getStatements());
+ visitAll(node.getComments());
+ }
+
+ public void visit(BinaryOperatorExpression node) {
+ visit(node.getLhs());
+ visit(node.getRhs());
+ }
+
+ public void visit(FuncallExpression node) {
+ visit(node.getFunction());
+ visitAll(node.getArguments());
+ }
+
+ public void visit(Ident node) {
+ }
+
+ public void visit(ListComprehension node) {
+ visit(node.getElementExpression());
+ for (Map.Entry<Ident, Expression> list : node.getLists()) {
+ visit(list.getKey());
+ visit(list.getValue());
+ }
+ }
+
+ public void accept(DictComprehension node) {
+ visit(node.getKeyExpression());
+ visit(node.getValueExpression());
+ visit(node.getLoopVar());
+ visit(node.getListExpression());
+ }
+
+ public void visit(ListLiteral node) {
+ visitAll(node.getElements());
+ }
+
+ public void visit(IntegerLiteral node) {
+ }
+
+ public void visit(StringLiteral node) {
+ }
+
+ public void visit(AssignmentStatement node) {
+ visit(node.getLValue());
+ visit(node.getExpression());
+ }
+
+ public void visit(ExpressionStatement node) {
+ visit(node.getExpression());
+ }
+
+ public void visit(IfStatement node) {
+ for (ConditionalStatements stmt : node.getThenBlocks()) {
+ visit(stmt);
+ }
+ for (Statement stmt : node.getElseBlock()) {
+ visit(stmt);
+ }
+ }
+
+ public void visit(ConditionalStatements node) {
+ visit(node.getCondition());
+ for (Statement stmt : node.getStmts()) {
+ visit(stmt);
+ }
+ }
+
+ public void visit(FunctionDefStatement node) {
+ visit(node.getIdent());
+ for (Argument arg : node.getArgs()) {
+ visit(arg);
+ }
+ for (Statement stmt : node.getStatements()) {
+ visit(stmt);
+ }
+ }
+
+ public void visit(DictionaryLiteral node) {
+ for (DictionaryEntryLiteral entry : node.getEntries()) {
+ visit(entry);
+ }
+ }
+
+ public void visit(DictionaryEntryLiteral node) {
+ visit(node.getKey());
+ visit(node.getValue());
+ }
+
+ public void visit(NotExpression node) {
+ visit(node.getExpression());
+ }
+
+ public void visit(Comment node) {
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Token.java b/src/main/java/com/google/devtools/build/lib/syntax/Token.java
new file mode 100644
index 0000000..e3bcfec
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Token.java
@@ -0,0 +1,50 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * A Token represents an actual lexeme; that is, a lexical unit, its location in
+ * the input text, its lexical kind (TokenKind) and any associated value.
+ */
+public class Token {
+
+ public final TokenKind kind;
+ public final int left;
+ public final int right;
+ public final Object value;
+
+ public Token(TokenKind kind, int left, int right) {
+ this(kind, left, right, null);
+ }
+
+ public Token(TokenKind kind, int left, int right, Object value) {
+ this.kind = kind;
+ this.left = left;
+ this.right = right;
+ this.value = value;
+ }
+
+ /**
+ * Constructs an easy-to-read string representation of token, suitable for use
+ * in user error messages.
+ */
+ @Override
+ public String toString() {
+ // TODO(bazel-team): do proper escaping of string literals
+ return kind == TokenKind.STRING ? ("\"" + value + "\"")
+ : value == null ? kind.getPrettyName()
+ : value.toString();
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java b/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java
new file mode 100644
index 0000000..f6dad9f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java
@@ -0,0 +1,83 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+/**
+ * A TokenKind is an enumeration of each different kind of lexical symbol.
+ */
+public enum TokenKind {
+
+ AND("and"),
+ AS("as"),
+ CLASS("class"),
+ COLON(":"),
+ COMMA(","),
+ COMMENT("comment"),
+ DEF("def"),
+ DOT("."),
+ ELIF("elif"),
+ ELSE("else"),
+ EOF("EOF"),
+ EQUALS("="),
+ EQUALS_EQUALS("=="),
+ EXCEPT("except"),
+ FINALLY("finally"),
+ FOR("for"),
+ FROM("from"),
+ GREATER(">"),
+ GREATER_EQUALS(">="),
+ IDENTIFIER("identifier"),
+ IF("if"),
+ ILLEGAL("illegal character"),
+ IMPORT("import"),
+ IN("in"),
+ INDENT("indent"),
+ INT("integer"),
+ LBRACE("{"),
+ LBRACKET("["),
+ LESS("<"),
+ LESS_EQUALS("<="),
+ LPAREN("("),
+ MINUS("-"),
+ NEWLINE("newline"),
+ NOT("not"),
+ NOT_EQUALS("!="),
+ OR("or"),
+ OUTDENT("outdent"),
+ PERCENT("%"),
+ PLUS("+"),
+ PLUS_EQUALS("+="),
+ RBRACE("}"),
+ RBRACKET("]"),
+ RETURN("return"),
+ RPAREN(")"),
+ SEMI(";"),
+ STAR("*"),
+ STRING("string"),
+ TRY("try");
+
+ private final String prettyName;
+
+ private TokenKind(String prettyName) {
+ this.prettyName = prettyName;
+ }
+
+ /**
+ * Returns the pretty name for this token, for use in error messages for the user.
+ */
+ public String getPrettyName() {
+ return prettyName;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java b/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java
new file mode 100644
index 0000000..cd909a9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/UserDefinedFunction.java
@@ -0,0 +1,115 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.events.Location;
+
+/**
+ * The actual function registered in the environment. This function is defined in the
+ * parsed code using {@link FunctionDefStatement}.
+ */
+public class UserDefinedFunction extends MixedModeFunction {
+
+ private final ImmutableList<Argument> args;
+ private final ImmutableMap<String, Integer> argIndexes;
+ private final ImmutableMap<String, Object> defaultValues;
+ private final ImmutableList<Statement> statements;
+ private final SkylarkEnvironment definitionEnv;
+
+ private static ImmutableList<String> argumentToStringList(ImmutableList<Argument> args) {
+ Function<Argument, String> function = new Function<Argument, String>() {
+ @Override
+ public String apply(Argument id) {
+ return id.getArgName();
+ }
+ };
+ return ImmutableList.copyOf(Lists.transform(args, function));
+ }
+
+ private static int mandatoryArgNum(ImmutableList<Argument> args) {
+ int mandatoryArgNum = 0;
+ for (Argument arg : args) {
+ if (!arg.hasValue()) {
+ mandatoryArgNum++;
+ }
+ }
+ return mandatoryArgNum;
+ }
+
+ UserDefinedFunction(Ident function, ImmutableList<Argument> args,
+ ImmutableMap<String, Object> defaultValues,
+ ImmutableList<Statement> statements, SkylarkEnvironment definitionEnv) {
+ super(function.getName(), argumentToStringList(args), mandatoryArgNum(args), false,
+ function.getLocation());
+ this.args = args;
+ this.statements = statements;
+ this.definitionEnv = definitionEnv;
+ this.defaultValues = defaultValues;
+
+ ImmutableMap.Builder<String, Integer> argIndexes = new ImmutableMap.Builder<> ();
+ int i = 0;
+ for (Argument arg : args) {
+ if (!arg.isKwargs()) { // TODO(bazel-team): add varargs support?
+ argIndexes.put(arg.getArgName(), i++);
+ }
+ }
+ this.argIndexes = argIndexes.build();
+ }
+
+ public ImmutableList<Argument> getArgs() {
+ return args;
+ }
+
+ public Integer getArgIndex(String s) {
+ return argIndexes.get(s);
+ }
+
+ ImmutableMap<String, Object> getDefaultValues() {
+ return defaultValues;
+ }
+
+ ImmutableList<Statement> getStatements() {
+ return statements;
+ }
+
+ Location getLocation() {
+ return location;
+ }
+
+ @Override
+ public Object call(Object[] namedArguments, FuncallExpression ast, Environment env)
+ throws EvalException, InterruptedException {
+ SkylarkEnvironment functionEnv = SkylarkEnvironment.createEnvironmentForFunctionCalling(
+ env, definitionEnv, this);
+
+ // Registering the functions's arguments as variables in the local Environment
+ int i = 0;
+ for (Object arg : namedArguments) {
+ functionEnv.update(args.get(i++).getArgName(), arg);
+ }
+
+ try {
+ for (Statement stmt : statements) {
+ stmt.exec(functionEnv);
+ }
+ } catch (ReturnStatement.ReturnException e) {
+ return e.getValue();
+ }
+ return Environment.NONE;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ValidationEnvironment.java b/src/main/java/com/google/devtools/build/lib/syntax/ValidationEnvironment.java
new file mode 100644
index 0000000..6afb96a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ValidationEnvironment.java
@@ -0,0 +1,244 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.collect.CollectionUtils;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.syntax.SkylarkType.SkylarkFunctionType;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+
+/**
+ * An Environment for the semantic checking of Skylark files.
+ *
+ * @see Statement#validate
+ * @see Expression#validate
+ */
+public class ValidationEnvironment {
+
+ private final ValidationEnvironment parent;
+
+ private Map<SkylarkType, Map<String, SkylarkType>> variableTypes = new HashMap<>();
+
+ private Map<String, Location> variableLocations = new HashMap<>();
+
+ private Set<String> readOnlyVariables = new HashSet<>();
+
+ // A stack of variable-sets which are read only but can be assigned in different
+ // branches of if-else statements.
+ private Stack<Set<String>> futureReadOnlyVariables = new Stack<>();
+
+ // The function we are currently validating.
+ private SkylarkFunctionType currentFunction;
+
+ // Whether this validation environment is not modified therefore clonable or not.
+ private boolean clonable;
+
+ public ValidationEnvironment(
+ ImmutableMap<SkylarkType, ImmutableMap<String, SkylarkType>> builtinVariableTypes) {
+ parent = null;
+ variableTypes = CollectionUtils.copyOf(builtinVariableTypes);
+ readOnlyVariables.addAll(builtinVariableTypes.get(SkylarkType.GLOBAL).keySet());
+ clonable = true;
+ }
+
+ private ValidationEnvironment(Map<SkylarkType, Map<String, SkylarkType>> builtinVariableTypes,
+ Set<String> readOnlyVariables) {
+ parent = null;
+ this.variableTypes = CollectionUtils.copyOf(builtinVariableTypes);
+ this.readOnlyVariables = new HashSet<>(readOnlyVariables);
+ clonable = false;
+ }
+
+ @Override
+ public ValidationEnvironment clone() {
+ Preconditions.checkState(clonable);
+ return new ValidationEnvironment(variableTypes, readOnlyVariables);
+ }
+
+ /**
+ * Creates a local ValidationEnvironment to validate user defined function bodies.
+ */
+ public ValidationEnvironment(ValidationEnvironment parent, SkylarkFunctionType currentFunction) {
+ this.parent = parent;
+ this.variableTypes.put(SkylarkType.GLOBAL, new HashMap<String, SkylarkType>());
+ this.currentFunction = currentFunction;
+ for (String var : parent.readOnlyVariables) {
+ if (!parent.variableLocations.containsKey(var)) {
+ // Mark built in global vars readonly. Variables defined in Skylark may be shadowed locally.
+ readOnlyVariables.add(var);
+ }
+ }
+ this.clonable = false;
+ }
+
+ /**
+ * Returns true if this ValidationEnvironment is top level i.e. has no parent.
+ */
+ public boolean isTopLevel() {
+ return parent == null;
+ }
+
+ /**
+ * Updates the variable type if the new type is "stronger" then the old one.
+ * The old and the new vartype has to be compatible, otherwise an EvalException is thrown.
+ * The new type is stronger if the old one doesn't exist or unknown.
+ */
+ public void update(String varname, SkylarkType newVartype, Location location)
+ throws EvalException {
+ checkReadonly(varname, location);
+ if (parent == null) { // top-level values are immutable
+ readOnlyVariables.add(varname);
+ if (!futureReadOnlyVariables.isEmpty()) {
+ // Currently validating an if-else statement
+ futureReadOnlyVariables.peek().add(varname);
+ }
+ }
+ SkylarkType oldVartype = variableTypes.get(SkylarkType.GLOBAL).get(varname);
+ if (oldVartype != null) {
+ newVartype = oldVartype.infer(newVartype, "variable '" + varname + "'",
+ location, variableLocations.get(varname));
+ }
+ variableTypes.get(SkylarkType.GLOBAL).put(varname, newVartype);
+ variableLocations.put(varname, location);
+ clonable = false;
+ }
+
+ private void checkReadonly(String varname, Location location) throws EvalException {
+ if (readOnlyVariables.contains(varname)) {
+ throw new EvalException(location, String.format("Variable %s is read only", varname));
+ }
+ }
+
+ public void checkIterable(SkylarkType type, Location loc) throws EvalException {
+ if (type == SkylarkType.UNKNOWN) {
+ // Until all the language is properly typed, we ignore Object types.
+ return;
+ }
+ if (!Iterable.class.isAssignableFrom(type.getType())
+ && !Map.class.isAssignableFrom(type.getType())
+ && !String.class.equals(type.getType())) {
+ throw new EvalException(loc,
+ "type '" + EvalUtils.getDataTypeNameFromClass(type.getType()) + "' is not iterable");
+ }
+ }
+
+ /**
+ * Returns true if the symbol exists in the validation environment.
+ */
+ public boolean hasSymbolInEnvironment(String varname) {
+ return variableTypes.get(SkylarkType.GLOBAL).containsKey(varname)
+ || topLevel().variableTypes.get(SkylarkType.GLOBAL).containsKey(varname);
+ }
+
+ /**
+ * Returns the type of the existing variable.
+ */
+ public SkylarkType getVartype(String varname) {
+ SkylarkType type = variableTypes.get(SkylarkType.GLOBAL).get(varname);
+ if (type == null && parent != null) {
+ type = parent.getVartype(varname);
+ }
+ return Preconditions.checkNotNull(type,
+ String.format("Variable %s is not found in the validation environment", varname));
+ }
+
+ public SkylarkFunctionType getCurrentFunction() {
+ return currentFunction;
+ }
+
+ /**
+ * Returns the return type of the function.
+ */
+ public SkylarkType getReturnType(String funcName, Location loc) throws EvalException {
+ return getReturnType(SkylarkType.GLOBAL, funcName, loc);
+ }
+
+ /**
+ * Returns the return type of the object function.
+ */
+ public SkylarkType getReturnType(SkylarkType objectType, String funcName, Location loc)
+ throws EvalException {
+ // All functions are registered in the top level ValidationEnvironment.
+ Map<String, SkylarkType> functions = topLevel().variableTypes.get(objectType);
+ // TODO(bazel-team): eventually not finding the return type should be a validation error,
+ // because it means the function doesn't exist. First we have to make sure that we register
+ // every possible function before.
+ if (functions != null) {
+ SkylarkType functionType = functions.get(funcName);
+ if (functionType != null && functionType != SkylarkType.UNKNOWN) {
+ if (!(functionType instanceof SkylarkFunctionType)) {
+ throw new EvalException(loc, (objectType == SkylarkType.GLOBAL ? "" : objectType + ".")
+ + funcName + " is not a function");
+ }
+ return ((SkylarkFunctionType) functionType).getReturnType();
+ }
+ }
+ return SkylarkType.UNKNOWN;
+ }
+
+ private ValidationEnvironment topLevel() {
+ return Preconditions.checkNotNull(parent == null ? this : parent);
+ }
+
+ /**
+ * Adds a user defined function to the validation environment is not exists.
+ */
+ public void updateFunction(String name, SkylarkFunctionType type, Location loc)
+ throws EvalException {
+ checkReadonly(name, loc);
+ if (variableTypes.get(SkylarkType.GLOBAL).containsKey(name)) {
+ throw new EvalException(loc, "function " + name + " already exists");
+ }
+ variableTypes.get(SkylarkType.GLOBAL).put(name, type);
+ clonable = false;
+ }
+
+ /**
+ * Starts a session with temporarily disabled readonly checking for variables between branches.
+ * This is useful to validate control flows like if-else when we know that certain parts of the
+ * code cannot both be executed.
+ */
+ public void startTemporarilyDisableReadonlyCheckSession() {
+ futureReadOnlyVariables.add(new HashSet<String>());
+ clonable = false;
+ }
+
+ /**
+ * Finishes the session with temporarily disabled readonly checking.
+ */
+ public void finishTemporarilyDisableReadonlyCheckSession() {
+ Set<String> variables = futureReadOnlyVariables.pop();
+ readOnlyVariables.addAll(variables);
+ if (!futureReadOnlyVariables.isEmpty()) {
+ futureReadOnlyVariables.peek().addAll(variables);
+ }
+ clonable = false;
+ }
+
+ /**
+ * Finishes a branch of temporarily disabled readonly checking.
+ */
+ public void finishTemporarilyDisableReadonlyCheckBranch() {
+ readOnlyVariables.removeAll(futureReadOnlyVariables.peek());
+ clonable = false;
+ }
+}