Allow evaluation from String
Lift the Evaluation code from the test files AbstractParserTestCase and
AbstractEvaluationTestCase into new files EvaluationContext.
Remove this code's dependency on FsApparatus (and thus to InMemoryFS),
by making the Lexer accept null as filename.
Also remove dependency on EventCollectionApparatus;
parameterized by an EventHandler.
Have the SkylarkSignatureProcessor use this Evaluation for defaultValue-s.
While refactoring evaluation, have SkylarkShell use it,
which fixes its ValidationEnvironment issues.
TODO: refactor the tests to use this new infrastructure.
--
MOS_MIGRATED_REVID=90824736
diff --git a/src/main/java/com/google/devtools/build/lib/rules/SkylarkModules.java b/src/main/java/com/google/devtools/build/lib/rules/SkylarkModules.java
index 7d1e176..500edcc 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/SkylarkModules.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/SkylarkModules.java
@@ -22,6 +22,7 @@
import com.google.devtools.build.lib.packages.MethodLibrary;
import com.google.devtools.build.lib.packages.SkylarkNativeModule;
import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.EvaluationContext;
import com.google.devtools.build.lib.syntax.Function;
import com.google.devtools.build.lib.syntax.SkylarkBuiltin;
import com.google.devtools.build.lib.syntax.SkylarkEnvironment;
@@ -38,7 +39,7 @@
/**
* A class to handle all Skylark modules, to create and setup Validation and regular Environments.
*/
-// TODO(bazel-team): move that to syntax/ and
+// TODO(bazel-team): move that to the syntax package and
// let each extension register itself in a static { } statement.
public class SkylarkModules {
@@ -138,6 +139,11 @@
return new ValidationEnvironment(CollectionUtils.toImmutable(builtIn));
}
+ public static EvaluationContext newEvaluationContext(EventHandler eventHandler) {
+ return EvaluationContext.newSkylarkContext(
+ getNewEnvironment(eventHandler), getValidationEnvironment());
+ }
+
/**
* Collects the SkylarkFunctions from the fields of the class of the object parameter
* and adds them into the builder.
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Environment.java b/src/main/java/com/google/devtools/build/lib/syntax/Environment.java
index 456007f..bd59512 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/Environment.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Environment.java
@@ -135,14 +135,18 @@
this.eventHandler = Preconditions.checkNotNull(eventHandler);
}
+ public EventHandler getEventHandler() {
+ return eventHandler;
+ }
+
// Sets up the global environment
private void setupGlobal() {
// In Python 2.x, True and False are global values and can be redefined by the user.
// In Python 3.x, they are keywords. We implement them as values, for the sake of
// simplicity. We define them as Boolean objects.
- env.put("False", FALSE);
- env.put("True", TRUE);
- env.put("None", NONE);
+ update("False", FALSE);
+ update("True", TRUE);
+ update("None", NONE);
}
public boolean isSkylarkEnabled() {
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/EvaluationContext.java b/src/main/java/com/google/devtools/build/lib/syntax/EvaluationContext.java
new file mode 100644
index 0000000..c361bb3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/syntax/EvaluationContext.java
@@ -0,0 +1,223 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.syntax;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.packages.CachingPackageLocator;
+import com.google.devtools.build.lib.syntax.Environment.NoSuchVariableException;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/**
+ * Context for the evaluation of programs.
+ */
+public final class EvaluationContext {
+
+ @Nullable private EventHandler eventHandler;
+ private Environment env;
+ @Nullable private ValidationEnvironment validationEnv;
+ private boolean parsePython;
+
+ private EvaluationContext(EventHandler eventHandler, Environment env,
+ @Nullable ValidationEnvironment validationEnv, boolean parsePython) {
+ this.eventHandler = eventHandler;
+ this.env = env;
+ this.validationEnv = validationEnv;
+ this.parsePython = parsePython;
+ }
+
+ /**
+ * The fail fast handler, which throws a runtime exception whenever we encounter an error event.
+ */
+ public static final EventHandler FAIL_FAST_HANDLER = new EventHandler() {
+ @Override
+ public void handle(Event event) {
+ Preconditions.checkArgument(
+ !EventKind.ERRORS_AND_WARNINGS.contains(event.getKind()), event);
+ }
+ };
+
+ public static final EventHandler PRINT_HANDLER = new EventHandler() {
+ @Override
+ public void handle(Event event) {
+ System.out.print(event.getMessage());
+ }
+ };
+
+ public static EvaluationContext newBuildContext(EventHandler eventHandler, Environment env,
+ boolean parsePython) {
+ return new EvaluationContext(eventHandler, env, null, parsePython);
+ }
+
+ public static EvaluationContext newBuildContext(EventHandler eventHandler, Environment env) {
+ return newBuildContext(eventHandler, env, false);
+ }
+
+ public static EvaluationContext newBuildContext(EventHandler eventHandler) {
+ return newBuildContext(eventHandler, new Environment());
+ }
+
+ public static EvaluationContext newSkylarkContext(
+ Environment env, ValidationEnvironment validationEnv) {
+ return new EvaluationContext(env.getEventHandler(), env, validationEnv, false);
+ }
+
+ public static EvaluationContext newSkylarkContext(EventHandler eventHandler) {
+ return newSkylarkContext(new SkylarkEnvironment(eventHandler), new ValidationEnvironment());
+ }
+
+ public Environment getEnvironment() {
+ return env;
+ }
+
+ public EventHandler getEventHandler() {
+ return eventHandler;
+ }
+
+ public ValidationEnvironment getValidationEnvironment() {
+ return validationEnv;
+ }
+
+ /** Base context for Skylark evaluation for internal use only, while initializing builtins */
+ static final EvaluationContext SKYLARK_INITIALIZATION = newSkylarkContext(FAIL_FAST_HANDLER);
+
+ /** Mock package locator */
+ private static final class EmptyPackageLocator implements CachingPackageLocator {
+ @Override
+ public Path getBuildFileForPackage(String packageName) {
+ return null;
+ }
+ }
+
+ /** An empty package locator */
+ private static final CachingPackageLocator EMPTY_PACKAGE_LOCATOR = new EmptyPackageLocator();
+
+ /** Create a Lexer without a supporting file */
+ @VisibleForTesting
+ Lexer createLexer(String... input) {
+ return new Lexer(ParserInputSource.create(Joiner.on("\n").join(input), null),
+ eventHandler);
+ }
+
+ /** Is this a Skylark evaluation context? */
+ public boolean isSkylark() {
+ return env.isSkylarkEnabled();
+ }
+
+ /** Parse a string without a supporting file, returning statements and comments */
+ @VisibleForTesting
+ Parser.ParseResult parseFileWithComments(String... input) {
+ return isSkylark()
+ ? Parser.parseFileForSkylark(createLexer(input), eventHandler, null, validationEnv)
+ : Parser.parseFile(createLexer(input), eventHandler, EMPTY_PACKAGE_LOCATOR, parsePython);
+ }
+
+ /** Parse a string without a supporting file, returning statements only */
+ @VisibleForTesting
+ List<Statement> parseFile(String... input) {
+ return parseFileWithComments(input).statements;
+ }
+
+ /** Parse an Expression from string without a supporting file */
+ @VisibleForTesting
+ Expression parseExpression(String... input) {
+ return Parser.parseExpression(createLexer(input), eventHandler);
+ }
+
+ /** Evaluate an Expression */
+ @VisibleForTesting
+ Object evalExpression(Expression expression) throws EvalException, InterruptedException {
+ return expression.eval(env);
+ }
+
+ /** Evaluate an Expression as parsed from String-s */
+ Object evalExpression(String... input) throws EvalException, InterruptedException {
+ return evalExpression(parseExpression(input));
+ }
+
+ /** Parse a build (not Skylark) Statement from string without a supporting file */
+ @VisibleForTesting
+ Statement parseStatement(String... input) {
+ return Parser.parseStatement(createLexer(input), eventHandler);
+ }
+
+ /**
+ * Evaluate a Statement
+ * @param statement the Statement
+ * @return the value of the evaluation, if it's an Expression, or else null
+ */
+ @Nullable private Object eval(Statement statement) throws EvalException, InterruptedException {
+ if (statement instanceof ExpressionStatement) {
+ return evalExpression(((ExpressionStatement) statement).getExpression());
+ }
+ statement.exec(env);
+ return null;
+ }
+
+ /**
+ * Evaluate a list of Statement-s
+ * @return the value of the last statement if it's an Expression or else null
+ */
+ @Nullable private Object eval(List<Statement> statements)
+ throws EvalException, InterruptedException {
+ Object last = null;
+ for (Statement statement : statements) {
+ last = eval(statement);
+ }
+ return last;
+ }
+
+ /** Update a variable in the environment, in fluent style */
+ public EvaluationContext update(String varname, Object value) throws EvalException {
+ env.update(varname, value);
+ if (validationEnv != null) {
+ validationEnv.update(
+ varname, SkylarkType.typeForInference(SkylarkType.of(value.getClass())), null);
+ }
+ return this;
+ }
+
+ /** Lookup a variable in the environment */
+ public Object lookup(String varname) throws NoSuchVariableException {
+ return env.lookup(varname);
+ }
+
+ /** Print a String in this context */
+ public void print(String msg) {
+ if (msg != null) {
+ eventHandler.handle(new Event(EventKind.STDOUT, null, msg));
+ }
+ }
+
+ /** Print a String in this context */
+ public void println(String msg) {
+ if (msg != null) {
+ print(msg + "\n");
+ }
+ }
+
+ /** Evaluate a series of statements */
+ public Object eval(String... input) throws EvalException, InterruptedException {
+ return eval(parseFile(input));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
index fd68bde..86b8de7 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
@@ -164,7 +164,8 @@
@Override
public PathFragment getPath() {
- return lineNumberTable.getPath(getStartOffset()).asFragment();
+ Path path = lineNumberTable.getPath(getStartOffset());
+ return path != null ? path.asFragment() : null;
}
@Override
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Parser.java b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
index e8c30c3..75b8db8 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
@@ -212,7 +212,13 @@
@VisibleForTesting
public static Statement parseStatement(
Lexer lexer, EventHandler eventHandler) {
- return new Parser(lexer, eventHandler, null).parseSmallStatement();
+ Parser parser = new Parser(lexer, eventHandler, null);
+ Statement result = parser.parseSmallStatement();
+ while (parser.token.kind == TokenKind.NEWLINE) {
+ parser.nextToken();
+ }
+ parser.expect(TokenKind.EOF);
+ return result;
}
/**
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkSignatureProcessor.java b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkSignatureProcessor.java
index c772038..6d7b1db 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/SkylarkSignatureProcessor.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/SkylarkSignatureProcessor.java
@@ -82,15 +82,15 @@
paramList.add(getParameter(name, starParam, enforcedTypes, doc, undocumented,
/*mandatory=*/false, /*star=*/true, /*starStar=*/false, /*defaultValue=*/null));
}
+ for (Param param : annotation.mandatoryNamedOnly()) {
+ paramList.add(getParameter(name, param, enforcedTypes, doc, undocumented,
+ /*mandatory=*/true, /*star=*/false, /*starStar=*/false, /*defaultValue=*/null));
+ }
for (Param param : annotation.optionalNamedOnly()) {
paramList.add(getParameter(name, param, enforcedTypes, doc, undocumented,
/*mandatory=*/false, /*star=*/false, /*starStar=*/false,
/*defaultValue=*/getDefaultValue(param, defaultValuesIterator)));
}
- for (Param param : annotation.mandatoryNamedOnly()) {
- paramList.add(getParameter(name, param, enforcedTypes, doc, undocumented,
- /*mandatory=*/true, /*star=*/false, /*starStar=*/false, /*defaultValue=*/null));
- }
if (annotation.extraKeywords().length > 0) {
Preconditions.checkArgument(annotation.extraKeywords().length == 1);
paramList.add(
@@ -190,13 +190,11 @@
return Environment.NONE;
} else {
try {
- // TODO(bazel-team): when we have Evaluation, remove the throw and uncomment the return.
- throw new RuntimeException("Not Implemented Yet!");
- // return new Evaluation ().eval(param.defaultValue());
+ return EvaluationContext.SKYLARK_INITIALIZATION.evalExpression(param.defaultValue());
} catch (Exception e) {
throw new RuntimeException(String.format(
- "Exception while processing @SkylarkSignature.Param %s, default value %s: %s",
- param.name(), param.defaultValue(), e.getMessage()), e);
+ "Exception while processing @SkylarkSignature.Param %s, default value %s",
+ param.name(), param.defaultValue()), e);
}
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/SkylarkShell.java b/src/test/java/com/google/devtools/build/lib/syntax/SkylarkShell.java
index 6c394f9..96ebf02 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/SkylarkShell.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/SkylarkShell.java
@@ -13,18 +13,12 @@
// limitations under the License.
package com.google.devtools.build.lib.syntax;
-import com.google.common.collect.ImmutableMap;
-import com.google.devtools.build.lib.events.Event;
-import com.google.devtools.build.lib.events.EventHandler;
-import com.google.devtools.build.lib.events.util.EventCollectionApparatus;
-import com.google.devtools.build.lib.packages.CachingPackageLocator;
import com.google.devtools.build.lib.rules.SkylarkModules;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.util.FsApparatus;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.nio.charset.Charset;
/**
* SkylarkShell is a standalone shell executing Skylark. This is intended for
@@ -33,62 +27,48 @@
* bugs. Imports and includes are not supported.
*/
class SkylarkShell {
- static final EventCollectionApparatus syntaxEvents = new EventCollectionApparatus();
- static final FsApparatus scratch = FsApparatus.newInMemory();
- static final CachingPackageLocator locator = new AbstractParserTestCase.EmptyPackageLocator();
- static final Path path = scratch.path("stdin");
- private static void exec(String inputSource, Environment env) {
+ private static final String START_PROMPT = ">> ";
+ private static final String CONTINUATION_PROMPT = ".. ";
+
+ private final BufferedReader reader = new BufferedReader(
+ new InputStreamReader(System.in, Charset.defaultCharset()));
+ private final EvaluationContext ev =
+ SkylarkModules.newEvaluationContext(EvaluationContext.PRINT_HANDLER);
+
+ public String read() {
+ StringBuilder input = new StringBuilder();
+ ev.print(START_PROMPT);
try {
- ParserInputSource input = ParserInputSource.create(inputSource, path);
- Lexer lexer = new Lexer(input, syntaxEvents.reporter());
- Parser.ParseResult result =
- Parser.parseFileForSkylark(lexer, syntaxEvents.reporter(), locator,
- SkylarkModules.getValidationEnvironment(
- ImmutableMap.<String, SkylarkType>of()));
-
- Object last = null;
- for (Statement st : result.statements) {
- if (st instanceof ExpressionStatement) {
- last = ((ExpressionStatement) st).getExpression().eval(env);
- } else {
- st.exec(env);
- last = null;
+ while (true) {
+ String line = reader.readLine();
+ if (line == null) {
+ return null;
}
+ if (line.isEmpty()) {
+ return input.toString();
+ }
+ input.append("\n").append(line);
+ ev.print(CONTINUATION_PROMPT);
}
- if (last != null) {
- System.out.println(last);
+ } catch (IOException io) {
+ io.printStackTrace();
+ return null;
+ }
+ }
+
+ public void readEvalPrintLoop() {
+ String input;
+ while ((input = read()) != null) {
+ try {
+ ev.println(EvalUtils.prettyPrintValue(ev.eval(input)));
+ } catch (Exception e) {
+ e.printStackTrace();
}
- } catch (Throwable e) { // Catch everything to avoid killing the shell.
- e.printStackTrace();
}
}
public static void main(String[] args) {
- Environment env = SkylarkModules.getNewEnvironment(new EventHandler() {
- @Override
- public void handle(Event event) {
- System.out.println(event.getMessage());
- }
- });
- BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
-
- String currentInput = "";
- String line;
- System.out.print(">> ");
- try {
- while ((line = br.readLine()) != null) {
- if (line.isEmpty()) {
- exec(currentInput, env);
- currentInput = "";
- System.out.print(">> ");
- } else {
- currentInput = currentInput + "\n" + line;
- System.out.print(".. ");
- }
- }
- } catch (IOException io) {
- io.printStackTrace();
- }
+ new SkylarkShell().readEvalPrintLoop();
}
}
