src/test/java/com/google/devtools/build/lib/syntax/util/EvaluationTestCase.java - bazel - Git at Google

 // Copyright 2006 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package com.google.devtools.build.lib.syntax.util;

 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.Assert.fail;

 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.truth.Ordered;
 import com.google.devtools.build.lib.analysis.skylark.BazelStarlarkContext;
 import com.google.devtools.build.lib.analysis.skylark.SymbolGenerator;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.EventCollector;
 import com.google.devtools.build.lib.events.EventKind;
 import com.google.devtools.build.lib.events.ExtendedEventHandler;
 import com.google.devtools.build.lib.events.util.EventCollectionApparatus;
 import com.google.devtools.build.lib.packages.BazelLibrary;
 import com.google.devtools.build.lib.syntax.BuildFileAST;
 import com.google.devtools.build.lib.syntax.Environment;
 import com.google.devtools.build.lib.syntax.Environment.FailFastException;
 import com.google.devtools.build.lib.syntax.EvalException;
 import com.google.devtools.build.lib.syntax.Expression;
 import com.google.devtools.build.lib.syntax.Mutability;
 import com.google.devtools.build.lib.syntax.Parser;
 import com.google.devtools.build.lib.syntax.ParserInputSource;
 import com.google.devtools.build.lib.syntax.SkylarkUtils;
 import com.google.devtools.build.lib.syntax.SkylarkUtils.Phase;
 import com.google.devtools.build.lib.syntax.Statement;
 import com.google.devtools.build.lib.syntax.ValidationEnvironment;
 import com.google.devtools.build.lib.testutil.TestConstants;
 import com.google.devtools.build.lib.testutil.TestMode;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import org.junit.Before;

 /**
  * Base class for test cases that use parsing and evaluation services.
  */
 public class EvaluationTestCase {
   private EventCollectionApparatus eventCollectionApparatus =
       new EventCollectionApparatus(EventKind.ALL_EVENTS);
   private TestMode testMode = TestMode.SKYLARK;
   protected Environment env;
   protected Mutability mutability = Mutability.create("test");

   @Before
   public final void initialize() throws Exception {
     env = newEnvironment();
   }

   /**
    * Creates a standard Environment for tests in the BUILD language.
    * No PythonPreprocessing, mostly empty mutable Environment.
    */
   public Environment newBuildEnvironment() {
     BazelStarlarkContext context =
         new BazelStarlarkContext(
             TestConstants.TOOLS_REPOSITORY,
             /* repoMapping= */ ImmutableMap.of(),
             new SymbolGenerator<>(new Object()));
     Environment env =
         Environment.builder(mutability)
             .useDefaultSemantics()
             .setGlobals(BazelLibrary.GLOBALS)
             .setEventHandler(getEventHandler())
             .setStarlarkContext(context)
             .build();
     SkylarkUtils.setPhase(env, Phase.LOADING);
     return env;
   }

   /**
    * Creates an Environment for Skylark with a mostly empty initial environment.
    * For internal initialization or tests.
    */
   public Environment newSkylarkEnvironment() {
     return Environment.builder(mutability)
         .useDefaultSemantics()
         .setGlobals(BazelLibrary.GLOBALS)
         .setEventHandler(getEventHandler())
         .build();
   }

   /**
    * Creates a new Environment suitable for the test case. Subclasses may override it to fit their
    * purpose and e.g. call newBuildEnvironment or newSkylarkEnvironment; or they may play with the
    * testMode to run tests in either or both kinds of Environment. Note that all Environment-s may
    * share the same Mutability, so don't close it.
    *
    * @return a fresh Environment.
    */
   public Environment newEnvironment() throws Exception {
     return newEnvironmentWithSkylarkOptions();
   }

   protected Environment newEnvironmentWithSkylarkOptions(String... skylarkOptions)
       throws Exception {
     return newEnvironmentWithBuiltinsAndSkylarkOptions(ImmutableMap.of(), skylarkOptions);
   }

   protected Environment newEnvironmentWithBuiltinsAndSkylarkOptions(Map<String, Object> builtins,
       String... skylarkOptions) throws Exception {
     if (testMode == null) {
       throw new IllegalArgumentException(
           "TestMode is null. Please set a Testmode via setMode() or set the "
               + "Environment manually by overriding newEnvironment()");
     }
     return testMode.createEnvironment(getEventHandler(), builtins, skylarkOptions);
   }

   /**
    * Sets the specified {@code TestMode} and tries to create the appropriate {@code Environment}
    *
    * @param testMode
    * @throws Exception
    */
   protected void setMode(TestMode testMode, String... skylarkOptions) throws Exception {
     this.testMode = testMode;
     env = newEnvironmentWithSkylarkOptions(skylarkOptions);
   }

   protected void setMode(TestMode testMode, Map<String, Object> builtins,
       String... skylarkOptions) throws Exception {
     this.testMode = testMode;
     env = newEnvironmentWithBuiltinsAndSkylarkOptions(builtins, skylarkOptions);
   }

   protected void enableSkylarkMode(Map<String, Object> builtins,
       String... skylarkOptions) throws Exception {
     setMode(TestMode.SKYLARK, builtins, skylarkOptions);
   }

   protected void enableSkylarkMode(String... skylarkOptions) throws Exception {
     setMode(TestMode.SKYLARK, skylarkOptions);
   }

   protected void enableBuildMode(String... skylarkOptions) throws Exception {
     setMode(TestMode.BUILD, skylarkOptions);
   }

   public ExtendedEventHandler getEventHandler() {
     return eventCollectionApparatus.reporter();
   }

   public Environment getEnvironment() {
     return env;
   }

   protected BuildFileAST parseBuildFileASTWithoutValidation(String... input) {
     return BuildFileAST.parseString(getEventHandler(), input);
   }

   protected BuildFileAST parseBuildFileAST(String... input) {
     BuildFileAST ast = parseBuildFileASTWithoutValidation(input);
     return ast.validate(env, getEventHandler());
   }

   protected List<Statement> parseFile(String... input) {
     return parseBuildFileAST(input).getStatements();
   }

   /** Construct a ParserInputSource by concatenating multiple strings with newlines. */
   private ParserInputSource makeParserInputSource(String... input) {
     return ParserInputSource.create(Joiner.on("\n").join(input), null);
   }

   /** Parses a statement, possibly followed by newlines. */
   protected Statement parseStatement(Parser.ParsingLevel parsingLevel, String... input) {
     return Parser.parseStatement(makeParserInputSource(input), getEventHandler(), parsingLevel);
   }

   /** Parses an expression, possibly followed by newlines. */
   protected Expression parseExpression(String... input) {
     return Parser.parseExpression(makeParserInputSource(input), getEventHandler());
   }

   public EvaluationTestCase update(String varname, Object value) throws Exception {
     env.update(varname, value);
     return this;
   }

   public Object lookup(String varname) throws Exception {
     return env.moduleLookup(varname);
   }

   public Object eval(String... input) throws Exception {
     if (testMode == TestMode.SKYLARK) {
       return BuildFileAST.eval(env, input);
     }
     BuildFileAST ast = BuildFileAST.parseString(env.getEventHandler(), input);
     ValidationEnvironment.checkBuildSyntax(ast.getStatements(), env.getEventHandler(), env);
     return ast.eval(env);
   }

   public void checkEvalError(String msg, String... input) throws Exception {
     try {
       eval(input);
       fail("Expected error '" + msg + "' but got no error");
     } catch (EvalException | FailFastException e) {
       assertThat(e).hasMessageThat().isEqualTo(msg);
     }
   }

   public void checkEvalErrorContains(String msg, String... input) throws Exception {
     try {
       eval(input);
       fail("Expected error containing '" + msg + "' but got no error");
     } catch (EvalException | FailFastException e) {
       assertThat(e).hasMessageThat().contains(msg);
     }
   }

   public void checkEvalErrorDoesNotContain(String msg, String... input) throws Exception {
     try {
       eval(input);
     } catch (EvalException | FailFastException e) {
       assertThat(e).hasMessageThat().doesNotContain(msg);
     }
   }

   // Forward relevant methods to the EventCollectionApparatus
   public EvaluationTestCase setFailFast(boolean failFast) {
     eventCollectionApparatus.setFailFast(failFast);
     return this;
   }

   public EvaluationTestCase assertNoWarningsOrErrors() {
     eventCollectionApparatus.assertNoWarningsOrErrors();
     return this;
   }

   public EventCollector getEventCollector() {
     return eventCollectionApparatus.collector();
   }

   public Event assertContainsError(String expectedMessage) {
     return eventCollectionApparatus.assertContainsError(expectedMessage);
   }

   public Event assertContainsWarning(String expectedMessage) {
     return eventCollectionApparatus.assertContainsWarning(expectedMessage);
   }

   public Event assertContainsDebug(String expectedMessage) {
     return eventCollectionApparatus.assertContainsDebug(expectedMessage);
   }

   public EvaluationTestCase clearEvents() {
     eventCollectionApparatus.clear();
     return this;
   }

   /**
    * Encapsulates a separate test which can be executed by a {@code TestMode}
    */
   protected interface Testable {
     public void run() throws Exception;
   }

   /**
    * Base class for test cases that run in specific modes (e.g. Build and/or Skylark)
    */
   protected abstract class ModalTestCase {
     private final SetupActions setup;

     protected ModalTestCase() {
       setup = new SetupActions();
     }

     /**
      * Allows the execution of several statements before each following test
      * @param statements The statement(s) to be executed
      * @return This {@code ModalTestCase}
      */
     public ModalTestCase setUp(String... statements) {
       setup.registerEval(statements);
       return this;
     }

     /**
      * Allows the update of the specified variable before each following test
      * @param name The name of the variable that should be updated
      * @param value The new value of the variable
      * @return This {@code ModalTestCase}
      */
     public ModalTestCase update(String name, Object value) {
       setup.registerUpdate(name, value);
       return this;
     }

     /**
      * Evaluates two parameters and compares their results.
      * @param statement The statement to be evaluated
      * @param expectedEvalString The expression of the expected result
      * @return This {@code ModalTestCase}
      * @throws Exception
      */
     public ModalTestCase testEval(String statement, String expectedEvalString) throws Exception {
       runTest(createComparisonTestable(statement, expectedEvalString, true));
       return this;
     }

     /**
      * Evaluates the given statement and compares its result to the expected object
      * @param statement
      * @param expected
      * @return This {@code ModalTestCase}
      * @throws Exception
      */
     public ModalTestCase testStatement(String statement, Object expected) throws Exception {
       runTest(createComparisonTestable(statement, expected, false));
       return this;
     }

     /**
      * Evaluates the given statement and compares its result to the collection of expected objects
      * without considering their order
      * @param statement The statement to be evaluated
      * @param items The expected items
      * @return This {@code ModalTestCase}
      * @throws Exception
      */
     public ModalTestCase testCollection(String statement, Object... items) throws Exception {
       runTest(collectionTestable(statement, false, items));
       return this;
     }

     /**
      * Evaluates the given statement and compares its result to the collection of expected objects
      * while considering their order
      * @param statement The statement to be evaluated
      * @param items The expected items, in order
      * @return This {@code ModalTestCase}
      * @throws Exception
      */
     public ModalTestCase testExactOrder(String statement, Object... items) throws Exception {
       runTest(collectionTestable(statement, true, items));
       return this;
     }

     /**
      * Evaluates the given statement and checks whether the given error message appears
      * @param expectedError The expected error message
      * @param statements The statement(s) to be evaluated
      * @return This ModalTestCase
      * @throws Exception
      */
     public ModalTestCase testIfExactError(String expectedError, String... statements)
         throws Exception {
       runTest(errorTestable(true, expectedError, statements));
       return this;
     }

     /**
      * Evaluates the given statement and checks whether an error that contains the expected message
      * occurs
      * @param expectedError
      * @param statements
      * @return This ModalTestCase
      * @throws Exception
      */
     public ModalTestCase testIfErrorContains(String expectedError, String... statements)
         throws Exception {
       runTest(errorTestable(false, expectedError, statements));
       return this;
     }

     /**
      * Looks up the value of the specified variable and compares it to the expected value
      * @param name
      * @param expected
      * @return This ModalTestCase
      * @throws Exception
      */
     public ModalTestCase testLookup(String name, Object expected) throws Exception {
       runTest(createLookUpTestable(name, expected));
       return this;
     }

     /**
      * Creates a Testable that checks whether the evaluation of the given statement leads to the
      * expected error
      * @param statements
      * @param error
      * @param exactMatch If true, the error message has to be identical to the expected error
      * @return An instance of Testable that runs the error check
      */
     protected Testable errorTestable(
         final boolean exactMatch, final String error, final String... statements) {
       return new Testable() {
         @Override
         public void run() throws Exception {
           if (exactMatch) {
             checkEvalError(error, statements);
           } else {
             checkEvalErrorContains(error, statements);
           }
         }
       };
     }

     /**
      * Creates a testable that checks whether the evaluation of the given statement leads to a list
      * that contains exactly the expected objects
      * @param statement The statement to be evaluated
      * @param ordered Determines whether the order of the elements is checked as well
      * @param expected Expected objects
      * @return An instance of Testable that runs the check
      */
     protected Testable collectionTestable(
         final String statement, final boolean ordered, final Object... expected) {
       return new Testable() {
         @Override
         public void run() throws Exception {
           Ordered tmp = assertThat((Iterable<?>) eval(statement)).containsExactly(expected);

           if (ordered) {
             tmp.inOrder();
           }
         }
       };
     }

     /**
      * Creates a testable that compares the evaluation of the given statement to a specified result
      *
      * @param statement The statement to be evaluated
      * @param expected Either the expected object or an expression whose evaluation leads to the
      *  expected object
      * @param expectedIsExpression Signals whether {@code expected} is an object or an expression
      * @return An instance of Testable that runs the comparison
      */
     protected Testable createComparisonTestable(
         final String statement, final Object expected, final boolean expectedIsExpression) {
       return new Testable() {
         @Override
         public void run() throws Exception {
           Object actual = eval(statement);
           Object realExpected = expected;

           // We could also print the actual object and compare the string to the expected
           // expression, but then the order of elements would matter.
           if (expectedIsExpression) {
             realExpected = eval((String) expected);
           }

           assertThat(actual).isEqualTo(realExpected);
         }
       };
     }

     /**
      * Creates a Testable that looks up the given variable and compares its value to the expected
      * value
      * @param name
      * @param expected
      * @return An instance of Testable that does both lookup and comparison
      */
     protected Testable createLookUpTestable(final String name, final Object expected) {
       return new Testable() {
         @Override
         public void run() throws Exception {
           assertThat(lookup(name)).isEqualTo(expected);
         }
       };
     }

     /**
      * Executes the given Testable
      * @param testable
      * @throws Exception
      */
     protected void runTest(Testable testable) throws Exception {
       run(new TestableDecorator(setup, testable));
     }

     protected abstract void run(Testable testable) throws Exception;
   }

   /**
    * A simple decorator that allows the execution of setup actions before running a {@code Testable}
    */
   static class TestableDecorator implements Testable {
     private final SetupActions setup;
     private final Testable decorated;

     public TestableDecorator(SetupActions setup, Testable decorated) {
       this.setup = setup;
       this.decorated = decorated;
     }

     /**
      * Executes all stored actions and updates plus the actual {@code Testable}
      */
     @Override
     public void run() throws Exception {
       setup.executeAll();
       decorated.run();
     }
   }

   /**
    * A container for collection actions that should be executed before a test
    */
   class SetupActions {
     private List<Testable> setup;

     public SetupActions() {
       setup = new LinkedList<>();
     }

     /**
      * Registers a variable that has to be updated before a test
      *
      * @param name
      * @param value
      */
     public void registerUpdate(final String name, final Object value) {
       setup.add(
           new Testable() {
             @Override
             public void run() throws Exception {
               EvaluationTestCase.this.update(name, value);
             }
           });
     }

     /**
      * Registers a statement for evaluation prior to a test
      *
      * @param statements
      */
     public void registerEval(final String... statements) {
       setup.add(
           new Testable() {
             @Override
             public void run() throws Exception {
               EvaluationTestCase.this.eval(statements);
             }
           });
     }

     /**
      * Executes all stored actions and updates
      * @throws Exception
      */
     public void executeAll() throws Exception {
       for (Testable testable : setup) {
         testable.run();
       }
     }
   }

   /**
    * A class that executes each separate test in both modes (Build and Skylark)
    */
   protected class BothModesTest extends ModalTestCase {
     private final String[] skylarkOptions;

     public BothModesTest(String... skylarkOptions) {
       this.skylarkOptions = skylarkOptions;
     }

     /**
      * Executes the given Testable in both Build and Skylark mode
      */
     @Override
     protected void run(Testable testable) throws Exception {
       enableSkylarkMode(skylarkOptions);
       try {
         testable.run();
       } catch (Exception e) {
         throw new Exception("While in Skylark mode", e);
       }

       enableBuildMode(skylarkOptions);
       try {
         testable.run();
       } catch (Exception e) {
         throw new Exception("While in Build mode", e);
       }
     }
   }

   /**
    * A class that runs all tests in Build mode
    */
   protected class BuildTest extends ModalTestCase {
     private final String[] skylarkOptions;

     public BuildTest(String... skylarkOptions) {
       this.skylarkOptions = skylarkOptions;
     }

     @Override
     protected void run(Testable testable) throws Exception {
       enableBuildMode(skylarkOptions);
       testable.run();
     }
   }

   /**
    * A class that runs all tests in Skylark mode
    */
   protected class SkylarkTest extends ModalTestCase {
     private final String[] skylarkOptions;
     private final Map<String, Object> builtins;

     public SkylarkTest(String... skylarkOptions) {
       this(ImmutableMap.of(), skylarkOptions);
     }

     public SkylarkTest(Map<String, Object> builtins, String... skylarkOptions) {
       this.builtins = builtins;
       this.skylarkOptions = skylarkOptions;
     }

     @Override
     protected void run(Testable testable) throws Exception {
       enableSkylarkMode(builtins, skylarkOptions);
       testable.run();
     }
   }
 }
	// Copyright 2006 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.
	package com.google.devtools.build.lib.syntax.util;

	import static com.google.common.truth.Truth.assertThat;
	import static org.junit.Assert.fail;

	import com.google.common.base.Joiner;
	import com.google.common.collect.ImmutableMap;
	import com.google.common.truth.Ordered;
	import com.google.devtools.build.lib.analysis.skylark.BazelStarlarkContext;
	import com.google.devtools.build.lib.analysis.skylark.SymbolGenerator;
	import com.google.devtools.build.lib.events.Event;
	import com.google.devtools.build.lib.events.EventCollector;
	import com.google.devtools.build.lib.events.EventKind;
	import com.google.devtools.build.lib.events.ExtendedEventHandler;
	import com.google.devtools.build.lib.events.util.EventCollectionApparatus;
	import com.google.devtools.build.lib.packages.BazelLibrary;
	import com.google.devtools.build.lib.syntax.BuildFileAST;
	import com.google.devtools.build.lib.syntax.Environment;
	import com.google.devtools.build.lib.syntax.Environment.FailFastException;
	import com.google.devtools.build.lib.syntax.EvalException;
	import com.google.devtools.build.lib.syntax.Expression;
	import com.google.devtools.build.lib.syntax.Mutability;
	import com.google.devtools.build.lib.syntax.Parser;
	import com.google.devtools.build.lib.syntax.ParserInputSource;
	import com.google.devtools.build.lib.syntax.SkylarkUtils;
	import com.google.devtools.build.lib.syntax.SkylarkUtils.Phase;
	import com.google.devtools.build.lib.syntax.Statement;
	import com.google.devtools.build.lib.syntax.ValidationEnvironment;
	import com.google.devtools.build.lib.testutil.TestConstants;
	import com.google.devtools.build.lib.testutil.TestMode;
	import java.util.LinkedList;
	import java.util.List;
	import java.util.Map;
	import org.junit.Before;

	/**
	* Base class for test cases that use parsing and evaluation services.
	*/
	public class EvaluationTestCase {
	private EventCollectionApparatus eventCollectionApparatus =
	new EventCollectionApparatus(EventKind.ALL_EVENTS);
	private TestMode testMode = TestMode.SKYLARK;
	protected Environment env;
	protected Mutability mutability = Mutability.create("test");

	@Before
	public final void initialize() throws Exception {
	env = newEnvironment();
	}

	/**
	* Creates a standard Environment for tests in the BUILD language.
	* No PythonPreprocessing, mostly empty mutable Environment.
	*/
	public Environment newBuildEnvironment() {
	BazelStarlarkContext context =
	new BazelStarlarkContext(
	TestConstants.TOOLS_REPOSITORY,
	/* repoMapping= */ ImmutableMap.of(),
	new SymbolGenerator<>(new Object()));
	Environment env =
	Environment.builder(mutability)
	.useDefaultSemantics()
	.setGlobals(BazelLibrary.GLOBALS)
	.setEventHandler(getEventHandler())
	.setStarlarkContext(context)
	.build();
	SkylarkUtils.setPhase(env, Phase.LOADING);
	return env;
	}

	/**
	* Creates an Environment for Skylark with a mostly empty initial environment.
	* For internal initialization or tests.
	*/
	public Environment newSkylarkEnvironment() {
	return Environment.builder(mutability)
	.useDefaultSemantics()
	.setGlobals(BazelLibrary.GLOBALS)
	.setEventHandler(getEventHandler())
	.build();
	}

	/**
	* Creates a new Environment suitable for the test case. Subclasses may override it to fit their
	* purpose and e.g. call newBuildEnvironment or newSkylarkEnvironment; or they may play with the
	* testMode to run tests in either or both kinds of Environment. Note that all Environment-s may
	* share the same Mutability, so don't close it.
	*
	* @return a fresh Environment.
	*/
	public Environment newEnvironment() throws Exception {
	return newEnvironmentWithSkylarkOptions();
	}

	protected Environment newEnvironmentWithSkylarkOptions(String... skylarkOptions)
	throws Exception {
	return newEnvironmentWithBuiltinsAndSkylarkOptions(ImmutableMap.of(), skylarkOptions);
	}

	protected Environment newEnvironmentWithBuiltinsAndSkylarkOptions(Map<String, Object> builtins,
	String... skylarkOptions) throws Exception {
	if (testMode == null) {
	throw new IllegalArgumentException(
	"TestMode is null. Please set a Testmode via setMode() or set the "
	+ "Environment manually by overriding newEnvironment()");
	}
	return testMode.createEnvironment(getEventHandler(), builtins, skylarkOptions);
	}

	/**
	* Sets the specified {@code TestMode} and tries to create the appropriate {@code Environment}
	*
	* @param testMode
	* @throws Exception
	*/
	protected void setMode(TestMode testMode, String... skylarkOptions) throws Exception {
	this.testMode = testMode;
	env = newEnvironmentWithSkylarkOptions(skylarkOptions);
	}

	protected void setMode(TestMode testMode, Map<String, Object> builtins,
	String... skylarkOptions) throws Exception {
	this.testMode = testMode;
	env = newEnvironmentWithBuiltinsAndSkylarkOptions(builtins, skylarkOptions);
	}

	protected void enableSkylarkMode(Map<String, Object> builtins,
	String... skylarkOptions) throws Exception {
	setMode(TestMode.SKYLARK, builtins, skylarkOptions);
	}

	protected void enableSkylarkMode(String... skylarkOptions) throws Exception {
	setMode(TestMode.SKYLARK, skylarkOptions);
	}

	protected void enableBuildMode(String... skylarkOptions) throws Exception {
	setMode(TestMode.BUILD, skylarkOptions);
	}

	public ExtendedEventHandler getEventHandler() {
	return eventCollectionApparatus.reporter();
	}

	public Environment getEnvironment() {
	return env;
	}

	protected BuildFileAST parseBuildFileASTWithoutValidation(String... input) {
	return BuildFileAST.parseString(getEventHandler(), input);
	}

	protected BuildFileAST parseBuildFileAST(String... input) {
	BuildFileAST ast = parseBuildFileASTWithoutValidation(input);
	return ast.validate(env, getEventHandler());
	}

	protected List<Statement> parseFile(String... input) {
	return parseBuildFileAST(input).getStatements();
	}

	/** Construct a ParserInputSource by concatenating multiple strings with newlines. */
	private ParserInputSource makeParserInputSource(String... input) {
	return ParserInputSource.create(Joiner.on("\n").join(input), null);
	}

	/** Parses a statement, possibly followed by newlines. */
	protected Statement parseStatement(Parser.ParsingLevel parsingLevel, String... input) {
	return Parser.parseStatement(makeParserInputSource(input), getEventHandler(), parsingLevel);
	}

	/** Parses an expression, possibly followed by newlines. */
	protected Expression parseExpression(String... input) {
	return Parser.parseExpression(makeParserInputSource(input), getEventHandler());
	}

	public EvaluationTestCase update(String varname, Object value) throws Exception {
	env.update(varname, value);
	return this;
	}

	public Object lookup(String varname) throws Exception {
	return env.moduleLookup(varname);
	}

	public Object eval(String... input) throws Exception {
	if (testMode == TestMode.SKYLARK) {
	return BuildFileAST.eval(env, input);
	}
	BuildFileAST ast = BuildFileAST.parseString(env.getEventHandler(), input);
	ValidationEnvironment.checkBuildSyntax(ast.getStatements(), env.getEventHandler(), env);
	return ast.eval(env);
	}

	public void checkEvalError(String msg, String... input) throws Exception {
	try {
	eval(input);
	fail("Expected error '" + msg + "' but got no error");
	} catch (EvalException \| FailFastException e) {
	assertThat(e).hasMessageThat().isEqualTo(msg);
	}
	}

	public void checkEvalErrorContains(String msg, String... input) throws Exception {
	try {
	eval(input);
	fail("Expected error containing '" + msg + "' but got no error");
	} catch (EvalException \| FailFastException e) {
	assertThat(e).hasMessageThat().contains(msg);
	}
	}

	public void checkEvalErrorDoesNotContain(String msg, String... input) throws Exception {
	try {
	eval(input);
	} catch (EvalException \| FailFastException e) {
	assertThat(e).hasMessageThat().doesNotContain(msg);
	}
	}

	// Forward relevant methods to the EventCollectionApparatus
	public EvaluationTestCase setFailFast(boolean failFast) {
	eventCollectionApparatus.setFailFast(failFast);
	return this;
	}

	public EvaluationTestCase assertNoWarningsOrErrors() {
	eventCollectionApparatus.assertNoWarningsOrErrors();
	return this;
	}

	public EventCollector getEventCollector() {
	return eventCollectionApparatus.collector();
	}

	public Event assertContainsError(String expectedMessage) {
	return eventCollectionApparatus.assertContainsError(expectedMessage);
	}

	public Event assertContainsWarning(String expectedMessage) {
	return eventCollectionApparatus.assertContainsWarning(expectedMessage);
	}

	public Event assertContainsDebug(String expectedMessage) {
	return eventCollectionApparatus.assertContainsDebug(expectedMessage);
	}

	public EvaluationTestCase clearEvents() {
	eventCollectionApparatus.clear();
	return this;
	}

	/**
	* Encapsulates a separate test which can be executed by a {@code TestMode}
	*/
	protected interface Testable {
	public void run() throws Exception;
	}

	/**
	* Base class for test cases that run in specific modes (e.g. Build and/or Skylark)
	*/
	protected abstract class ModalTestCase {
	private final SetupActions setup;

	protected ModalTestCase() {
	setup = new SetupActions();
	}

	/**
	* Allows the execution of several statements before each following test
	* @param statements The statement(s) to be executed
	* @return This {@code ModalTestCase}
	*/
	public ModalTestCase setUp(String... statements) {
	setup.registerEval(statements);
	return this;
	}

	/**
	* Allows the update of the specified variable before each following test
	* @param name The name of the variable that should be updated
	* @param value The new value of the variable
	* @return This {@code ModalTestCase}
	*/
	public ModalTestCase update(String name, Object value) {
	setup.registerUpdate(name, value);
	return this;
	}

	/**
	* Evaluates two parameters and compares their results.
	* @param statement The statement to be evaluated
	* @param expectedEvalString The expression of the expected result
	* @return This {@code ModalTestCase}
	* @throws Exception
	*/
	public ModalTestCase testEval(String statement, String expectedEvalString) throws Exception {
	runTest(createComparisonTestable(statement, expectedEvalString, true));
	return this;
	}

	/**
	* Evaluates the given statement and compares its result to the expected object
	* @param statement
	* @param expected
	* @return This {@code ModalTestCase}
	* @throws Exception
	*/
	public ModalTestCase testStatement(String statement, Object expected) throws Exception {
	runTest(createComparisonTestable(statement, expected, false));
	return this;
	}

	/**
	* Evaluates the given statement and compares its result to the collection of expected objects
	* without considering their order
	* @param statement The statement to be evaluated
	* @param items The expected items
	* @return This {@code ModalTestCase}
	* @throws Exception
	*/
	public ModalTestCase testCollection(String statement, Object... items) throws Exception {
	runTest(collectionTestable(statement, false, items));
	return this;
	}

	/**
	* Evaluates the given statement and compares its result to the collection of expected objects
	* while considering their order
	* @param statement The statement to be evaluated
	* @param items The expected items, in order
	* @return This {@code ModalTestCase}
	* @throws Exception
	*/
	public ModalTestCase testExactOrder(String statement, Object... items) throws Exception {
	runTest(collectionTestable(statement, true, items));
	return this;
	}

	/**
	* Evaluates the given statement and checks whether the given error message appears
	* @param expectedError The expected error message
	* @param statements The statement(s) to be evaluated
	* @return This ModalTestCase
	* @throws Exception
	*/
	public ModalTestCase testIfExactError(String expectedError, String... statements)
	throws Exception {
	runTest(errorTestable(true, expectedError, statements));
	return this;
	}

	/**
	* Evaluates the given statement and checks whether an error that contains the expected message
	* occurs
	* @param expectedError
	* @param statements
	* @return This ModalTestCase
	* @throws Exception
	*/
	public ModalTestCase testIfErrorContains(String expectedError, String... statements)
	throws Exception {
	runTest(errorTestable(false, expectedError, statements));
	return this;
	}

	/**
	* Looks up the value of the specified variable and compares it to the expected value
	* @param name
	* @param expected
	* @return This ModalTestCase
	* @throws Exception
	*/
	public ModalTestCase testLookup(String name, Object expected) throws Exception {
	runTest(createLookUpTestable(name, expected));
	return this;
	}

	/**
	* Creates a Testable that checks whether the evaluation of the given statement leads to the
	* expected error
	* @param statements
	* @param error
	* @param exactMatch If true, the error message has to be identical to the expected error
	* @return An instance of Testable that runs the error check
	*/
	protected Testable errorTestable(
	final boolean exactMatch, final String error, final String... statements) {
	return new Testable() {
	@Override
	public void run() throws Exception {
	if (exactMatch) {
	checkEvalError(error, statements);
	} else {
	checkEvalErrorContains(error, statements);
	}
	}
	};
	}

	/**
	* Creates a testable that checks whether the evaluation of the given statement leads to a list
	* that contains exactly the expected objects
	* @param statement The statement to be evaluated
	* @param ordered Determines whether the order of the elements is checked as well
	* @param expected Expected objects
	* @return An instance of Testable that runs the check
	*/
	protected Testable collectionTestable(
	final String statement, final boolean ordered, final Object... expected) {
	return new Testable() {
	@Override
	public void run() throws Exception {
	Ordered tmp = assertThat((Iterable<?>) eval(statement)).containsExactly(expected);

	if (ordered) {
	tmp.inOrder();
	}
	}
	};
	}

	/**
	* Creates a testable that compares the evaluation of the given statement to a specified result
	*
	* @param statement The statement to be evaluated
	* @param expected Either the expected object or an expression whose evaluation leads to the
	* expected object
	* @param expectedIsExpression Signals whether {@code expected} is an object or an expression
	* @return An instance of Testable that runs the comparison
	*/
	protected Testable createComparisonTestable(
	final String statement, final Object expected, final boolean expectedIsExpression) {
	return new Testable() {
	@Override
	public void run() throws Exception {
	Object actual = eval(statement);
	Object realExpected = expected;

	// We could also print the actual object and compare the string to the expected
	// expression, but then the order of elements would matter.
	if (expectedIsExpression) {
	realExpected = eval((String) expected);
	}

	assertThat(actual).isEqualTo(realExpected);
	}
	};
	}

	/**
	* Creates a Testable that looks up the given variable and compares its value to the expected
	* value
	* @param name
	* @param expected
	* @return An instance of Testable that does both lookup and comparison
	*/
	protected Testable createLookUpTestable(final String name, final Object expected) {
	return new Testable() {
	@Override
	public void run() throws Exception {
	assertThat(lookup(name)).isEqualTo(expected);
	}
	};
	}

	/**
	* Executes the given Testable
	* @param testable
	* @throws Exception
	*/
	protected void runTest(Testable testable) throws Exception {
	run(new TestableDecorator(setup, testable));
	}

	protected abstract void run(Testable testable) throws Exception;
	}

	/**
	* A simple decorator that allows the execution of setup actions before running a {@code Testable}
	*/
	static class TestableDecorator implements Testable {
	private final SetupActions setup;
	private final Testable decorated;

	public TestableDecorator(SetupActions setup, Testable decorated) {
	this.setup = setup;
	this.decorated = decorated;
	}

	/**
	* Executes all stored actions and updates plus the actual {@code Testable}
	*/
	@Override
	public void run() throws Exception {
	setup.executeAll();
	decorated.run();
	}
	}

	/**
	* A container for collection actions that should be executed before a test
	*/
	class SetupActions {
	private List<Testable> setup;

	public SetupActions() {
	setup = new LinkedList<>();
	}

	/**
	* Registers a variable that has to be updated before a test
	*
	* @param name
	* @param value
	*/
	public void registerUpdate(final String name, final Object value) {
	setup.add(
	new Testable() {
	@Override
	public void run() throws Exception {
	EvaluationTestCase.this.update(name, value);
	}
	});
	}

	/**
	* Registers a statement for evaluation prior to a test
	*
	* @param statements
	*/
	public void registerEval(final String... statements) {
	setup.add(
	new Testable() {
	@Override
	public void run() throws Exception {
	EvaluationTestCase.this.eval(statements);
	}
	});
	}

	/**
	* Executes all stored actions and updates
	* @throws Exception
	*/
	public void executeAll() throws Exception {
	for (Testable testable : setup) {
	testable.run();
	}
	}
	}

	/**
	* A class that executes each separate test in both modes (Build and Skylark)
	*/
	protected class BothModesTest extends ModalTestCase {
	private final String[] skylarkOptions;

	public BothModesTest(String... skylarkOptions) {
	this.skylarkOptions = skylarkOptions;
	}

	/**
	* Executes the given Testable in both Build and Skylark mode
	*/
	@Override
	protected void run(Testable testable) throws Exception {
	enableSkylarkMode(skylarkOptions);
	try {
	testable.run();
	} catch (Exception e) {
	throw new Exception("While in Skylark mode", e);
	}

	enableBuildMode(skylarkOptions);
	try {
	testable.run();
	} catch (Exception e) {
	throw new Exception("While in Build mode", e);
	}
	}
	}

	/**
	* A class that runs all tests in Build mode
	*/
	protected class BuildTest extends ModalTestCase {
	private final String[] skylarkOptions;

	public BuildTest(String... skylarkOptions) {
	this.skylarkOptions = skylarkOptions;
	}

	@Override
	protected void run(Testable testable) throws Exception {
	enableBuildMode(skylarkOptions);
	testable.run();
	}
	}

	/**
	* A class that runs all tests in Skylark mode
	*/
	protected class SkylarkTest extends ModalTestCase {
	private final String[] skylarkOptions;
	private final Map<String, Object> builtins;

	public SkylarkTest(String... skylarkOptions) {
	this(ImmutableMap.of(), skylarkOptions);
	}

	public SkylarkTest(Map<String, Object> builtins, String... skylarkOptions) {
	this.builtins = builtins;
	this.skylarkOptions = skylarkOptions;
	}

	@Override
	protected void run(Testable testable) throws Exception {
	enableSkylarkMode(builtins, skylarkOptions);
	testable.run();
	}
	}
	}