Add timing info for tests and correct caching vs. run ratio
Now prints:
//foo:bar (cached) PASSED in 0.1s
instead of:
//foo:bar (1/0 cached) PASSED
Fixes #218.
--
MOS_MIGRATED_REVID=105210302
diff --git a/src/test/java/com/google/devtools/build/lib/runtime/AbstractCommandTest.java b/src/test/java/com/google/devtools/build/lib/runtime/AbstractCommandTest.java
new file mode 100644
index 0000000..7abb490
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/runtime/AbstractCommandTest.java
@@ -0,0 +1,118 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests {@link BlazeCommand}.
+ */
+@RunWith(JUnit4.class)
+public class AbstractCommandTest {
+
+ public static class FooOptions extends OptionsBase {
+ @Option(name = "foo", category = "one", defaultValue = "0")
+ public int foo;
+ }
+
+ public static class BarOptions extends OptionsBase {
+ @Option(name = "bar", category = "two", defaultValue = "42")
+ public int foo;
+
+ @Option(name = "baz", category = "one", defaultValue = "oops")
+ public String baz;
+ }
+
+ private static class ConcreteCommand implements BlazeCommand {
+ @Override
+ public ExitCode exec(CommandEnvironment env, OptionsProvider options) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void editOptions(CommandEnvironment env, OptionsParser optionsParser) {}
+ }
+
+ @Command(name = "test_name",
+ help = "Usage: some funny usage for %{command} ...;\n\n%{options}; end",
+ options = {FooOptions.class, BarOptions.class},
+ shortDescription = "a short description",
+ allowResidue = false)
+ private static class TestCommand extends ConcreteCommand {}
+
+ @Test
+ public void testGetNameYieldsAnnotatedName() {
+ assertEquals("test_name",
+ new TestCommand().getClass().getAnnotation(Command.class).name());
+ }
+
+ @Test
+ public void testGetOptionsYieldsAnnotatedOptions() {
+ ConfiguredRuleClassProvider ruleClassProvider = new ConfiguredRuleClassProvider.Builder()
+ .build();
+
+ MoreAsserts.assertSameContents(optionClassesWithDefault(FooOptions.class, BarOptions.class),
+ BlazeCommandUtils.getOptions(
+ TestCommand.class, ImmutableList.<BlazeModule>of(), ruleClassProvider));
+ }
+
+ /***************************************************************************
+ * The tests below test how a command interacts with the dispatcher except *
+ * for execution, which is tested in {@link BlazeCommandDispatcherTest}. *
+ ***************************************************************************/
+
+ @Command(name = "a", options = {FooOptions.class}, shortDescription = "", help = "")
+ private static class CommandA extends ConcreteCommand {}
+
+ @Command(name = "b", options = {BarOptions.class}, inherits = {CommandA.class},
+ shortDescription = "", help = "")
+ private static class CommandB extends ConcreteCommand {}
+
+ @Test
+ public void testOptionsAreInherited() {
+ ConfiguredRuleClassProvider ruleClassProvider = new ConfiguredRuleClassProvider.Builder()
+ .build();
+ MoreAsserts.assertSameContents(optionClassesWithDefault(FooOptions.class),
+ BlazeCommandUtils.getOptions(
+ CommandA.class, ImmutableList.<BlazeModule>of(), ruleClassProvider));
+ MoreAsserts.assertSameContents(optionClassesWithDefault(FooOptions.class, BarOptions.class),
+ BlazeCommandUtils.getOptions(
+ CommandB.class, ImmutableList.<BlazeModule>of(), ruleClassProvider));
+ }
+
+ private Collection<Class<?>> optionClassesWithDefault(Class<?>... optionClasses) {
+ List<Class<?>> result = new ArrayList<>();
+ Collections.addAll(result, optionClasses);
+ result.add(BlazeCommandEventHandler.Options.class);
+ result.add(CommonCommandOptions.class);
+ return result;
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptionsTest.java b/src/test/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptionsTest.java
new file mode 100644
index 0000000..44f48a1
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptionsTest.java
@@ -0,0 +1,39 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.common.options.OptionsParser;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import static org.junit.Assert.assertNull;
+
+/**
+ * A regression test for {@link BlazeServerStartupOptions}.
+ */
+@RunWith(JUnit4.class)
+public class BlazeServerStartupOptionsTest {
+
+ // A regression test to make sure that the output_base option is correctly parsed if no explicit
+ // value is provided.
+ @Test
+ public void testOutputBaseIsNullByDefault() throws Exception {
+ OptionsParser parser = OptionsParser.newOptionsParser(BlazeServerStartupOptions.class);
+ parser.parse();
+ BlazeServerStartupOptions result = parser.getOptions(BlazeServerStartupOptions.class);
+ assertNull(result.outputBase);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/runtime/BlazeVersionInfoTest.java b/src/test/java/com/google/devtools/build/lib/runtime/BlazeVersionInfoTest.java
new file mode 100644
index 0000000..3f3c5ca
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/runtime/BlazeVersionInfoTest.java
@@ -0,0 +1,71 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import static java.util.Collections.singletonMap;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.util.StringUtilities;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Tests {@link BlazeVersionInfo}.
+ */
+@RunWith(JUnit4.class)
+public class BlazeVersionInfoTest {
+
+ @Test
+ public void testEmptyVersionInfoMeansNotAvailable() {
+ BlazeVersionInfo info = new BlazeVersionInfo(Collections.<String, String>emptyMap());
+ assertFalse(info.isAvailable());
+ assertNull(info.getSummary());
+ assertEquals("development version", info.getReleaseName());
+ }
+
+ @Test
+ public void testReleaseNameIsDevelopmentIfBuildLabelIsNull() {
+ Map<String, String> data = singletonMap("Build label", "");
+ BlazeVersionInfo info = new BlazeVersionInfo(data);
+ assertEquals("development version", info.getReleaseName());
+ }
+
+ @Test
+ public void testReleaseNameIfBuildLabelIsPresent() {
+ Map<String, String> data = singletonMap("Build label", "3/4/2009 (gold)");
+ BlazeVersionInfo info = new BlazeVersionInfo(data);
+ assertEquals("release 3/4/2009 (gold)", info.getReleaseName());
+ }
+
+ @Test
+ public void testFancySummaryFormatting() {
+ Map<String, String> data = new HashMap<>();
+ data.put("Some entry", "foo");
+ data.put("Another entry", "bar");
+ data.put("And a third entry", "baz");
+ BlazeVersionInfo info = new BlazeVersionInfo(data);
+ Map<String, String> sortedData = new TreeMap<>(data);
+ assertEquals(StringUtilities.layoutTable(sortedData), info.getSummary());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/runtime/InvocationPolicyEnforcerTest.java b/src/test/java/com/google/devtools/build/lib/runtime/InvocationPolicyEnforcerTest.java
new file mode 100644
index 0000000..67bdd44
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/runtime/InvocationPolicyEnforcerTest.java
@@ -0,0 +1,780 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.io.BaseEncoding;
+import com.google.devtools.build.lib.runtime.proto.InvocationPolicyOuterClass.InvocationPolicy;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.ByteArrayOutputStream;
+import java.util.List;
+
+@RunWith(JUnit4.class)
+public class InvocationPolicyEnforcerTest {
+
+ private static final String STRING_FLAG_DEFAULT = "test string default";
+
+ public static class TestOptions extends OptionsBase {
+
+ /*
+ * Basic types
+ */
+
+ @Option(name = "test_string", defaultValue = STRING_FLAG_DEFAULT)
+ public String testString;
+
+ /*
+ * Repeated flags
+ */
+
+ @Option(
+ name = "test_multiple_string",
+ defaultValue = "", // default value is ignored when allowMultiple = true.
+ allowMultiple = true)
+ public List<String> testMultipleString;
+
+ /*
+ * Expansion flags
+ */
+
+ @Option(
+ name = "test_expansion",
+ defaultValue = "null",
+ expansion = {"--test_expansion_a", "--test_expansion_b", "--test_expansion_c"})
+ public Void testExpansion;
+
+ @Option(name = "test_expansion_a", defaultValue = "false")
+ public boolean testExpansionA;
+
+ @Option(name = "test_expansion_b", defaultValue = "false")
+ public boolean testExpansionB;
+
+ @Option(name = "test_expansion_c", defaultValue = "false")
+ public boolean testExpansionC;
+
+ /*
+ * Implicit requirement flags
+ */
+
+ @Option(
+ name = "test_implicit_requirement",
+ defaultValue = "test implicit requirement default",
+ implicitRequirements = {"--an_implicit_requirement=foo"})
+ public String testImplicitRequirement;
+
+ @Option(
+ name = "an_implicit_requirement",
+ defaultValue = "implicit default")
+ public String anImplicitRequirement;
+
+ }
+
+ private static InvocationPolicyEnforcer createOptionsPolicyEnforcer(
+ InvocationPolicy.Builder invocationPolicyBuilder) throws Exception {
+ InvocationPolicy policyProto = invocationPolicyBuilder.build();
+
+ // An OptionsPolicyEnforcer could be constructed in the test directly from the InvocationPolicy
+ // proto, however Blaze will actually take the policy as another flag with a Base64 encoded
+ // binary proto and parse that, so exercise that code path in the test.
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ policyProto.writeTo(out);
+ String policyBase64 = BaseEncoding.base64().encode(out.toByteArray());
+
+ OptionsParser startupOptionsParser = OptionsParser.newOptionsParser(
+ BlazeServerStartupOptions.class);
+ String policyOption = "--invocation_policy=" + policyBase64;
+ startupOptionsParser.parse(policyOption);
+
+ return InvocationPolicyEnforcer.create(startupOptionsParser);
+ }
+
+ private OptionsParser parser;
+
+ @Before
+ public void setUp() throws Exception {
+ parser = OptionsParser.newOptionsParser(TestOptions.class);
+ }
+
+ private TestOptions getTestOptions() {
+ return parser.getOptions(TestOptions.class);
+ }
+
+ /*************************************************************************************************
+ * Tests for SetValue
+ ************************************************************************************************/
+
+ /**
+ * Tests that policy overrides a value when that value is from the user.
+ */
+ @Test
+ public void testSetValueOverridesUser() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getSetValueBuilder()
+ .addFlagValue("policy value");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_string=user value");
+
+ TestOptions testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testString);
+
+ enforcer.enforce(parser, "build");
+
+ // Get the options again after policy enforcement.
+ testOptions = getTestOptions();
+ assertEquals("policy value", testOptions.testString);
+ }
+
+ /**
+ * Tests that policy overrides a value when the user doesn't specify the value (i.e., the value
+ * is from the flag's default from its definition).
+ */
+ @Test
+ public void testSetValueOverridesDefault() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getSetValueBuilder()
+ .addFlagValue("policy value");
+
+ // No user value.
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+
+ // All the flags should be their default value.
+ TestOptions testOptions = getTestOptions();
+ assertEquals(STRING_FLAG_DEFAULT, testOptions.testString);
+
+ enforcer.enforce(parser, "build");
+
+ // Get the options again after policy enforcement.
+ testOptions = getTestOptions();
+ assertEquals("policy value", testOptions.testString);
+ }
+
+ /**
+ * Tests that SetValue overrides the user's value when the flag allows multiple values.
+ */
+ @Test
+ public void testSetValueWithMultipleValuesOverridesUser() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_multiple_string")
+ .getSetValueBuilder()
+ .addFlagValue("policy value 1")
+ .addFlagValue("policy value 2");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_multiple_string=user value 1", "--test_multiple_string=user value 2");
+
+ // Options should not be modified by running the parser through OptionsPolicyEnforcer.create().
+ TestOptions testOptions = getTestOptions();
+ assertThat(testOptions.testMultipleString)
+ .containsExactly("user value 1", "user value 2").inOrder();
+ //assertEquals(, testOptions.test_multiple_string);
+
+ enforcer.enforce(parser, "build");
+
+ // Get the options again after policy enforcement.
+ testOptions = getTestOptions();
+ assertThat(testOptions.testMultipleString)
+ .containsExactly("policy value 1", "policy value 2").inOrder();
+ }
+
+ /**
+ * Tests that policy overrides the default value when the flag allows multiple values and the user
+ * doesn't provide a value.
+ */
+ @Test
+ public void testSetValueWithMultipleValuesOverridesDefault() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_multiple_string")
+ .getSetValueBuilder()
+ .addFlagValue("policy value 1")
+ .addFlagValue("policy value 2");
+
+ // No user value.
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+
+ // Repeatable flags always default to the empty list.
+ TestOptions testOptions = getTestOptions();
+ assertThat(testOptions.testMultipleString).isEmpty();
+
+ enforcer.enforce(parser, "build");
+
+ // Options should now be the values from the policy.
+ testOptions = getTestOptions();
+ assertThat(testOptions.testMultipleString)
+ .containsExactly("policy value 1", "policy value 2").inOrder();
+ }
+
+ @Test
+ public void testSetValueHasMultipleValuesButFlagIsNotMultiple() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string") // Not repeatable flag.
+ .getSetValueBuilder()
+ .addFlagValue("policy value 1") // Has multiple values.
+ .addFlagValue("policy value 2");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+
+ try {
+ enforcer.enforce(parser, "build");
+ fail();
+ } catch (OptionsParsingException e) {
+ // expected.
+ }
+ }
+
+ @Test
+ public void testSetValueWithExpansionFlags() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_expansion_b")
+ .getSetValueBuilder()
+ .addFlagValue("false");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_expansion");
+
+ // --test_expansion should turn on test_expansion a, b, and c
+ TestOptions testOptions = getTestOptions();
+ assertTrue(testOptions.testExpansionA);
+ assertTrue(testOptions.testExpansionB);
+ assertTrue(testOptions.testExpansionC);
+
+ enforcer.enforce(parser, "build");
+
+ // After policy enforcement, test_expansion_b should be set to false, but the
+ // other two should remain the same.
+ testOptions = getTestOptions();
+ assertTrue(testOptions.testExpansionA);
+ assertFalse(testOptions.testExpansionB);
+ assertTrue(testOptions.testExpansionC);
+ }
+
+ @Test
+ public void testSetValueWithImplicitlyRequiredFlags() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("an_implicit_requirement")
+ .getSetValueBuilder()
+ .addFlagValue("policy value");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_implicit_requirement=user value");
+
+ // test_implicit_requirement sets an_implicit_requirement to "foo"
+ TestOptions testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testImplicitRequirement);
+ assertEquals("foo", testOptions.anImplicitRequirement);
+
+ enforcer.enforce(parser, "build");
+
+ testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testImplicitRequirement);
+ assertEquals("policy value", testOptions.anImplicitRequirement);
+ }
+
+ @Test
+ public void testSetValueOverridable() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getSetValueBuilder()
+ .addFlagValue("policy value")
+ .setOverridable(true);
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_string=user value");
+
+ // Repeatable flags always default to the empty list.
+ TestOptions testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testString);
+
+ enforcer.enforce(parser, "build");
+
+ // Even though the policy sets the value for test_string, the policy is overridable and the
+ // user set the value, so it should be the user's value.
+ testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testString);
+ }
+
+ @Test
+ public void testSetValueWithNoValueThrows() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getSetValueBuilder(); // No value.
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_string=user value");
+
+ // Repeatable flags always default to the empty list.
+ TestOptions testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testString);
+
+ try {
+ enforcer.enforce(parser, "build");
+ fail();
+ } catch (OptionsParsingException e) {
+ // expected.
+ }
+ }
+
+ /*************************************************************************************************
+ * Tests for UseDefault
+ ************************************************************************************************/
+
+ @Test
+ public void testUseDefault() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getUseDefaultBuilder();
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_string=user value");
+
+ // Options should be the user specified value before enforcing policy.
+ TestOptions testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testString);
+
+ enforcer.enforce(parser, "build");
+
+ // Get the options again after policy enforcement: The flag should now be back to its default
+ // value
+ testOptions = getTestOptions();
+ assertEquals(STRING_FLAG_DEFAULT, testOptions.testString);
+ }
+
+ /**
+ * Tests UseDefault when the user never actually specified the flag.
+ */
+ @Test
+ public void testUseDefaultWhenFlagWasntSet() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getUseDefaultBuilder();
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+
+ // Options should be the default since the user never specified it.
+ TestOptions testOptions = getTestOptions();
+ assertEquals(STRING_FLAG_DEFAULT, testOptions.testString);
+
+ enforcer.enforce(parser, "build");
+
+ // Still the default.
+ testOptions = getTestOptions();
+ assertEquals(STRING_FLAG_DEFAULT, testOptions.testString);
+ }
+
+ @Test
+ public void testUseDefaultWithExpansionFlags() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_expansion_b")
+ .getUseDefaultBuilder();
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_expansion");
+
+ // --test_expansion should turn on test_expansion a, b, and c
+ TestOptions testOptions = getTestOptions();
+ assertTrue(testOptions.testExpansionA);
+ assertTrue(testOptions.testExpansionB);
+ assertTrue(testOptions.testExpansionC);
+
+ enforcer.enforce(parser, "build");
+
+ // After policy enforcement, test_expansion_b should be back to its default (false), but the
+ // other two should remain the same.
+ testOptions = getTestOptions();
+ assertTrue(testOptions.testExpansionA);
+ assertFalse(testOptions.testExpansionB);
+ assertTrue(testOptions.testExpansionC);
+ }
+
+ @Test
+ public void testUseDefaultWithImplicitlyRequiredFlags() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("an_implicit_requirement")
+ .getUseDefaultBuilder();
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_implicit_requirement=user value",
+ "--an_implicit_requirement=implicit user value");
+
+ // test_implicit_requirement sets an_implicit_requirement to "foo", which ignores the user's
+ // value because the parser processes implicit values last.
+ TestOptions testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testImplicitRequirement);
+ assertEquals("foo", testOptions.anImplicitRequirement);
+
+ // Then policy puts an_implicit_requirement back to its default.
+ enforcer.enforce(parser, "build");
+
+ testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testImplicitRequirement);
+ assertEquals("implicit default", testOptions.anImplicitRequirement);
+ }
+
+ /*************************************************************************************************
+ * Tests for AllowValues
+ ************************************************************************************************/
+
+ /**
+ * Tests that AllowValues works in the normal case where the value the user specified is allowed
+ * by the policy.
+ */
+ @Test
+ public void testAllowValuesAllowsValue() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getAllowValuesBuilder()
+ .addAllowedValues(STRING_FLAG_DEFAULT)
+ .addAllowedValues("foo")
+ .addAllowedValues("bar");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_string=foo");
+
+ // Option should be "foo" as specified by the user.
+ TestOptions testOptions = getTestOptions();
+ assertEquals("foo", testOptions.testString);
+
+ enforcer.enforce(parser, "build");
+
+ // Still "foo" since "foo" is allowed by the policy.
+ testOptions = getTestOptions();
+ assertEquals("foo", testOptions.testString);
+ }
+
+ @Test
+ public void testAllowValuesDisallowsValue() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getAllowValuesBuilder()
+ // no foo!
+ .addAllowedValues(STRING_FLAG_DEFAULT)
+ .addAllowedValues("bar");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_string=foo");
+
+ // Option should be "foo" as specified by the user.
+ TestOptions testOptions = getTestOptions();
+ assertEquals("foo", testOptions.testString);
+
+ try {
+ // Should throw because "foo" is not allowed.
+ enforcer.enforce(parser, "build");
+ fail();
+ } catch (OptionsParsingException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testAllowValuesDisallowsMultipleValues() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_multiple_string")
+ .getAllowValuesBuilder()
+ .addAllowedValues("foo")
+ .addAllowedValues("bar");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_multiple_string=baz", "--test_multiple_string=bar");
+
+ // Option should be "baz" and "bar" as specified by the user.
+ TestOptions testOptions = getTestOptions();
+ assertThat(testOptions.testMultipleString).containsExactly("baz", "bar").inOrder();
+
+ try {
+ enforcer.enforce(parser, "build");
+ fail();
+ } catch (OptionsParsingException e) {
+ // expected, since baz is not allowed.
+ }
+ }
+
+ /**
+ * Tests that AllowValues sets its default value when the user doesn't provide a value and the
+ * flag's default value is disallowed.
+ */
+ @Test
+ public void testAllowValuesSetsNewDefaultWhenFlagDefaultIsDisallowed() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getAllowValuesBuilder()
+ // default value from flag's definition is not allowed
+ .addAllowedValues("foo")
+ .addAllowedValues("bar")
+ .setNewDefaultValue("new default");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+
+ // Option should be its default
+ TestOptions testOptions = getTestOptions();
+ assertEquals(STRING_FLAG_DEFAULT, testOptions.testString);
+
+ enforcer.enforce(parser, "build");
+
+ // Flag's value should be the default value from the policy.
+ testOptions = getTestOptions();
+ assertEquals("new default", testOptions.testString);
+ }
+
+ @Test
+ public void testAllowValuesDisallowsFlagDefaultButNoPolicyDefault() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getAllowValuesBuilder()
+ // default value from flag's definition is not allowed, and no alternate default
+ // is given.
+ .addAllowedValues("foo")
+ .addAllowedValues("bar");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+
+ // Option should be its default
+ TestOptions testOptions = getTestOptions();
+ assertEquals(STRING_FLAG_DEFAULT, testOptions.testString);
+
+ try {
+ enforcer.enforce(parser, "build");
+ fail();
+ } catch (OptionsParsingException e) {
+ // expected.
+ }
+ }
+
+ /*************************************************************************************************
+ * Tests for DisallowValues
+ ************************************************************************************************/
+
+ @Test
+ public void testDisallowValuesAllowsValue() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getDisallowValuesBuilder()
+ .addDisallowedValues("foo")
+ .addDisallowedValues("bar");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_string=baz");
+
+ // Option should be "baz" as specified by the user.
+ TestOptions testOptions = getTestOptions();
+ assertEquals("baz", testOptions.testString);
+
+ enforcer.enforce(parser, "build");
+
+ // Still "baz" since "baz" is allowed by the policy.
+ testOptions = getTestOptions();
+ assertEquals("baz", testOptions.testString);
+ }
+
+ @Test
+ public void testDisallowValuesDisallowsValue() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getDisallowValuesBuilder()
+ .addDisallowedValues("foo")
+ .addDisallowedValues("bar");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_string=foo");
+
+ // Option should be "foo" as specified by the user.
+ TestOptions testOptions = getTestOptions();
+ assertEquals("foo", testOptions.testString);
+
+ try {
+ enforcer.enforce(parser, "build");
+ fail();
+ } catch (OptionsParsingException e) {
+ // expected, since foo is disallowed.
+ }
+ }
+
+ @Test
+ public void testDisallowValuesDisallowsMultipleValues() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_multiple_string")
+ .getDisallowValuesBuilder()
+ .addDisallowedValues("foo")
+ .addDisallowedValues("bar");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_multiple_string=baz", "--test_multiple_string=bar");
+
+ // Option should be "baz" and "bar" as specified by the user.
+ TestOptions testOptions = getTestOptions();
+ assertThat(testOptions.testMultipleString).containsExactly("baz", "bar").inOrder();
+
+ try {
+ enforcer.enforce(parser, "build");
+ fail();
+ } catch (OptionsParsingException e) {
+ // expected, since bar is disallowed.
+ }
+ }
+
+ @Test
+ public void testDisallowValuesSetsNewDefaultWhenFlagDefaultIsDisallowed() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getDisallowValuesBuilder()
+ .addDisallowedValues(STRING_FLAG_DEFAULT)
+ .setNewDefaultValue("baz");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+
+ // Option should be the default since the use didn't specify a value.
+ TestOptions testOptions = getTestOptions();
+ assertEquals(STRING_FLAG_DEFAULT, testOptions.testString);
+
+ enforcer.enforce(parser, "build");
+
+ // Should now be "baz" because the policy set the new default to "baz"
+ testOptions = getTestOptions();
+ assertEquals("baz", testOptions.testString);
+ }
+
+ @Test
+ public void testDisallowValuesDisallowsFlagDefaultButNoPolicyDefault() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getDisallowValuesBuilder()
+ // No new default is set
+ .addDisallowedValues(STRING_FLAG_DEFAULT);
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+
+ // Option should be the default since the use didn't specify a value.
+ TestOptions testOptions = getTestOptions();
+ assertEquals(STRING_FLAG_DEFAULT, testOptions.testString);
+
+ try {
+ enforcer.enforce(parser, "build");
+ fail();
+ } catch (OptionsParsingException e) {
+ // expected.
+ }
+ }
+
+ /*************************************************************************************************
+ * Other tests
+ ************************************************************************************************/
+
+ @Test
+ public void testFlagPolicyDoesNotApply() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .addCommands("build")
+ .getSetValueBuilder()
+ .addFlagValue("policy value");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_string=user value");
+
+ TestOptions testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testString);
+
+ enforcer.enforce(parser, "test");
+
+ // Still user value.
+ testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testString);
+ }
+
+ @Test
+ public void testNonExistantFlagFromPolicy() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("i_do_not_exist")
+ .getSetValueBuilder()
+ .addFlagValue("policy value 1");
+ invocationPolicyBuilder.addFlagPoliciesBuilder()
+ .setFlagName("test_string")
+ .getSetValueBuilder()
+ .addFlagValue("policy value 2");
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_string=user value");
+
+ TestOptions testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testString);
+
+ enforcer.enforce(parser, "test");
+
+ // Still user value.
+ testOptions = getTestOptions();
+ assertEquals("policy value 2", testOptions.testString);
+ }
+
+ @Test
+ public void testOperationNotSet() throws Exception {
+ InvocationPolicy.Builder invocationPolicyBuilder = InvocationPolicy.newBuilder();
+ invocationPolicyBuilder.addFlagPoliciesBuilder();
+ // No operations added to the flag policy
+
+ InvocationPolicyEnforcer enforcer = createOptionsPolicyEnforcer(invocationPolicyBuilder);
+ parser.parse("--test_string=user value");
+
+ TestOptions testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testString);
+
+ // Shouldn't throw.
+ enforcer.enforce(parser, "test");
+
+ // Still user value.
+ testOptions = getTestOptions();
+ assertEquals("user value", testOptions.testString);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/runtime/TestResultAnalyzerTest.java b/src/test/java/com/google/devtools/build/lib/runtime/TestResultAnalyzerTest.java
new file mode 100644
index 0000000..05c94c1
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/runtime/TestResultAnalyzerTest.java
@@ -0,0 +1,137 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.packages.TestTimeout;
+import com.google.devtools.build.lib.rules.test.TestProvider;
+import com.google.devtools.build.lib.rules.test.TestProvider.TestParams;
+import com.google.devtools.build.lib.rules.test.TestResult;
+import com.google.devtools.build.lib.rules.test.TestRunnerAction;
+import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions;
+import com.google.devtools.build.lib.testutil.Suite;
+import com.google.devtools.build.lib.testutil.TestSpec;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.TestResultData;
+import com.google.devtools.common.options.OptionsParser;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@TestSpec(size = Suite.SMALL_TESTS)
+@RunWith(JUnit4.class)
+public class TestResultAnalyzerTest {
+
+ private TestResultAnalyzer underTest;
+
+ @Before
+ public void setUp() {
+ Path mockPath = mock(Path.class);
+ OptionsParser testSpecificOptions = OptionsParser.newOptionsParser(
+ TestSummaryOptions.class, ExecutionOptions.class);
+ EventBus mockBus = mock(EventBus.class);
+ underTest = new TestResultAnalyzer(
+ mockPath,
+ testSpecificOptions.getOptions(TestSummaryOptions.class),
+ testSpecificOptions.getOptions(ExecutionOptions.class),
+ mockBus);
+ }
+
+ @Test
+ public void testIncrementalAnalyzeSetsActionRanTrueWhenThereAreNonCachedResults() {
+ TestSummary.Builder summaryBuilder = makeTestSummaryBuilder();
+ assertFalse(summaryBuilder.peek().actionRan());
+
+ TestResultData testResultData = TestResultData.newBuilder().setRemotelyCached(false).build();
+ TestResult result = new TestResult(
+ mock(TestRunnerAction.class),
+ testResultData,
+ /*cached=*/false);
+
+ TestSummary.Builder newSummaryBuilder = underTest.incrementalAnalyze(summaryBuilder, result);
+ assertTrue(newSummaryBuilder.peek().actionRan());
+ }
+
+ @Test
+ public void testIncrementalAnalyzeSetsActionRanFalseForLocallyCachedTests() {
+ TestSummary.Builder summaryBuilder = makeTestSummaryBuilder();
+ assertFalse(summaryBuilder.peek().actionRan());
+
+ TestResultData testResultData = TestResultData.newBuilder().setRemotelyCached(false).build();
+ TestResult result = new TestResult(
+ mock(TestRunnerAction.class),
+ testResultData,
+ /*cached=*/true);
+
+ TestSummary.Builder newSummaryBuilder = underTest.incrementalAnalyze(summaryBuilder, result);
+ assertFalse(newSummaryBuilder.peek().actionRan());
+ }
+
+ @Test
+ public void testIncrementalAnalyzeSetsActionRanFalseForRemotelyCachedTests() {
+ TestSummary.Builder summaryBuilder = makeTestSummaryBuilder();
+ assertFalse(summaryBuilder.peek().actionRan());
+
+ TestResultData testResultData = TestResultData.newBuilder().setRemotelyCached(true).build();
+ TestResult result = new TestResult(
+ mock(TestRunnerAction.class),
+ testResultData,
+ /*cached=*/false);
+
+ TestSummary.Builder newSummaryBuilder = underTest.incrementalAnalyze(summaryBuilder, result);
+ assertFalse(newSummaryBuilder.peek().actionRan());
+ }
+
+ @Test
+ public void testIncrementalAnalyzeKeepsActionRanTrueWhenAlreadyTrueAndNewCachedResults() {
+ TestSummary.Builder summaryBuilder = makeTestSummaryBuilder().setActionRan(true);
+
+ TestResultData testResultData = TestResultData.newBuilder().setRemotelyCached(true).build();
+ TestResult result = new TestResult(
+ mock(TestRunnerAction.class),
+ testResultData,
+ /*cached=*/true);
+
+ TestSummary.Builder newSummaryBuilder = underTest.incrementalAnalyze(summaryBuilder, result);
+ assertTrue(newSummaryBuilder.peek().actionRan());
+ }
+
+ private TestSummary.Builder makeTestSummaryBuilder() {
+ // a lot of mocks to mock out fetching the TestTimeout configuration needed by
+ // {@link TestResultAnalyzer#shouldEmitTestSizeWarningInSummary(...)
+ TestParams mockParams = mock(TestParams.class);
+ when(mockParams.getTimeout()).thenReturn(TestTimeout.LONG);
+ TestProvider testProvider = new TestProvider(mockParams, ImmutableList.<String>of());
+
+ ConfiguredTarget mockTarget = mock(ConfiguredTarget.class);
+ when(mockTarget.getProvider(TestProvider.class)).thenReturn(testProvider);
+
+ return TestSummary.newBuilder()
+ .setStatus(BlazeTestStatus.PASSED)
+ .setTarget(mockTarget);
+
+ }
+
+}
diff --git a/src/test/java/com/google/devtools/build/lib/runtime/TestSummaryTest.java b/src/test/java/com/google/devtools/build/lib/runtime/TestSummaryTest.java
new file mode 100644
index 0000000..e08df35
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/runtime/TestSummaryTest.java
@@ -0,0 +1,492 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.AdditionalMatchers.find;
+import static org.mockito.AdditionalMatchers.not;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Matchers.contains;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.FailedTestCasesStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+@RunWith(JUnit4.class)
+public class TestSummaryTest {
+
+ private static final String ANY_STRING = ".*?";
+ private static final String PATH = "package";
+ private static final String TARGET_NAME = "name";
+ private ConfiguredTarget stubTarget;
+ private static final List<Long> SMALL_TIMING = ImmutableList.of(1L, 2L, 3L, 4L);
+
+ private static final int CACHED = SMALL_TIMING.size();
+ private static final int NOT_CACHED = 0;
+
+ private FileSystem fs;
+ private TestSummary.Builder basicBuilder;
+
+ @Before
+ public void setUp() throws Exception {
+ fs = new InMemoryFileSystem(BlazeClock.instance());
+ stubTarget = stubTarget();
+ basicBuilder = getTemplateBuilder();
+ }
+
+ private TestSummary.Builder getTemplateBuilder() {
+ return TestSummary.newBuilder()
+ .setTarget(stubTarget)
+ .setStatus(BlazeTestStatus.PASSED)
+ .setNumCached(NOT_CACHED)
+ .setActionRan(true)
+ .setRanRemotely(false)
+ .setWasUnreportedWrongSize(false);
+ }
+
+ private List<Path> getPathList(String... names) {
+ List<Path> list = new ArrayList<>();
+ for (String name : names) {
+ list.add(fs.getPath(name));
+ }
+ return list;
+ }
+
+ @Test
+ public void testShouldProperlyTestLabels() throws Exception {
+ ConfiguredTarget target = target("somepath", "MyTarget");
+ String expectedString = ANY_STRING + "//somepath:MyTarget" + ANY_STRING;
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+
+ TestSummary summaryStatus = createTestSummary(target, BlazeTestStatus.PASSED, CACHED);
+ TestSummaryPrinter.print(summaryStatus, terminalPrinter, true, false);
+ terminalPrinter.print(find(expectedString));
+ }
+
+ @Test
+ public void testShouldPrintPassedStatus() throws Exception {
+ String expectedString = ANY_STRING + "INFO" + ANY_STRING + BlazeTestStatus.PASSED + ANY_STRING;
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+
+ TestSummary summary = createTestSummary(stubTarget, BlazeTestStatus.PASSED, NOT_CACHED);
+ TestSummaryPrinter.print(summary, terminalPrinter, true, false);
+
+ terminalPrinter.print(find(expectedString));
+ }
+
+ @Test
+ public void testShouldPrintFailedStatus() throws Exception {
+ String expectedString = ANY_STRING + "ERROR" + ANY_STRING + BlazeTestStatus.FAILED + ANY_STRING;
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+
+ TestSummary summary = createTestSummary(stubTarget, BlazeTestStatus.FAILED, NOT_CACHED);
+
+ TestSummaryPrinter.print(summary, terminalPrinter, true, false);
+
+ terminalPrinter.print(find(expectedString));
+ }
+
+ @Test
+ public void testShouldPrintCachedStatus() throws Exception {
+ String expectedString = ANY_STRING + "\\(cached" + ANY_STRING;
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+
+ TestSummary summary = createTestSummary(stubTarget, BlazeTestStatus.PASSED, CACHED);
+
+ TestSummaryPrinter.print(summary, terminalPrinter, true, false);
+
+ terminalPrinter.print(find(expectedString));
+ }
+
+ @Test
+ public void testPartialCachedStatus() throws Exception {
+ String expectedString = ANY_STRING + "\\(3/4 cached" + ANY_STRING;
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+
+ TestSummary summary = createTestSummary(stubTarget, BlazeTestStatus.PASSED, CACHED - 1);
+ TestSummaryPrinter.print(summary, terminalPrinter, true, false);
+ terminalPrinter.print(find(expectedString));
+ }
+
+ @Test
+ public void testIncompleteCached() throws Exception {
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+ TestSummary summary = createTestSummary(stubTarget, BlazeTestStatus.INCOMPLETE, CACHED - 1);
+ TestSummaryPrinter.print(summary, terminalPrinter, true, false);
+ verify(terminalPrinter).print(not(contains("cached")));
+ }
+
+ @Test
+ public void testShouldPrintUncachedStatus() throws Exception {
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+ TestSummary summary = createTestSummary(stubTarget, BlazeTestStatus.PASSED, NOT_CACHED);
+ TestSummaryPrinter.print(summary, terminalPrinter, true, false);
+ verify(terminalPrinter).print(not(contains("cached")));
+ }
+
+ @Test
+ public void testNoTiming() throws Exception {
+ String expectedString = ANY_STRING + "INFO" + ANY_STRING + BlazeTestStatus.PASSED;
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+
+ TestSummary summary = createTestSummary(stubTarget, BlazeTestStatus.PASSED, NOT_CACHED);
+
+ TestSummaryPrinter.print(summary, terminalPrinter, true, false);
+ terminalPrinter.print(find(expectedString));
+ }
+
+ @Test
+ public void testBuilder() throws Exception {
+ // No need to copy if built twice in a row; no direct setters on the object.
+ TestSummary summary = basicBuilder.build();
+ TestSummary sameSummary = basicBuilder.build();
+ assertSame(summary, sameSummary);
+
+ basicBuilder.addTestTimes(ImmutableList.of(40L));
+
+ TestSummary summaryCopy = basicBuilder.build();
+ assertEquals(summary.getTarget(), summaryCopy.getTarget());
+ assertEquals(summary.getStatus(), summaryCopy.getStatus());
+ assertEquals(summary.numCached(), summaryCopy.numCached());
+ assertNotSame(summary, summaryCopy);
+ assertEquals(0, summary.totalRuns());
+ assertEquals(1, summaryCopy.totalRuns());
+
+ // Check that the builder can add a new warning to the copy,
+ // despite the immutability of the original.
+ basicBuilder.addTestTimes(ImmutableList.of(60L));
+
+ TestSummary fiftyCached = basicBuilder.setNumCached(50).build();
+ assertEquals(summary.getStatus(), fiftyCached.getStatus());
+ assertEquals(50, fiftyCached.numCached());
+ assertEquals(2, fiftyCached.totalRuns());
+
+ TestSummary sixtyCached = basicBuilder.setNumCached(60).build();
+ assertEquals(60, sixtyCached.numCached());
+ assertEquals(50, fiftyCached.numCached());
+
+ TestSummary failedCacheTemplate = TestSummary.newBuilderFromExisting(fiftyCached)
+ .setStatus(BlazeTestStatus.FAILED)
+ .build();
+ assertEquals(50, failedCacheTemplate.numCached());
+ assertEquals(BlazeTestStatus.FAILED, failedCacheTemplate.getStatus());
+ }
+
+ @Test
+ public void testSingleTime() throws Exception {
+ String expectedString = ANY_STRING + "INFO" + ANY_STRING + BlazeTestStatus.PASSED + ANY_STRING +
+ "in 3.4s";
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+
+ TestSummary summary = basicBuilder.addTestTimes(ImmutableList.of(3412L)).build();
+ TestSummaryPrinter.print(summary, terminalPrinter, true, false);
+ terminalPrinter.print(find(expectedString));
+ }
+
+ @Test
+ public void testNoTime() throws Exception {
+ // The last part matches anything not containing "in".
+ String expectedString = ANY_STRING + "INFO" + ANY_STRING + BlazeTestStatus.PASSED + "(?!in)*";
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+
+ TestSummary summary = basicBuilder.addTestTimes(ImmutableList.of(3412L)).build();
+ TestSummaryPrinter.print(summary, terminalPrinter, false, false);
+ terminalPrinter.print(find(expectedString));
+ }
+
+ @Test
+ public void testMultipleTimes() throws Exception {
+ String expectedString = ANY_STRING + "INFO" + ANY_STRING + BlazeTestStatus.PASSED + ANY_STRING +
+ "\n Stats over 3 runs: max = 3.0s, min = 1.0s, " +
+ "avg = 2.0s, dev = 0.8s";
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+ TestSummary summary = basicBuilder
+ .addTestTimes(ImmutableList.of(1000L, 2000L, 3000L))
+ .build();
+ TestSummaryPrinter.print(summary, terminalPrinter, true, false);
+ terminalPrinter.print(find(expectedString));
+ }
+
+ @Test
+ public void testCoverageDataReferences() throws Exception {
+ List<Path> paths = getPathList("/cov1.dat", "/cov2.dat", "/cov3.dat", "/cov4.dat");
+ FileSystemUtils.writeContentAsLatin1(paths.get(1), "something");
+ FileSystemUtils.writeContentAsLatin1(paths.get(3), "");
+ FileSystemUtils.writeContentAsLatin1(paths.get(3), "something else");
+ TestSummary summary = basicBuilder.addCoverageFiles(paths).build();
+
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+ TestSummaryPrinter.print(summary, terminalPrinter, true, false);
+ verify(terminalPrinter).print(find(ANY_STRING + "INFO" + ANY_STRING + BlazeTestStatus.PASSED));
+ verify(terminalPrinter).print(find(" /cov2.dat"));
+ verify(terminalPrinter).print(find(" /cov4.dat"));
+ }
+
+ @Test
+ public void testFlakyAttempts() throws Exception {
+ String expectedString = ANY_STRING + "WARNING" + ANY_STRING + BlazeTestStatus.FLAKY +
+ ANY_STRING + ", failed in 2 out of 3";
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+
+ TestSummary summary = basicBuilder
+ .setStatus(BlazeTestStatus.FLAKY)
+ .addPassedLogs(getPathList("/a"))
+ .addFailedLogs(getPathList("/b", "/c"))
+ .build();
+ TestSummaryPrinter.print(summary, terminalPrinter, true, false);
+ terminalPrinter.print(find(expectedString));
+ }
+
+ @Test
+ public void testNumberOfFailedRuns() throws Exception {
+ String expectedString = ANY_STRING + "ERROR" + ANY_STRING + BlazeTestStatus.FAILED +
+ ANY_STRING + "in 2 out of 3";
+ AnsiTerminalPrinter terminalPrinter = Mockito.mock(AnsiTerminalPrinter.class);
+
+ TestSummary summary = basicBuilder
+ .setStatus(BlazeTestStatus.FAILED)
+ .addPassedLogs(getPathList("/a"))
+ .addFailedLogs(getPathList("/b", "/c"))
+ .build();
+ TestSummaryPrinter.print(summary, terminalPrinter, true, false);
+ terminalPrinter.print(find(expectedString));
+ }
+
+ @Test
+ public void testFileNamesNotShown() throws Exception {
+ List<TestCase> emptyDetails = ImmutableList.of();
+ TestSummary summary = basicBuilder
+ .setStatus(BlazeTestStatus.FAILED)
+ .addPassedLogs(getPathList("/apple"))
+ .addFailedLogs(getPathList("/pear"))
+ .addCoverageFiles(getPathList("/maracuja"))
+ .addFailedTestCases(emptyDetails, FailedTestCasesStatus.FULL)
+ .build();
+
+ // Check that only //package:name is printed.
+ AnsiTerminalPrinter printer = Mockito.mock(AnsiTerminalPrinter.class);
+ TestSummaryPrinter.print(summary, printer, true, true);
+ verify(printer).print(contains("//package:name"));
+ }
+
+ @Test
+ public void testMessageShownWhenTestCasesMissing() throws Exception {
+ ImmutableList<TestCase> emptyList = ImmutableList.of();
+ TestSummary summary = createTestSummaryWithDetails(
+ BlazeTestStatus.FAILED, emptyList, FailedTestCasesStatus.NOT_AVAILABLE);
+
+ AnsiTerminalPrinter printer = Mockito.mock(AnsiTerminalPrinter.class);
+ TestSummaryPrinter.print(summary, printer, true, true);
+ verify(printer).print(contains("//package:name"));
+ verify(printer).print(contains("not available"));
+ }
+
+ @Test
+ public void testMessageShownForPartialResults() throws Exception {
+ ImmutableList<TestCase> testCases =
+ ImmutableList.of(newDetail("orange", TestCase.Status.FAILED, 1500L));
+ TestSummary summary = createTestSummaryWithDetails(BlazeTestStatus.FAILED, testCases,
+ FailedTestCasesStatus.PARTIAL);
+
+ AnsiTerminalPrinter printer = Mockito.mock(AnsiTerminalPrinter.class);
+ TestSummaryPrinter.print(summary, printer, true, true);
+ verify(printer).print(contains("//package:name"));
+ verify(printer).print(find("FAILED.*orange"));
+ verify(printer).print(contains("incomplete"));
+ }
+
+ private TestCase newDetail(String name, TestCase.Status status, long duration) {
+ return TestCase.newBuilder()
+ .setName(name)
+ .setStatus(status)
+ .setRunDurationMillis(duration).build();
+ }
+
+ @Test
+ public void testTestCaseNamesShownWhenNeeded() throws Exception {
+ TestCase detailPassed =
+ newDetail("strawberry", TestCase.Status.PASSED, 1000L);
+ TestCase detailFailed =
+ newDetail("orange", TestCase.Status.FAILED, 1500L);
+
+ TestSummary summaryPassed = createTestSummaryWithDetails(
+ BlazeTestStatus.PASSED, Arrays.asList(detailPassed));
+
+ TestSummary summaryFailed = createTestSummaryWithDetails(
+ BlazeTestStatus.FAILED, Arrays.asList(detailPassed, detailFailed));
+ assertEquals(BlazeTestStatus.FAILED, summaryFailed.getStatus());
+
+ AnsiTerminalPrinter printerPassed = Mockito.mock(AnsiTerminalPrinter.class);
+ TestSummaryPrinter.print(summaryPassed, printerPassed, true, true);
+ verify(printerPassed).print(contains("//package:name"));
+
+ AnsiTerminalPrinter printerFailed = Mockito.mock(AnsiTerminalPrinter.class);
+ TestSummaryPrinter.print(summaryFailed, printerFailed, true, true);
+ verify(printerFailed).print(contains("//package:name"));
+ verify(printerFailed).print(find("FAILED.*orange *\\(1\\.5"));
+ }
+
+ @Test
+ public void testTestCaseNamesOrdered() throws Exception {
+ TestCase[] details = {
+ newDetail("apple", TestCase.Status.FAILED, 1000L),
+ newDetail("banana", TestCase.Status.FAILED, 1000L),
+ newDetail("cranberry", TestCase.Status.FAILED, 1000L)
+ };
+
+ // The exceedingly dumb approach: writing all the permutations down manually
+ // is simply easier than any way of generating them.
+ int[][] permutations = {
+ { 0, 1, 2 },
+ { 0, 2, 1 },
+ { 1, 0, 2 },
+ { 1, 2, 0 },
+ { 2, 0, 1 },
+ { 2, 1, 0 }
+ };
+
+ for (int[] permutation : permutations) {
+ List<TestCase> permutatedDetails = new ArrayList<>();
+
+ for (int element : permutation) {
+ permutatedDetails.add(details[element]);
+ }
+
+ TestSummary summary = createTestSummaryWithDetails(BlazeTestStatus.FAILED, permutatedDetails);
+
+ // A mock that checks the ordering of method calls
+ AnsiTerminalPrinter printer = Mockito.mock(AnsiTerminalPrinter.class);
+ TestSummaryPrinter.print(summary, printer, true, true);
+ InOrder order = Mockito.inOrder(printer);
+ order.verify(printer).print(contains("//package:name"));
+ order.verify(printer).print(find("FAILED.*apple"));
+ order.verify(printer).print(find("FAILED.*banana"));
+ order.verify(printer).print(find("FAILED.*cranberry"));
+ }
+ }
+
+ @Test
+ public void testCachedResultsFirstInSort() throws Exception {
+ TestSummary summaryFailedCached = createTestSummary(BlazeTestStatus.FAILED, CACHED);
+ TestSummary summaryFailedNotCached = createTestSummary(BlazeTestStatus.FAILED, NOT_CACHED);
+ TestSummary summaryPassedCached = createTestSummary(BlazeTestStatus.PASSED, CACHED);
+ TestSummary summaryPassedNotCached = createTestSummary(BlazeTestStatus.PASSED, NOT_CACHED);
+
+ // This way we can make the test independent from the sort order of FAILEd
+ // and PASSED.
+
+ assertTrue(summaryFailedCached.compareTo(summaryPassedNotCached) < 0);
+ assertTrue(summaryPassedCached.compareTo(summaryFailedNotCached) < 0);
+ }
+
+ @Test
+ public void testCollectingFailedDetails() throws Exception {
+ TestCase rootCase = TestCase.newBuilder()
+ .setName("tests")
+ .setRunDurationMillis(5000L)
+ .addChild(newDetail("apple", TestCase.Status.FAILED, 1000L))
+ .addChild(newDetail("banana", TestCase.Status.PASSED, 1000L))
+ .addChild(newDetail("cherry", TestCase.Status.ERROR, 1000L))
+ .build();
+
+ TestSummary summary = getTemplateBuilder()
+ .collectFailedTests(rootCase)
+ .setStatus(BlazeTestStatus.FAILED)
+ .build();
+
+ AnsiTerminalPrinter printer = Mockito.mock(AnsiTerminalPrinter.class);
+ TestSummaryPrinter.print(summary, printer, true, true);
+ verify(printer).print(contains("//package:name"));
+ verify(printer).print(find("FAILED.*apple"));
+ verify(printer).print(find("ERROR.*cherry"));
+ }
+
+ private ConfiguredTarget target(String path, String targetName) throws Exception {
+ ConfiguredTarget target = Mockito.mock(ConfiguredTarget.class);
+ when(target.getLabel()).thenReturn(Label.create(path, targetName));
+ return target;
+ }
+
+ private ConfiguredTarget stubTarget() throws Exception {
+ return target(PATH, TARGET_NAME);
+ }
+
+ private TestSummary createTestSummaryWithDetails(BlazeTestStatus status,
+ List<TestCase> details) {
+ TestSummary summary = getTemplateBuilder()
+ .setStatus(status)
+ .addFailedTestCases(details, FailedTestCasesStatus.FULL)
+ .build();
+ return summary;
+ }
+
+ private TestSummary createTestSummaryWithDetails(
+ BlazeTestStatus status, List<TestCase> testCaseList,
+ FailedTestCasesStatus detailsStatus) {
+ TestSummary summary = getTemplateBuilder()
+ .setStatus(status)
+ .addFailedTestCases(testCaseList, detailsStatus)
+ .build();
+ return summary;
+ }
+
+ private static TestSummary createTestSummary(ConfiguredTarget target, BlazeTestStatus status,
+ int numCached) {
+ ImmutableList<TestCase> emptyList = ImmutableList.of();
+ TestSummary summary = TestSummary.newBuilder()
+ .setTarget(target)
+ .setStatus(status)
+ .setNumCached(numCached)
+ .setActionRan(true)
+ .setRanRemotely(false)
+ .setWasUnreportedWrongSize(false)
+ .addFailedTestCases(emptyList, FailedTestCasesStatus.FULL)
+ .addTestTimes(SMALL_TIMING)
+ .build();
+ return summary;
+ }
+
+ private TestSummary createTestSummary(BlazeTestStatus status, int numCached) {
+ TestSummary summary = getTemplateBuilder()
+ .setStatus(status)
+ .setNumCached(numCached)
+ .addTestTimes(SMALL_TIMING)
+ .build();
+ return summary;
+ }
+}
