commit 6dbfafe639507c60a00201baa14e293c69315695
author nharmata <nharmata@google.com> Tue Feb 05 08:55:07 2019 -0800
committer Copybara-Service <copybara-piper@google.com> Tue Feb 05 08:57:00 2019 -0800
tree 075c886c7230bab9b606443bee203ae00d4bc7b5
parent d922a1e5faa83d209556694674c6c93c8538c39f

In Parser, dedupe the String instances referred to by StringLiteral instances. Implement the same optimization in the serialization code.

Some .bzl files have tons of duplicate string constants.

We can't dedupe the StringLiteral instances themselves, because they all have different Locations.

RELNOTES: None
PiperOrigin-RevId: 232495067

src/test/java/com/google/devtools/build/lib/syntax/ParserTest.java

diff --git a/src/test/java/com/google/devtools/build/lib/syntax/ParserTest.java b/src/test/java/com/google/devtools/build/lib/syntax/ParserTest.java
index 78de4f9..4e59cdd 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/ParserTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/ParserTest.java
@@ -18,6 +18,7 @@
 import static org.junit.Assert.fail;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
 import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.events.Location;
 import com.google.devtools.build.lib.events.Location.LineAndColumn;
@@ -27,6 +28,7 @@
 import com.google.devtools.build.lib.syntax.util.EvaluationTestCase;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Set;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -47,11 +49,14 @@
     return parseFileWithComments(input).getStatements();
   }
 
+  private BuildFileAST parseFileForSkylarkAsAST(String... input) {
+    BuildFileAST ast = BuildFileAST.parseString(getEventHandler(), input);
+    return ast.validate(env, getEventHandler());
+  }
+
   /** Parses Skylark code */
   private List<Statement> parseFileForSkylark(String... input) {
-    BuildFileAST ast = BuildFileAST.parseString(getEventHandler(), input);
-    ast = ast.validate(env, getEventHandler());
-    return ast.getStatements();
+    return parseFileForSkylarkAsAST(input).getStatements();
   }
 
   private static String getText(String text, ASTNode node) {
@@ -1504,4 +1509,20 @@
         ")\n");
     assertContainsError(":4:5: non-keyword arg after keyword arg");
   }
+
+  @Test
+  public void testStringsAreDeduped() throws Exception {
+    BuildFileAST buildFileAST =
+        parseFileForSkylarkAsAST("L1 = ['cat', 'dog', 'fish']", "L2 = ['dog', 'fish', 'cat']");
+    Set<String> uniqueStringInstances = Sets.newIdentityHashSet();
+    SyntaxTreeVisitor collectAllStringsInStringLiteralsVisitor =
+        new SyntaxTreeVisitor() {
+          @Override
+          public void visit(StringLiteral stringLiteral) {
+            uniqueStringInstances.add(stringLiteral.getValue());
+          }
+        };
+    collectAllStringsInStringLiteralsVisitor.visit(buildFileAST);
+    assertThat(uniqueStringInstances).containsExactly("cat", "dog", "fish");
+  }
 }