bazel syntax: avoid overloading of ParserInput.create
Now: ParserInput.from{String,CharArray,Latin1,UTF8}.
Also, avoid a handful of redundant array copies/conversions.
PiperOrigin-RevId: 319105574
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkActionFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkActionFactory.java
index f0b4cf4..b5576b0 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkActionFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkActionFactory.java
@@ -688,10 +688,10 @@
ImmutableList.Builder<Substitution> substitutionsBuilder = ImmutableList.builder();
for (Map.Entry<String, String> substitution :
Dict.cast(substitutionsUnchecked, String.class, String.class, "substitutions").entrySet()) {
- // ParserInput.create(Path) uses Latin1 when reading BUILD files, which might
+ // Blaze calls ParserInput.fromLatin1 when reading BUILD files, which might
// contain UTF-8 encoded symbols as part of template substitution.
// As a quick fix, the substitution values are corrected before being passed on.
- // In the long term, fixing ParserInput.create(Path) would be a better approach.
+ // In the long term, avoiding ParserInput.fromLatin would be a better approach.
substitutionsBuilder.add(
Substitution.of(substitution.getKey(), convertLatin1ToUtf8(substitution.getValue())));
}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java
index 70ca4e7..5a0fac6 100644
--- a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java
@@ -524,9 +524,7 @@
Globber globber =
createLegacyGlobber(
buildFile.asPath().getParentDirectory(), packageId, ImmutableSet.of(), locator);
- ParserInput input =
- ParserInput.create(
- FileSystemUtils.convertFromLatin1(buildFileBytes), buildFile.asPath().toString());
+ ParserInput input = ParserInput.fromLatin1(buildFileBytes, buildFile.asPath().toString());
// Options for processing BUILD files. (No prelude, so recordScope(true) is safe.)
FileOptions options =
FileOptions.builder()
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedFileFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedFileFunction.java
index 84caf2d..a1ac6eb 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedFileFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedFileFunction.java
@@ -71,7 +71,7 @@
// parse
StarlarkFile file =
- StarlarkFile.parse(ParserInput.create(bytes, key.getPath().asPath().toString()));
+ StarlarkFile.parse(ParserInput.fromLatin1(bytes, key.getPath().asPath().toString()));
if (!file.ok()) {
Event.replayEventsOn(env.getListener(), file.errors());
throw resolvedValueError("Failed to parse resolved file " + key.getPath());
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java
index da1dd39..e769ac6 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ASTFileLookupFunction.java
@@ -142,7 +142,7 @@
? FileSystemUtils.readContent(path)
: FileSystemUtils.readWithKnownFileSize(path, fileValue.getSize());
digest = getDigestFromFileValueOrFromKnownFileContents(fileValue, bytes, digestHashFunction);
- ParserInput input = ParserInput.create(bytes, path.toString());
+ ParserInput input = ParserInput.fromLatin1(bytes, path.toString());
file = StarlarkFile.parse(input, options);
} catch (IOException e) {
throw new ErrorReadingStarlarkExtensionException(e, Transience.TRANSIENT);
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java
index fdf14ac..83d9f1a 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageFunction.java
@@ -1248,7 +1248,7 @@
// If control flow reaches here, we're in territory that is deliberately unsound.
// See the javadoc for ActionOnIOExceptionReadingBuildFile.
}
- input = ParserInput.create(buildFileBytes, inputFile.toString());
+ input = ParserInput.fromLatin1(buildFileBytes, inputFile.toString());
// Options for processing BUILD files.
FileOptions options =
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunction.java
index e759b4b..4658eae 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceASTFunction.java
@@ -98,7 +98,7 @@
try {
StarlarkFile file =
StarlarkFile.parse(
- ParserInput.create(
+ ParserInput.fromString(
ruleClassProvider.getDefaultWorkspacePrefix(), "/DEFAULT.WORKSPACE"),
options);
if (!file.ok()) {
@@ -108,7 +108,7 @@
if (newWorkspaceFileContents != null) {
file =
StarlarkFile.parseWithPrelude(
- ParserInput.create(
+ ParserInput.fromString(
newWorkspaceFileContents, resolvedFile.get().asPath().toString()),
file.getStatements(),
// The WORKSPACE.resolved file breaks through the usual privacy mechanism.
@@ -118,7 +118,9 @@
FileSystemUtils.readWithKnownFileSize(repoWorkspace, repoWorkspace.getFileSize());
file =
StarlarkFile.parseWithPrelude(
- ParserInput.create(bytes, repoWorkspace.toString()), file.getStatements(), options);
+ ParserInput.fromLatin1(bytes, repoWorkspace.toString()),
+ file.getStatements(),
+ options);
if (!file.ok()) {
Event.replayEventsOn(env.getListener(), file.errors());
throw resolvedValueError("Failed to parse WORKSPACE file");
@@ -150,7 +152,7 @@
file =
StarlarkFile.parseWithPrelude(
- ParserInput.create(suffix, "/DEFAULT.WORKSPACE.SUFFIX"),
+ ParserInput.fromString(suffix, "/DEFAULT.WORKSPACE.SUFFIX"),
file.getStatements(),
// The DEFAULT.WORKSPACE.SUFFIX file breaks through the usual privacy mechanism.
options.toBuilder().allowLoadPrivateSymbols(true).build());
diff --git a/src/main/java/com/google/devtools/build/lib/starlarkdebug/server/ThreadHandler.java b/src/main/java/com/google/devtools/build/lib/starlarkdebug/server/ThreadHandler.java
index d707aa0..8384537 100644
--- a/src/main/java/com/google/devtools/build/lib/starlarkdebug/server/ThreadHandler.java
+++ b/src/main/java/com/google/devtools/build/lib/starlarkdebug/server/ThreadHandler.java
@@ -302,7 +302,7 @@
// TODO(adonovan): opt: don't parse and resolve the expression every time we hit a breakpoint
// (!).
- ParserInput input = ParserInput.create(content, "<debug eval>");
+ ParserInput input = ParserInput.fromString(content, "<debug eval>");
// TODO(adonovan): the module or call frame should be a parameter.
Module module = Module.ofInnermostEnclosingStarlarkFunction(thread);
return EvalUtils.exec(input, FileOptions.DEFAULT, module, thread);
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/ParserInput.java b/src/main/java/com/google/devtools/build/lib/syntax/ParserInput.java
index 388d5da..79bb694 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/ParserInput.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/ParserInput.java
@@ -15,11 +15,16 @@
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
+import java.nio.charset.StandardCharsets;
/**
* The apparent name and contents of a source file, for consumption by the parser. The file name
* appears in the location information in the syntax tree, and in error messages, but the Starlark
* interpreter will not attempt to open the file.
+ *
+ * <p>The parser consumes a stream of chars (UTF-16 codes), and the syntax positions reported by
+ * {@link Node#getStartOffset} and {@link Location.column} are effectively indices into a char
+ * array.
*/
public final class ParserInput {
@@ -43,30 +48,50 @@
/** Returns an unnamed input source that reads from a list of strings, joined by newlines. */
public static ParserInput fromLines(String... lines) {
- return create(Joiner.on("\n").join(lines), "");
+ return fromString(Joiner.on("\n").join(lines), "");
}
- /** Returns an import source that reads from a Latin-1 encoded byte array. */
- public static ParserInput create(byte[] bytes, String file) {
- char[] content = convertFromLatin1(bytes);
- return new ParserInput(content, file);
+ /**
+ * Returns an input source that reads from a UTF-8-encoded byte array. The caller is free to
+ * subsequently mutate the array.
+ */
+ public static ParserInput fromUTF8(byte[] bytes, String file) {
+ // TODO(adonovan): opt: avoid one of the two copies.
+ return fromString(new String(bytes, StandardCharsets.UTF_8), file);
+ }
+
+ /**
+ * Returns an input source that reads from a Latin1-encoded byte array. The caller is free to
+ * subsequently mutate the array.
+ *
+ * @deprecated this function exists to support legacy uses of Latin1 in Bazel. Do not use Latin1
+ * in new applications.
+ */
+ @Deprecated
+ public static ParserInput fromLatin1(byte[] bytes, String file) {
+ char[] chars = new char[bytes.length];
+ for (int i = 0; i < bytes.length; i++) {
+ chars[i] = (char) (0xff & bytes[i]);
+ }
+ return new ParserInput(chars, file);
}
/** Returns an input source that reads from the given string. */
+ public static ParserInput fromString(String content, String file) {
+ return fromCharArray(content.toCharArray(), file);
+ }
+
+ /** Deprecated alias for {@link #fromString}. */
+ @Deprecated
public static ParserInput create(String content, String file) {
- return create(content.toCharArray(), file);
+ return fromString(content, file);
}
- /** Returns an input source that reads from the given char array. */
- public static ParserInput create(char[] content, String file) {
+ /**
+ * Returns an input source that reads from the given char array. The caller must not subsequently
+ * modify the array.
+ */
+ public static ParserInput fromCharArray(char[] content, String file) {
return new ParserInput(content, file);
}
-
- private static char[] convertFromLatin1(byte[] content) {
- char[] latin1 = new char[content.length];
- for (int i = 0; i < latin1.length; i++) { // yeah, latin1 is this easy! :-)
- latin1[i] = (char) (0xff & content[i]);
- }
- return latin1;
- }
}
diff --git a/src/main/java/com/google/devtools/build/skydoc/FilesystemFileAccessor.java b/src/main/java/com/google/devtools/build/skydoc/FilesystemFileAccessor.java
index 68138cf..4015cfd 100644
--- a/src/main/java/com/google/devtools/build/skydoc/FilesystemFileAccessor.java
+++ b/src/main/java/com/google/devtools/build/skydoc/FilesystemFileAccessor.java
@@ -24,7 +24,7 @@
@Override
public ParserInput inputSource(String filename) throws IOException {
- return ParserInput.create(Files.readAllBytes(Paths.get(filename)), filename);
+ return ParserInput.fromLatin1(Files.readAllBytes(Paths.get(filename)), filename);
}
@Override
diff --git a/src/main/java/net/starlark/java/cmd/Starlark.java b/src/main/java/net/starlark/java/cmd/Starlark.java
index 6077f6d..0bd0498 100644
--- a/src/main/java/net/starlark/java/cmd/Starlark.java
+++ b/src/main/java/net/starlark/java/cmd/Starlark.java
@@ -92,7 +92,7 @@
// lines only until the parse is complete.
while ((line = prompt()) != null) {
- ParserInput input = ParserInput.create(line, "<stdin>");
+ ParserInput input = ParserInput.fromString(line, "<stdin>");
try {
Object result = EvalUtils.exec(input, options, module, thread);
if (result != com.google.devtools.build.lib.syntax.Starlark.NONE) {
@@ -125,7 +125,7 @@
/** Execute a Starlark file. */
private int execute(String filename, String content) {
try {
- EvalUtils.exec(ParserInput.create(content, filename), options, module, thread);
+ EvalUtils.exec(ParserInput.fromString(content, filename), options, module, thread);
return 0;
} catch (SyntaxError.Exception ex) {
for (SyntaxError error : ex.errors()) {
diff --git a/src/test/java/com/google/devtools/build/lib/packages/WorkspaceFactoryTestHelper.java b/src/test/java/com/google/devtools/build/lib/packages/WorkspaceFactoryTestHelper.java
index b291ffd..2950a82 100644
--- a/src/test/java/com/google/devtools/build/lib/packages/WorkspaceFactoryTestHelper.java
+++ b/src/test/java/com/google/devtools/build/lib/packages/WorkspaceFactoryTestHelper.java
@@ -84,7 +84,7 @@
byte[] bytes =
FileSystemUtils.readWithKnownFileSize(workspaceFilePath, workspaceFilePath.getFileSize());
factory.parseForTesting(
- ParserInput.create(bytes, workspaceFilePath.toString()), eventHandler);
+ ParserInput.fromLatin1(bytes, workspaceFilePath.toString()), eventHandler);
} catch (BuildFileContainsErrorsException e) {
exception = e;
} catch (IOException | InterruptedException e) {
diff --git a/src/test/java/com/google/devtools/build/lib/packages/util/PackageFactoryApparatus.java b/src/test/java/com/google/devtools/build/lib/packages/util/PackageFactoryApparatus.java
index 4fc948d..831caaa 100644
--- a/src/test/java/com/google/devtools/build/lib/packages/util/PackageFactoryApparatus.java
+++ b/src/test/java/com/google/devtools/build/lib/packages/util/PackageFactoryApparatus.java
@@ -146,7 +146,7 @@
// TODO(adonovan): inline this into all callers. It has nothing to do with PackageFactory.
public StarlarkFile parse(Path buildFile) throws IOException {
byte[] bytes = FileSystemUtils.readWithKnownFileSize(buildFile, buildFile.getFileSize());
- ParserInput input = ParserInput.create(bytes, buildFile.toString());
+ ParserInput input = ParserInput.fromLatin1(bytes, buildFile.toString());
StarlarkFile file = StarlarkFile.parse(input);
Event.replayEventsOn(eventHandler, file.errors());
return file;
diff --git a/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java b/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java
index 930cfb8..52e06d7 100644
--- a/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java
@@ -191,7 +191,7 @@
private void exec(String... lines)
throws SyntaxError.Exception, EvalException, InterruptedException {
- ParserInput input = ParserInput.create(Joiner.on("\n").join(lines), "a.star");
+ ParserInput input = ParserInput.fromString(Joiner.on("\n").join(lines), "a.star");
Module module =
Module.withPredeclared(
StarlarkSemantics.DEFAULT,
diff --git a/src/test/java/com/google/devtools/build/lib/skylark/StarlarkRuleImplementationFunctionsTest.java b/src/test/java/com/google/devtools/build/lib/skylark/StarlarkRuleImplementationFunctionsTest.java
index 9c5947a..ec3ff44 100644
--- a/src/test/java/com/google/devtools/build/lib/skylark/StarlarkRuleImplementationFunctionsTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skylark/StarlarkRuleImplementationFunctionsTest.java
@@ -836,10 +836,9 @@
* usually write those files using UTF-8 encoding. Currently, the string-valued 'substitutions'
* parameter of the template_action function contains a hack that assumes its input is a UTF-8
* encoded string which has been ingested as Latin 1. The hack converts the string to its
- * "correct" UTF-8 value. Once {@link
- * com.google.devtools.build.lib.syntax.ParserInput#create(byte[],
- * com.google.devtools.build.lib.vfs.PathFragment)} parses files using UTF-8 and the hack for the
- * substituations parameter is removed, this test will fail.
+ * "correct" UTF-8 value. Once Blaze starts calling {@link
+ * com.google.devtools.build.lib.syntax.ParserInput#fromUTF8} instead of {@code fromLatin1} and
+ * the hack for the substituations parameter is removed, this test will fail.
*/
@Test
public void testCreateTemplateActionWithWrongEncoding() throws Exception {
diff --git a/src/test/java/com/google/devtools/build/lib/starlarkdebug/server/StarlarkDebugServerTest.java b/src/test/java/com/google/devtools/build/lib/starlarkdebug/server/StarlarkDebugServerTest.java
index a481c3a..8e385df 100644
--- a/src/test/java/com/google/devtools/build/lib/starlarkdebug/server/StarlarkDebugServerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/starlarkdebug/server/StarlarkDebugServerTest.java
@@ -762,7 +762,7 @@
}
private static ParserInput createInput(String filename, String... lines) {
- return ParserInput.create(Joiner.on("\n").join(lines), filename);
+ return ParserInput.fromString(Joiner.on("\n").join(lines), filename);
}
/**
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java b/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java
index cb15fd4..255f528 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java
@@ -38,7 +38,7 @@
* error handler beforehand.
*/
private Lexer createLexer(String input) {
- ParserInput inputSource = ParserInput.create(input, "/some/path.txt");
+ ParserInput inputSource = ParserInput.fromString(input, "/some/path.txt");
errors.clear();
lastError = null;
return new Lexer(inputSource, FileOptions.DEFAULT, errors);
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/ParserInputTest.java b/src/test/java/com/google/devtools/build/lib/syntax/ParserInputTest.java
index 1ab4c61..5b87ba5 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/ParserInputTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/ParserInputTest.java
@@ -36,7 +36,7 @@
String content = joinLines("Line 1", "Line 2", "Line 3", "");
Path file = scratch.file("/tmp/my/file.txt", content.getBytes(StandardCharsets.UTF_8));
byte[] bytes = FileSystemUtils.readWithKnownFileSize(file, file.getFileSize());
- ParserInput input = ParserInput.create(bytes, file.toString());
+ ParserInput input = ParserInput.fromLatin1(bytes, file.toString());
assertThat(new String(input.getContent())).isEqualTo(content);
assertThat(input.getFile()).isEqualTo("/tmp/my/file.txt");
}
@@ -45,7 +45,7 @@
public void testCreateFromString() {
String content = "Content provided as a string.";
String pathName = "/the/name/of/the/content.txt";
- ParserInput input = ParserInput.create(content, pathName);
+ ParserInput input = ParserInput.fromString(content, pathName);
assertThat(new String(input.getContent())).isEqualTo(content);
assertThat(input.getFile()).isEqualTo(pathName);
}
@@ -55,19 +55,19 @@
String content = "Content provided as a string.";
String pathName = "/the/name/of/the/content.txt";
char[] contentChars = content.toCharArray();
- ParserInput input = ParserInput.create(contentChars, pathName);
+ ParserInput input = ParserInput.fromCharArray(contentChars, pathName);
assertThat(new String(input.getContent())).isEqualTo(content);
assertThat(input.getFile()).isEqualTo(pathName);
}
@Test
public void testWillNotTryToReadInputFileIfContentProvidedAsString() {
- ParserInput.create("Content provided as string.", "/will/not/try/to/read");
+ ParserInput.fromString("Content provided as string.", "/will/not/try/to/read");
}
@Test
public void testWillNotTryToReadInputFileIfContentProvidedAsChars() {
char[] content = "Content provided as char array.".toCharArray();
- ParserInput.create(content, "/will/not/try/to/read");
+ ParserInput.fromCharArray(content, "/will/not/try/to/read");
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/ParserTest.java b/src/test/java/com/google/devtools/build/lib/syntax/ParserTest.java
index d64aae2..dd16ea3 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/ParserTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/ParserTest.java
@@ -1310,6 +1310,6 @@
for (int i = 0; i < 1000; i++) {
s.append("{");
}
- return ParserInput.create(s.toString(), "foo.star");
+ return ParserInput.fromString(s.toString(), "foo.star");
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/StarlarkFileTest.java b/src/test/java/com/google/devtools/build/lib/syntax/StarlarkFileTest.java
index ab318af..576c913 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/StarlarkFileTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/StarlarkFileTest.java
@@ -31,7 +31,7 @@
*/
private static StarlarkFile parseFile(String... lines) {
String src = Joiner.on("\n").join(lines);
- ParserInput input = ParserInput.create(src, "foo.star");
+ ParserInput input = ParserInput.fromString(src, "foo.star");
return StarlarkFile.parse(input);
}
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadDebuggingTest.java b/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadDebuggingTest.java
index 3553cb9..fa336c3 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadDebuggingTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/StarlarkThreadDebuggingTest.java
@@ -85,7 +85,7 @@
// Execute a small file that calls f.
ParserInput input =
- ParserInput.create(
+ ParserInput.fromString(
"def g(a, y, z):\n" // shadows global a
+ " f()\n"
+ "g(4, 5, 6)",
diff --git a/src/test/java/com/google/devtools/build/skydoc/SkydocTest.java b/src/test/java/com/google/devtools/build/skydoc/SkydocTest.java
index c054324..b5724c1 100644
--- a/src/test/java/com/google/devtools/build/skydoc/SkydocTest.java
+++ b/src/test/java/com/google/devtools/build/skydoc/SkydocTest.java
@@ -62,7 +62,7 @@
public ParserInput inputSource(String pathString) throws IOException {
Path path = fileSystem.getPath("/" + pathString);
byte[] bytes = FileSystemUtils.asByteSource(path).read();
- return ParserInput.create(bytes, path.toString());
+ return ParserInput.fromLatin1(bytes, path.toString());
}
@Override
