Merge ByteBufferFragment and ByteFragmentAtOffset and call it FileFragment.
It seemed odd that the object that contains the start and end of a fragment is inside the object that contains the ByteBuffer. It's more logical the other way round, but after going this far, I realized that it's a bit nicer to not have two objects at all.
RELNOTES: None.
PiperOrigin-RevId: 301761088
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/BufferSplitter.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/BufferSplitter.java
deleted file mode 100644
index 2e1db30..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/BufferSplitter.java
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.file;
-
-import com.google.common.collect.Lists;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.concurrent.Callable;
-
-/**
- * Task for splitting the contents of the {@link ByteBufferFragment} (with underlying {@link
- * ByteBuffer}). Intended to be called in parallel for the fragments of the {@link ByteBuffer} for
- * lexing the contents into independent logical declarations.
- *
- * <p>{@link ParallelFileProcessing}
- */
-public class BufferSplitter implements Callable<List<ByteFragmentAtOffset>> {
- private final ByteBufferFragment bufferFragment;
- private final DeclarationConsumer consumer;
- private final SeparatorFinder separatorFinder;
- private final int offset;
-
- /**
- * @param bufferFragment {@link ByteBufferFragment}, fragment of which should be splitted
- * @param consumer declaration consumer
- * @param separatorFinder finds declaration separators
- * @param offset start offset of <code>buffer</code> from the beginning of the file
- */
- public BufferSplitter(
- ByteBufferFragment bufferFragment,
- DeclarationConsumer consumer,
- SeparatorFinder separatorFinder,
- int offset) {
- this.bufferFragment = bufferFragment;
- this.consumer = consumer;
- this.separatorFinder = separatorFinder;
- this.offset = offset;
- }
-
- /**
- * Returns the list of {@link ByteFragmentAtOffset} - fragments on the bounds of the current
- * fragment, which should be potentially merged with fragments from the neighbor buffer fragments.
- *
- * <p>Combined list of {@link ByteFragmentAtOffset} is passed to {@link DeclarationAssembler} for
- * merging.
- */
- @Override
- public List<ByteFragmentAtOffset> call() throws Exception {
- List<ByteFragmentAtOffset> fragments = Lists.newArrayList();
- int start = 0;
- while (true) {
- int end = separatorFinder.findNextSeparator(bufferFragment, start, -1);
- if (end < 0) {
- break;
- }
- ByteBufferFragment fragment = bufferFragment.subFragment(start, end + 1);
- ByteFragmentAtOffset fragmentAtOffset = new ByteFragmentAtOffset(offset, fragment);
- if (start > 0) {
- consumer.declaration(fragmentAtOffset);
- } else {
- fragments.add(fragmentAtOffset);
- }
- start = end + 1;
- }
- // There is always at least one byte at the bounds of the fragment.
- ByteBufferFragment lastFragment = bufferFragment.subFragment(start, bufferFragment.length());
- fragments.add(new ByteFragmentAtOffset(offset, lastFragment));
- return fragments;
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/ByteFragmentAtOffset.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/ByteFragmentAtOffset.java
deleted file mode 100644
index 89499ce..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/ByteFragmentAtOffset.java
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.file;
-
-/**
- * Represents a fragment of file as a {@link ByteBufferFragment} of {@link java.nio.ByteBuffer},
- * starting at {@link #bufferOffset}. The first byte should be read at {@link #getFragmentOffset()},
- * the length of fragment is {@link ByteBufferFragment#length()}.
- */
-public class ByteFragmentAtOffset {
- /** The offset in the file the {@code ByteBuffer} backing this fragment starts at. */
- private final long bufferOffset;
-
- private final ByteBufferFragment fragment;
-
- public ByteFragmentAtOffset(long bufferOffset, ByteBufferFragment fragment) {
- this.bufferOffset = bufferOffset;
- this.fragment = fragment;
- }
-
- /** The offset in the file the {@code ByteBuffer} backing this fragment starts at. */
- public long getBufferOffset() {
- return bufferOffset;
- }
-
- /** The offset in the file this fragment starts at. */
- public long getFragmentOffset() {
- return bufferOffset + fragment.getStartIncl();
- }
-
- public ByteBufferFragment getFragment() {
- return fragment;
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationAssembler.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationAssembler.java
index 9d04150..2cea656 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationAssembler.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationAssembler.java
@@ -25,7 +25,7 @@
import java.util.List;
/**
- * A {@link BufferSplitter} callback interface implementation, that assembles fragments of
+ * A {@link FileFragmentSplitter} callback interface implementation, that assembles fragments of
* declarations (that may occur on the edges of byte buffer fragments) together and passes all
* declarations to delegate {@link DeclarationConsumer}, which does further processing / parsing.
*/
@@ -48,18 +48,17 @@
/**
* Should be called after all work for processing of individual buffer fragments is complete.
*
- * @param fragments list of {@link ByteFragmentAtOffset} - pieces on the bounds of sub-fragments.
+ * @param fragments list of {@link FileFragment} - pieces on the bounds of sub-fragments.
* @throws GenericParsingException thrown by delegate {@link #declarationConsumer}
*/
- public void wrapUp(List<ByteFragmentAtOffset> fragments)
- throws GenericParsingException, IOException {
- fragments.sort(Comparator.comparingLong(ByteFragmentAtOffset::getFragmentOffset));
+ public void wrapUp(List<FileFragment> fragments) throws GenericParsingException, IOException {
+ fragments.sort(Comparator.comparingLong(FileFragment::getFragmentOffset));
- List<ByteFragmentAtOffset> list = Lists.newArrayList();
+ List<FileFragment> list = Lists.newArrayList();
long previous = -1;
- for (ByteFragmentAtOffset edge : fragments) {
+ for (FileFragment edge : fragments) {
long start = edge.getFragmentOffset();
- ByteBufferFragment fragment = edge.getFragment();
+ FileFragment fragment = edge;
if (previous >= 0 && previous != start) {
sendMerged(list);
list.clear();
@@ -72,10 +71,9 @@
}
}
- private void sendMerged(List<ByteFragmentAtOffset> list)
- throws GenericParsingException, IOException {
+ private void sendMerged(List<FileFragment> list) throws GenericParsingException, IOException {
Preconditions.checkArgument(!list.isEmpty());
- ByteFragmentAtOffset first = list.get(0);
+ FileFragment first = list.get(0);
if (list.size() == 1) {
declarationConsumer.declaration(first);
return;
@@ -91,12 +89,10 @@
// 4. Later we will check only interestingRanges for separators, and create corresponding
// fragments; the underlying common ByteBuffer will be reused, so we are not performing
// extensive copying.
- long firstOffset = first.getBufferOffset();
- List<ByteBufferFragment> fragments = new ArrayList<>();
+ List<FileFragment> fragments = new ArrayList<>();
List<Range<Integer>> interestingRanges = Lists.newArrayList();
int fragmentShift = 0;
- for (ByteFragmentAtOffset byteFragmentAtOffset : list) {
- ByteBufferFragment fragment = byteFragmentAtOffset.getFragment();
+ for (FileFragment fragment : list) {
fragments.add(fragment);
if (fragmentShift > 0) {
// We are only looking for the separators between fragments.
@@ -112,21 +108,7 @@
fragmentShift += fragment.length();
}
- ByteBufferFragment merged = ByteBufferFragment.merge(fragments);
-
- long newOffset;
- if (Iterables.getLast(list).getBufferOffset() == firstOffset) {
- // If all fragment offsets were the same (which is the case if all their originating buffers
- // were the same), then the merged fragment has the start index of the first originating
- // fragment, and the end index of the last originating fragment.
- // The old offset can be used without issue.
- newOffset = firstOffset;
- } else {
- // If fragment offsets differed, then the new merged fragment has a different offset.
- // In this case, the merged fragment has relative indices [0, len), which means that its
- // offset should reflect the absolute "real" start offset.
- newOffset = first.getFragmentOffset();
- }
+ FileFragment merged = FileFragment.merge(fragments);
int previousEnd = 0;
for (Range<Integer> range : interestingRanges) {
@@ -136,12 +118,11 @@
// There should always be a previous fragment, as we are checking non-intersecting ranges,
// starting from the connection point between first and second fragments.
Preconditions.checkState(idx > previousEnd);
- declarationConsumer.declaration(
- new ByteFragmentAtOffset(newOffset, merged.subFragment(previousEnd, idx + 1)));
+ declarationConsumer.declaration(merged.subFragment(previousEnd, idx + 1));
previousEnd = idx + 1;
}
}
- declarationConsumer.declaration(
- new ByteFragmentAtOffset(newOffset, merged.subFragment(previousEnd, merged.length())));
+
+ declarationConsumer.declaration(merged.subFragment(previousEnd, merged.length()));
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationConsumer.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationConsumer.java
index 93176d2..48b7885 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationConsumer.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationConsumer.java
@@ -23,10 +23,9 @@
/**
* Accepts a declaration for further processing / parsing.
*
- * @param byteFragmentAtOffset a fragment of {@link java.nio.ByteBuffer} in the form of {@link
- * ByteBufferFragment}, starting at offset in the underlying file.
+ * @param fragment a fragment of {@link java.nio.ByteBuffer} in the form of {@link FileFragment},
+ * starting at offset in the underlying file.
* @throws GenericParsingException if declaration processing discovered the wrong syntax
*/
- void declaration(ByteFragmentAtOffset byteFragmentAtOffset)
- throws GenericParsingException, IOException;
+ void declaration(FileFragment fragment) throws GenericParsingException, IOException;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/ByteBufferFragment.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/FileFragment.java
similarity index 70%
rename from src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/ByteBufferFragment.java
rename to src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/FileFragment.java
index b418896..58a7c38 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/ByteBufferFragment.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/FileFragment.java
@@ -24,30 +24,53 @@
import java.util.List;
/** Represents the fragment of immutable {@link ByteBuffer} for parallel processing. */
-public class ByteBufferFragment {
+public class FileFragment {
+ /** The backing {@link ByteBuffer}. May be shared with other {@link FileFragment} instances. */
private final ByteBuffer buffer;
- /** The start of this fragment in the backing {@link ByteBuffer}. */
+ /** The offset in the file the backing {@link ByteBuffer} starts at. */
+ private final long fileOffset;
+
+ /**
+ * The start of this fragment in the backing {@link ByteBuffer}.
+ *
+ * <p>The file offset of the fragment is the sum of this value and the offset of the backing
+ * buffer ({@code fileOffset}).
+ */
private final int startIncl;
/** The end of this fragment in the backing {@link ByteBuffer}. */
private final int endExcl;
- public ByteBufferFragment(ByteBuffer buffer, int startIncl, int endExcl) {
+ public FileFragment(ByteBuffer buffer, long fileOffset, int startIncl, int endExcl) {
if (endExcl < startIncl) {
throw new IndexOutOfBoundsException();
}
this.buffer = buffer;
+ this.fileOffset = fileOffset;
this.startIncl = startIncl;
this.endExcl = endExcl;
}
- /** Returns the start of this fragment in the backing {@link ByteBuffer}. */
+ /**
+ * Returns the start of this fragment in the backing {@link ByteBuffer}, that is, relative to the
+ * {@code fileOffset} in the file.
+ */
@VisibleForTesting
public int getStartIncl() {
return startIncl;
}
+ /** The byte offset in the file the backing {@link ByteBuffer} starts at. */
+ public long getFileOffset() {
+ return fileOffset;
+ }
+
+ /** The offset in the file this fragment starts at. */
+ public long getFragmentOffset() {
+ return fileOffset + startIncl;
+ }
+
/** Returns the length of fragment. */
public int length() {
return endExcl - startIncl;
@@ -59,9 +82,9 @@
* @param from start index, inclusive
* @param to end index, exclusive, or the size of the buffer, if the last symbol is included
*/
- public ByteBufferFragment subFragment(int from, int to) {
+ public FileFragment subFragment(int from, int to) {
checkSubBounds(from, to);
- return new ByteBufferFragment(buffer, startIncl + from, startIncl + to);
+ return new FileFragment(buffer, fileOffset, startIncl + from, startIncl + to);
}
public byte byteAt(int index) {
@@ -86,16 +109,6 @@
}
}
- public byte[] getBytes(int from, int to) {
- checkSubBounds(from, to);
- int length = to - from;
- byte[] bytes = new byte[length];
- ByteBuffer copy = buffer.duplicate();
- copy.position(startIncl + from);
- copy.get(bytes, 0, length);
- return bytes;
- }
-
/**
* Helper method for forming error messages with text fragment around a place with a problem.
*
@@ -111,6 +124,16 @@
return new String(bytes, StandardCharsets.ISO_8859_1);
}
+ public byte[] getBytes(int from, int to) {
+ checkSubBounds(from, to);
+ int length = to - from;
+ byte[] bytes = new byte[length];
+ ByteBuffer copy = buffer.duplicate();
+ copy.position(startIncl + from);
+ copy.get(bytes, 0, length);
+ return bytes;
+ }
+
private void getBytes(byte[] dst, int offset) {
if (dst.length - offset < length()) {
throw new IndexOutOfBoundsException(
@@ -139,39 +162,43 @@
}
/**
- * Merges passed buffer fragments into the new buffer fragment. If the list contains only one
- * fragment, returns that fragment. Otherwise, creates the new buffer and copies the contents of
- * passed buffer fragments into it.
+ * Merges multiple file fragments into a single one.
*
- * @param list non-empty list of buffer fragments
- * @return buffer fragment with the contents, merged from all passed buffers. If only one buffer
- * fragment was passed, returns it.
+ * <p>If needed, also creates a new backing {@link ByteBuffer} (if the fragments on the input were
+ * not contiguous in a single buffer)
+ *
+ * @param list non-empty list of file fragments
+ * @return the merged file fragment
*/
@SuppressWarnings("ReferenceEquality")
- public static ByteBufferFragment merge(List<ByteBufferFragment> list) {
+ public static FileFragment merge(List<FileFragment> list) {
Preconditions.checkState(!list.isEmpty());
if (list.size() == 1) {
return list.get(0);
}
ByteBuffer first = list.get(0).buffer;
- // We compare contained buffers to be exactly same objects here, i.e. changing to use
+ long firstOffset = list.get(0).fileOffset;
+
+ // We compare contained fragments to be exactly same objects here, i.e. changing to use
// of equals() is not needed. (Warning suppressed.)
- List<ByteBufferFragment> tail = list.subList(1, list.size());
+ List<FileFragment> tail = list.subList(1, list.size());
if (tail.stream().allMatch(el -> el.buffer == first)) {
int previousEnd = list.get(0).endExcl;
- for (ByteBufferFragment fragment : tail) {
+ for (FileFragment fragment : tail) {
Preconditions.checkState(fragment.startIncl == previousEnd);
previousEnd = fragment.endExcl;
}
- return new ByteBufferFragment(first, list.get(0).startIncl, Iterables.getLast(list).endExcl);
+ return new FileFragment(
+ first, firstOffset, list.get(0).startIncl, Iterables.getLast(list).endExcl);
}
- int len = list.stream().mapToInt(ByteBufferFragment::length).sum();
+ int len = list.stream().mapToInt(FileFragment::length).sum();
byte[] bytes = new byte[len];
int position = 0;
- for (ByteBufferFragment current : list) {
+ for (FileFragment current : list) {
current.getBytes(bytes, position);
position += current.length();
}
- return new ByteBufferFragment(ByteBuffer.wrap(bytes), 0, len);
+ return new FileFragment(
+ ByteBuffer.wrap(bytes), firstOffset + list.get(0).getStartIncl(), 0, len);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/FileFragmentSplitter.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/FileFragmentSplitter.java
new file mode 100644
index 0000000..d782abc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/FileFragmentSplitter.java
@@ -0,0 +1,75 @@
+// Copyright 2019 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package com.google.devtools.build.lib.bazel.rules.ninja.file;
+
+import com.google.common.collect.Lists;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+/**
+ * Task for splitting the contents of the {@link FileFragment} (with underlying {@link ByteBuffer}).
+ * Intended to be called in parallel for the fragments of the {@link ByteBuffer} for lexing the
+ * contents into independent logical declarations.
+ *
+ * <p>{@link ParallelFileProcessing}
+ */
+public class FileFragmentSplitter implements Callable<List<FileFragment>> {
+ private final FileFragment fileFragment;
+ private final DeclarationConsumer consumer;
+ private final SeparatorFinder separatorFinder;
+
+ /**
+ * @param fileFragment {@link FileFragment}, fragment of which should be splitted
+ * @param consumer declaration consumer
+ * @param separatorFinder finds declaration separators
+ */
+ public FileFragmentSplitter(
+ FileFragment fileFragment, DeclarationConsumer consumer, SeparatorFinder separatorFinder) {
+ this.fileFragment = fileFragment;
+ this.consumer = consumer;
+ this.separatorFinder = separatorFinder;
+ }
+
+ /**
+ * Returns the list of {@link FileFragment} - fragments on the bounds of the current fragment,
+ * which should be potentially merged with neighbor fragments.
+ *
+ * <p>Combined list of {@link FileFragment} is passed to {@link DeclarationAssembler} for merging.
+ */
+ @Override
+ public List<FileFragment> call() throws Exception {
+ List<FileFragment> fragments = Lists.newArrayList();
+ int start = 0;
+ while (true) {
+ int end = separatorFinder.findNextSeparator(fileFragment, start, -1);
+ if (end < 0) {
+ break;
+ }
+ FileFragment fragment = fileFragment.subFragment(start, end + 1);
+ if (start > 0) {
+ consumer.declaration(fragment);
+ } else {
+ fragments.add(fragment);
+ }
+ start = end + 1;
+ }
+ // There is always at least one byte at the bounds of the fragment.
+ FileFragment lastFragment = fileFragment.subFragment(start, fileFragment.length());
+ fragments.add(lastFragment);
+ return fragments;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/IncorrectSeparatorException.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/IncorrectSeparatorException.java
index 4d0c6c1..7c712d1 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/IncorrectSeparatorException.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/IncorrectSeparatorException.java
@@ -15,7 +15,7 @@
package com.google.devtools.build.lib.bazel.rules.ninja.file;
-/** Thrown by {@link BufferSplitter} when incorrect file separators are used ('\r'). */
+/** Thrown by {@link FileFragmentSplitter} when incorrect file separators are used ('\r'). */
public class IncorrectSeparatorException extends GenericParsingException {
public IncorrectSeparatorException(String message) {
super(message);
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/NinjaSeparatorFinder.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/NinjaSeparatorFinder.java
index eb6121a..2d11ce4 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/NinjaSeparatorFinder.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/NinjaSeparatorFinder.java
@@ -40,7 +40,7 @@
private NinjaSeparatorFinder() {}
@Override
- public int findNextSeparator(ByteBufferFragment fragment, int startingFrom, int untilExcluded)
+ public int findNextSeparator(FileFragment fragment, int startingFrom, int untilExcluded)
throws IncorrectSeparatorException {
Preconditions.checkState(startingFrom < fragment.length());
Preconditions.checkState(untilExcluded < 0 || untilExcluded <= fragment.length());
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/ParallelFileProcessing.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/ParallelFileProcessing.java
index cac08df..fda5972 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/ParallelFileProcessing.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/ParallelFileProcessing.java
@@ -50,9 +50,9 @@
/**
* Processes file in parallel: {@link java.nio.channels.FileChannel} is used to read contents into
* a sequence of buffers. Each buffer is split into chunks, which are tokenized in parallel by
- * {@link BufferSplitter}, using the <code>predicate</code>. Fragments of tokens (on the bounds of
- * buffer fragments) are assembled by {@link DeclarationAssembler}. The resulting tokens (each can
- * be further parsed independently) are passed to {@link DeclarationConsumer}.
+ * {@link FileFragmentSplitter}, using the <code>predicate</code>. Fragments of tokens (on the
+ * bounds of buffer fragments) are assembled by {@link DeclarationAssembler}. The resulting tokens
+ * (each can be further parsed independently) are passed to {@link DeclarationConsumer}.
*
* <p>The main ideas behind this implementation are:
*
@@ -105,9 +105,9 @@
DeclarationAssembler assembler =
new DeclarationAssembler(tokenConsumerFactory.get(), predicate);
- CollectingListFuture<List<ByteFragmentAtOffset>, GenericParsingException> future =
+ CollectingListFuture<List<FileFragment>, GenericParsingException> future =
new CollectingListFuture<>(GenericParsingException.class);
- List<List<ByteFragmentAtOffset>> listOfLists;
+ List<List<FileFragment>> listOfLists;
int offset = 0;
boolean keepReading = true;
while (keepReading) {
@@ -120,7 +120,7 @@
}
}
listOfLists = future.getResult();
- List<ByteFragmentAtOffset> fragments =
+ List<FileFragment> fragments =
listOfLists.stream().flatMap(List::stream).collect(Collectors.toList());
assembler.wrapUp(fragments);
@@ -140,7 +140,7 @@
private void tokenizeFragments(
ByteBuffer bb,
int offset,
- CollectingListFuture<List<ByteFragmentAtOffset>, GenericParsingException> future) {
+ CollectingListFuture<List<FileFragment>, GenericParsingException> future) {
int from = 0;
int blockSize = parameters.getTokenizeBlockSize();
while (from < bb.limit()) {
@@ -150,8 +150,8 @@
to = bb.limit();
}
DeclarationConsumer consumer = tokenConsumerFactory.get();
- ByteBufferFragment fragment = new ByteBufferFragment(bb, from, to);
- BufferSplitter tokenizer = new BufferSplitter(fragment, consumer, predicate, offset);
+ FileFragment fragment = new FileFragment(bb, offset, from, to);
+ FileFragmentSplitter tokenizer = new FileFragmentSplitter(fragment, consumer, predicate);
future.add(executorService.submit(tokenizer));
from = to;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/SeparatorFinder.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/SeparatorFinder.java
index c879646..e95e241 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/SeparatorFinder.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/SeparatorFinder.java
@@ -27,6 +27,6 @@
* @param untilExcluded index to stop search before (excluded from search).
* @throws IncorrectSeparatorException if the incorrect separator value (\r) is used
*/
- int findNextSeparator(ByteBufferFragment fragment, int startingFrom, int untilExcluded)
+ int findNextSeparator(FileFragment fragment, int startingFrom, int untilExcluded)
throws IncorrectSeparatorException;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexer.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexer.java
index b2812a2..fe57c2b 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexer.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexer.java
@@ -19,7 +19,7 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import com.google.devtools.build.lib.util.Pair;
import java.util.Arrays;
import java.util.List;
@@ -39,7 +39,7 @@
NinjaToken.POOL)
.collect(ImmutableMap.toImmutableMap(token -> token.getBytes()[0], nt -> nt));
- private final ByteBufferFragment fragment;
+ private final FileFragment fragment;
private NinjaLexerStep step;
private final List<Pair<Integer, Integer>> ranges;
private final List<NinjaToken> tokens;
@@ -49,7 +49,7 @@
private boolean interpretPoolAsVariable = false;
/** @param fragment fragment to do the lexing on */
- public NinjaLexer(ByteBufferFragment fragment) {
+ public NinjaLexer(FileFragment fragment) {
this.fragment = fragment;
step = new NinjaLexerStep(fragment, 0);
ranges = Lists.newArrayList();
@@ -223,7 +223,7 @@
return step.getError();
}
- public ByteBufferFragment getFragment() {
+ public FileFragment getFragment() {
return fragment;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexerStep.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexerStep.java
index b1569e6..73a288d 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexerStep.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexerStep.java
@@ -17,7 +17,7 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSortedSet;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import java.nio.charset.StandardCharsets;
import java.util.function.Predicate;
@@ -59,7 +59,7 @@
return builder.build();
}
- private final ByteBufferFragment fragment;
+ private final FileFragment fragment;
private final int position;
private boolean seenZero;
@@ -69,7 +69,7 @@
/**
* @param position start of the step inside a fragment; must point to a symbol inside fragment.
*/
- public NinjaLexerStep(ByteBufferFragment fragment, int position) {
+ public NinjaLexerStep(FileFragment fragment, int position) {
Preconditions.checkState(position < fragment.length());
this.fragment = fragment;
this.position = position;
@@ -96,7 +96,7 @@
return !seenZero && error == null && end < fragment.length();
}
- public ByteBufferFragment getFragment() {
+ public FileFragment getFragment() {
return fragment;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaFileParseResult.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaFileParseResult.java
index 6ccf57f..cd4a542 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaFileParseResult.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaFileParseResult.java
@@ -19,7 +19,7 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteFragmentAtOffset;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
import com.google.devtools.build.lib.util.Pair;
import java.io.IOException;
@@ -56,7 +56,7 @@
private final NavigableMap<String, List<Pair<Long, NinjaVariableValue>>> variables;
private final NavigableMap<String, List<Pair<Long, NinjaRule>>> rules;
private final NavigableMap<String, List<Pair<Long, NinjaPool>>> pools;
- private final List<ByteFragmentAtOffset> targets;
+ private final List<FileFragment> targets;
private final NavigableMap<Long, NinjaPromise<NinjaFileParseResult>> includedFilesFutures;
private final NavigableMap<Long, NinjaPromise<NinjaFileParseResult>> subNinjaFilesFutures;
@@ -77,7 +77,7 @@
subNinjaFilesFutures.put(offset, promise);
}
- public void addTarget(ByteFragmentAtOffset fragment) {
+ public void addTarget(FileFragment fragment) {
targets.add(fragment);
}
@@ -103,7 +103,7 @@
return rules;
}
- public List<ByteFragmentAtOffset> getTargets() {
+ public List<FileFragment> getTargets() {
return targets;
}
@@ -152,8 +152,7 @@
* Fills in passed {@link NinjaScope} with the expanded variables and rules, and <code>rawTargets
* </code> - map of NinjaScope to list of fragments with unparsed Ninja targets.
*/
- public void expandIntoScope(
- NinjaScope scope, Map<NinjaScope, List<ByteFragmentAtOffset>> rawTargets)
+ public void expandIntoScope(NinjaScope scope, Map<NinjaScope, List<FileFragment>> rawTargets)
throws InterruptedException, GenericParsingException, IOException {
scope.setRules(rules);
scope.setPools(pools);
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParser.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParser.java
index fa4d52e..f98faae 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParser.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParser.java
@@ -14,9 +14,8 @@
package com.google.devtools.build.lib.bazel.rules.ninja.parser;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteFragmentAtOffset;
import com.google.devtools.build.lib.bazel.rules.ninja.file.DeclarationConsumer;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaToken;
@@ -44,10 +43,8 @@
}
@Override
- public void declaration(ByteFragmentAtOffset byteFragmentAtOffset)
- throws GenericParsingException, IOException {
- ByteBufferFragment fragment = byteFragmentAtOffset.getFragment();
- long offset = byteFragmentAtOffset.getFragmentOffset();
+ public void declaration(FileFragment fragment) throws GenericParsingException, IOException {
+ long offset = fragment.getFragmentOffset();
NinjaLexer lexer = new NinjaLexer(fragment);
if (!lexer.hasNextToken()) {
@@ -93,16 +90,16 @@
parseResult.addSubNinjaScope(declarationStart, subNinjaFuture);
break;
case BUILD:
- ByteFragmentAtOffset targetFragment;
+ FileFragment targetFragment;
if (declarationStart == offset) {
- targetFragment = byteFragmentAtOffset;
+ targetFragment = fragment;
} else {
// Method subFragment accepts only the offset *inside fragment*.
// So we should subtract the offset of fragment's buffer in file
- // (byteFragmentAtOffset.getOffset()),
+ // (fragment.getFileOffset()),
// and start of fragment inside buffer (fragment.getStartIncl()).
long fragmentStart =
- declarationStart - byteFragmentAtOffset.getBufferOffset() - fragment.getStartIncl();
+ declarationStart - fragment.getFileOffset() - fragment.getStartIncl();
// While the absolute offset is typed as long (because of larger ninja files), the
// fragments are only at most Integer.MAX_VALUE long, so fragmentStart cannot be
@@ -114,10 +111,7 @@
+ "since each fragment is at most max-int long.",
fragmentStart));
}
- targetFragment =
- new ByteFragmentAtOffset(
- byteFragmentAtOffset.getBufferOffset(),
- fragment.subFragment((int) fragmentStart, fragment.length()));
+ targetFragment = fragment.subFragment((int) fragmentStart, fragment.length());
}
parseResult.addTarget(targetFragment);
break;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline/NinjaPipeline.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline/NinjaPipeline.java
index 81d4bcf..d45aaf6 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline/NinjaPipeline.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline/NinjaPipeline.java
@@ -23,8 +23,8 @@
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteFragmentAtOffset;
import com.google.devtools.build.lib.bazel.rules.ninja.file.CollectingListFuture;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
import com.google.devtools.build.lib.bazel.rules.ninja.file.NinjaSeparatorFinder;
import com.google.devtools.build.lib.bazel.rules.ninja.file.ParallelFileProcessing;
@@ -91,7 +91,7 @@
waitForFutureAndGetWithCheckedException(
scheduleParsing(mainFile), GenericParsingException.class, IOException.class);
- Map<NinjaScope, List<ByteFragmentAtOffset>> rawTargets = Maps.newHashMap();
+ Map<NinjaScope, List<FileFragment>> rawTargets = Maps.newHashMap();
NinjaScope scope = new NinjaScope();
// This will cause additional parsing of included/subninja scopes, and their recursive expand.
result.expandIntoScope(scope, rawTargets);
@@ -104,7 +104,7 @@
* included scopes, and parsing corresponding targets.
*/
private List<NinjaTarget> iterateScopesScheduleTargetsParsing(
- NinjaScope scope, Map<NinjaScope, List<ByteFragmentAtOffset>> rawTargets)
+ NinjaScope scope, Map<NinjaScope, List<FileFragment>> rawTargets)
throws GenericParsingException, InterruptedException {
ArrayDeque<NinjaScope> queue = new ArrayDeque<>();
queue.add(scope);
@@ -112,14 +112,14 @@
new CollectingListFuture<>(GenericParsingException.class);
while (!queue.isEmpty()) {
NinjaScope currentScope = queue.removeFirst();
- List<ByteFragmentAtOffset> targetFragments = rawTargets.get(currentScope);
+ List<FileFragment> targetFragments = rawTargets.get(currentScope);
Preconditions.checkNotNull(targetFragments);
- for (ByteFragmentAtOffset byteFragmentAtOffset : targetFragments) {
+ for (FileFragment fragment : targetFragments) {
future.add(
service.submit(
() ->
- new NinjaParserStep(new NinjaLexer(byteFragmentAtOffset.getFragment()))
- .parseNinjaTarget(currentScope, byteFragmentAtOffset.getFragmentOffset())));
+ new NinjaParserStep(new NinjaLexer(fragment))
+ .parseNinjaTarget(currentScope, fragment.getFragmentOffset())));
}
queue.addAll(currentScope.getIncludedScopes());
queue.addAll(currentScope.getSubNinjaScopes());
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/DeclarationAssemblerTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/DeclarationAssemblerTest.java
index 60de6bf..f345b7d 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/DeclarationAssemblerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/DeclarationAssemblerTest.java
@@ -21,9 +21,8 @@
import com.google.common.base.Strings;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteFragmentAtOffset;
import com.google.devtools.build.lib.bazel.rules.ninja.file.DeclarationAssembler;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
import com.google.devtools.build.lib.bazel.rules.ninja.file.NinjaSeparatorFinder;
import com.google.devtools.build.lib.util.Pair;
@@ -65,26 +64,22 @@
String s1 = "hello";
String s2 = "goodbye";
byte[] chars1 = (unrelatedFirstBuffer + s1).getBytes(ISO_8859_1);
+ ByteBuffer bytes1 = ByteBuffer.wrap(chars1);
byte[] chars2 = s2.getBytes(ISO_8859_1);
+ ByteBuffer bytes2 = ByteBuffer.wrap(chars2);
DeclarationAssembler assembler =
new DeclarationAssembler(
- (byteFragmentAtOffset) -> {
+ fragment -> {
offsetStringPairList.add(
- new Pair<>(
- byteFragmentAtOffset.getFragmentOffset(),
- byteFragmentAtOffset.getFragment().toString()));
+ new Pair<>(fragment.getFragmentOffset(), fragment.toString()));
},
NinjaSeparatorFinder.INSTANCE);
assembler.wrapUp(
Lists.newArrayList(
- new ByteFragmentAtOffset(
- 0,
- new ByteBufferFragment(
- ByteBuffer.wrap(chars1), unrelatedFirstBuffer.length(), chars1.length)),
- new ByteFragmentAtOffset(
- chars1.length, new ByteBufferFragment(ByteBuffer.wrap(chars2), 0, s2.length()))));
+ new FileFragment(bytes1, 0, unrelatedFirstBuffer.length(), chars1.length),
+ new FileFragment(bytes2, chars1.length, 0, s2.length())));
assertThat(Iterables.getOnlyElement(offsetStringPairList))
.isEqualTo(new Pair<>((long) unrelatedFirstBuffer.length(), "hellogoodbye"));
@@ -94,22 +89,22 @@
throws GenericParsingException, IOException {
List<String> list = Lists.newArrayList();
final byte[] chars1 = s1.getBytes(StandardCharsets.ISO_8859_1);
+ ByteBuffer bytes1 = ByteBuffer.wrap(chars1);
final byte[] chars2 = s2.getBytes(StandardCharsets.ISO_8859_1);
+ ByteBuffer bytes2 = ByteBuffer.wrap(chars2);
DeclarationAssembler assembler =
new DeclarationAssembler(
- (byteFragmentAtOffset) -> {
- list.add(byteFragmentAtOffset.getFragment().toString());
- assertThat(byteFragmentAtOffset.getBufferOffset()).isAnyOf(0L, (long) chars1.length);
+ fragment -> {
+ list.add(fragment.toString());
+ assertThat(fragment.getFileOffset()).isAnyOf(0L, (long) chars1.length);
},
NinjaSeparatorFinder.INSTANCE);
assembler.wrapUp(
Lists.newArrayList(
- new ByteFragmentAtOffset(
- 0, new ByteBufferFragment(ByteBuffer.wrap(chars1), 0, s1.length())),
- new ByteFragmentAtOffset(
- chars1.length, new ByteBufferFragment(ByteBuffer.wrap(chars2), 0, s2.length()))));
+ new FileFragment(bytes1, 0, 0, s1.length()),
+ new FileFragment(bytes2, chars1.length, 0, s2.length())));
assertThat(list).isEqualTo(Arrays.asList(expected));
}
@@ -120,19 +115,17 @@
List<String> list = Lists.newArrayList();
DeclarationAssembler assembler =
new DeclarationAssembler(
- (byteFragmentAtOffset) -> {
- list.add(byteFragmentAtOffset.getFragment().toString());
- assertThat(byteFragmentAtOffset.getBufferOffset()).isEqualTo(0);
+ fragment -> {
+ list.add(fragment.toString());
+ assertThat(fragment.getFileOffset()).isEqualTo(0);
},
NinjaSeparatorFinder.INSTANCE);
- final byte[] chars = s.getBytes(StandardCharsets.ISO_8859_1);
+ byte[] chars = s.getBytes(StandardCharsets.ISO_8859_1);
+ ByteBuffer bytes = ByteBuffer.wrap(chars);
assembler.wrapUp(
Lists.newArrayList(
- new ByteFragmentAtOffset(
- 0, new ByteBufferFragment(ByteBuffer.wrap(chars), start1, end1)),
- new ByteFragmentAtOffset(
- 0, new ByteBufferFragment(ByteBuffer.wrap(chars), start2, end2))));
+ new FileFragment(bytes, 0, start1, end1), new FileFragment(bytes, 0, start2, end2)));
assertThat(list).isEqualTo(Arrays.asList(expected));
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/BufferSplitterTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/FileFragmentSplitterTest.java
similarity index 63%
rename from src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/BufferSplitterTest.java
rename to src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/FileFragmentSplitterTest.java
index 9a8bc48..885debd 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/BufferSplitterTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/FileFragmentSplitterTest.java
@@ -20,10 +20,9 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.BufferSplitter;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteFragmentAtOffset;
import com.google.devtools.build.lib.bazel.rules.ninja.file.DeclarationConsumer;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragmentSplitter;
import com.google.devtools.build.lib.bazel.rules.ninja.file.NinjaSeparatorFinder;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@@ -33,9 +32,9 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-/** Tests for {@link BufferSplitter} */
+/** Tests for {@link FileFragmentSplitter} */
@RunWith(JUnit4.class)
-public class BufferSplitterTest {
+public class FileFragmentSplitterTest {
@Test
public void testTokenizeSimple() throws Exception {
List<String> list = ImmutableList.of("one", "two", "three");
@@ -43,20 +42,21 @@
List<String> result = Lists.newArrayList();
int offsetValue = 123;
DeclarationConsumer consumer =
- (byteFragmentAtOffset) -> {
- result.add(byteFragmentAtOffset.getFragment().toString());
- assertThat(byteFragmentAtOffset.getBufferOffset()).isEqualTo(offsetValue);
+ fragment -> {
+ result.add(fragment.toString());
+ assertThat(fragment.getFileOffset()).isEqualTo(offsetValue);
};
byte[] chars = String.join("\n", list).getBytes(StandardCharsets.ISO_8859_1);
- ByteBufferFragment fragment = new ByteBufferFragment(ByteBuffer.wrap(chars), 0, chars.length);
- BufferSplitter tokenizer =
- new BufferSplitter(fragment, consumer, NinjaSeparatorFinder.INSTANCE, offsetValue);
- List<ByteFragmentAtOffset> edges = tokenizer.call();
+ ByteBuffer buffer = ByteBuffer.wrap(chars);
+ FileFragment fragment = new FileFragment(buffer, offsetValue, 0, chars.length);
+ FileFragmentSplitter tokenizer =
+ new FileFragmentSplitter(fragment, consumer, NinjaSeparatorFinder.INSTANCE);
+ List<FileFragment> edges = tokenizer.call();
assertThat(result).containsExactly("two\n");
assertThat(
edges.stream()
- .map(pair -> Preconditions.checkNotNull(pair.getFragment()).toString())
+ .map(pair -> Preconditions.checkNotNull(pair).toString())
.collect(Collectors.toList()))
.containsExactly("one\n", "three")
.inOrder();
@@ -67,19 +67,19 @@
List<String> list =
ImmutableList.of("one", " one-detail", "two", "\ttwo-detail", "three", " three-detail");
byte[] chars = String.join("\n", list).getBytes(StandardCharsets.ISO_8859_1);
+ ByteBuffer bytes = ByteBuffer.wrap(chars);
List<String> result = Lists.newArrayList();
- DeclarationConsumer consumer =
- (byteFragmentAtOffset) -> result.add(byteFragmentAtOffset.getFragment().toString());
+ DeclarationConsumer consumer = fragment -> result.add(fragment.toString());
- ByteBufferFragment fragment = new ByteBufferFragment(ByteBuffer.wrap(chars), 0, chars.length);
- BufferSplitter tokenizer =
- new BufferSplitter(fragment, consumer, NinjaSeparatorFinder.INSTANCE, 0);
- List<ByteFragmentAtOffset> edges = tokenizer.call();
+ FileFragment fragment = new FileFragment(bytes, 0, 0, chars.length);
+ FileFragmentSplitter tokenizer =
+ new FileFragmentSplitter(fragment, consumer, NinjaSeparatorFinder.INSTANCE);
+ List<FileFragment> edges = tokenizer.call();
assertThat(result).containsExactly("two\n\ttwo-detail\n");
assertThat(
edges.stream()
- .map(pair -> Preconditions.checkNotNull(pair.getFragment()).toString())
+ .map(pair -> Preconditions.checkNotNull(pair).toString())
.collect(Collectors.toList()))
.containsExactly("one\n one-detail\n", "three\n three-detail")
.inOrder();
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/ByteBufferFragmentTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/FileFragmentTest.java
similarity index 72%
rename from src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/ByteBufferFragmentTest.java
rename to src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/FileFragmentTest.java
index a1346ba..ad7374e 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/ByteBufferFragmentTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/FileFragmentTest.java
@@ -18,20 +18,20 @@
import static com.google.common.truth.Truth.assertThat;
import com.google.common.collect.ImmutableList;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-/** Tests for {@link com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment} */
+/** Tests for {@link FileFragment} */
@RunWith(JUnit4.class)
-public class ByteBufferFragmentTest {
+public class FileFragmentTest {
@Test
public void testMethods() {
final byte[] bytes = "0123456789".getBytes(StandardCharsets.ISO_8859_1);
- ByteBufferFragment fragment = new ByteBufferFragment(ByteBuffer.wrap(bytes), 1, 9);
+ FileFragment fragment = new FileFragment(ByteBuffer.wrap(bytes), 0, 1, 9);
assertThat(fragment.length()).isEqualTo(8);
assertThat(fragment.toString()).isEqualTo("12345678");
assertThat(fragment.subFragment(2, 4).toString()).isEqualTo("34");
@@ -41,12 +41,12 @@
@Test
public void testMerge() {
final byte[] bytes = "0123456789".getBytes(StandardCharsets.ISO_8859_1);
- ByteBufferFragment first = new ByteBufferFragment(ByteBuffer.wrap(bytes), 1, 9);
+ FileFragment first = new FileFragment(ByteBuffer.wrap(bytes), 0, 1, 9);
final byte[] abcBytes = "abcdefg".getBytes(StandardCharsets.ISO_8859_1);
- ByteBufferFragment second = new ByteBufferFragment(ByteBuffer.wrap(abcBytes), 1, 4);
+ FileFragment second = new FileFragment(ByteBuffer.wrap(abcBytes), bytes.length, 1, 4);
- assertThat(ByteBufferFragment.merge(ImmutableList.of(first))).isSameInstanceAs(first);
- ByteBufferFragment merged = ByteBufferFragment.merge(ImmutableList.of(first, second));
+ assertThat(FileFragment.merge(ImmutableList.of(first))).isSameInstanceAs(first);
+ FileFragment merged = FileFragment.merge(ImmutableList.of(first, second));
assertThat(merged.length()).isEqualTo(11);
assertThat(merged.toString()).isEqualTo("12345678bcd");
}
@@ -54,7 +54,7 @@
@Test
public void testEscapeCharacters() {
final byte[] bytes = "\0\n\t\r".getBytes(StandardCharsets.ISO_8859_1);
- ByteBufferFragment fragment = new ByteBufferFragment(ByteBuffer.wrap(bytes), 0, 3);
+ FileFragment fragment = new FileFragment(ByteBuffer.wrap(bytes), 0, 0, 3);
assertThat(fragment.length()).isEqualTo(3);
assertThat(fragment.toString()).isEqualTo("\0\n\t");
assertThat(fragment.subFragment(1, 3).toString()).isEqualTo("\n\t");
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerStepTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerStepTest.java
index 7f007f1..6d8df0b 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerStepTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerStepTest.java
@@ -17,7 +17,7 @@
import static com.google.common.truth.Truth.assertThat;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexerStep;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@@ -198,6 +198,6 @@
private static NinjaLexerStep step(String text) {
ByteBuffer bb = ByteBuffer.wrap(text.getBytes(StandardCharsets.ISO_8859_1));
- return new NinjaLexerStep(new ByteBufferFragment(bb, 0, bb.limit()), 0);
+ return new NinjaLexerStep(new FileFragment(bb, 0, 0, bb.limit()), 0);
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerTest.java
index 7ea35cd..cbc03ec 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerTest.java
@@ -17,7 +17,7 @@
import static com.google.common.truth.Truth.assertThat;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer.TextKind;
import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaToken;
@@ -210,8 +210,7 @@
@Test
public void testZeroByte() {
byte[] bytes = {'a', 0, 'b'};
- NinjaLexer lexer =
- new NinjaLexer(new ByteBufferFragment(ByteBuffer.wrap(bytes), 0, bytes.length));
+ NinjaLexer lexer = new NinjaLexer(new FileFragment(ByteBuffer.wrap(bytes), 0, 0, bytes.length));
assertTokenBytes(lexer, NinjaToken.IDENTIFIER, null);
assertThat(lexer.hasNextToken()).isFalse();
}
@@ -234,6 +233,6 @@
private static NinjaLexer createLexer(String text) {
ByteBuffer buffer = ByteBuffer.wrap(text.getBytes(StandardCharsets.ISO_8859_1));
- return new NinjaLexer(new ByteBufferFragment(buffer, 0, buffer.limit()));
+ return new NinjaLexer(new FileFragment(buffer, 0, 0, buffer.limit()));
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaParserStepTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaParserStepTest.java
index 8e66f3e..296b781 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaParserStepTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaParserStepTest.java
@@ -21,7 +21,7 @@
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Maps;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaFileParseResult;
@@ -458,7 +458,7 @@
private static NinjaParserStep createParser(String text) {
ByteBuffer buffer = ByteBuffer.wrap(text.getBytes(StandardCharsets.ISO_8859_1));
- NinjaLexer lexer = new NinjaLexer(new ByteBufferFragment(buffer, 0, buffer.limit()));
+ NinjaLexer lexer = new NinjaLexer(new FileFragment(buffer, 0, 0, buffer.limit()));
return new NinjaParserStep(lexer);
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaPhonyTargetsUtilTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaPhonyTargetsUtilTest.java
index 059bfea..eafa493 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaPhonyTargetsUtilTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaPhonyTargetsUtilTest.java
@@ -23,7 +23,7 @@
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.bazel.rules.ninja.actions.NinjaPhonyTargetsUtil;
import com.google.devtools.build.lib.bazel.rules.ninja.actions.PhonyTarget;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaParserStep;
@@ -164,7 +164,7 @@
private static NinjaParserStep createParser(String text) {
ByteBuffer buffer = ByteBuffer.wrap(text.getBytes(StandardCharsets.ISO_8859_1));
- NinjaLexer lexer = new NinjaLexer(new ByteBufferFragment(buffer, 0, buffer.limit()));
+ NinjaLexer lexer = new NinjaLexer(new FileFragment(buffer, 0, 0, buffer.limit()));
return new NinjaParserStep(lexer);
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaScopeTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaScopeTest.java
index 099d664..908b6fc 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaScopeTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaScopeTest.java
@@ -21,7 +21,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Maps;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaFileParseResult;
import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaParserStep;
@@ -306,7 +306,7 @@
private static NinjaVariableValue parseValue(String text) throws Exception {
ByteBuffer bb = ByteBuffer.wrap(text.getBytes(StandardCharsets.ISO_8859_1));
- NinjaLexer lexer = new NinjaLexer(new ByteBufferFragment(bb, 0, bb.limit()));
+ NinjaLexer lexer = new NinjaLexer(new FileFragment(bb, 0, 0, bb.limit()));
return new NinjaParserStep(lexer).parseVariableValue();
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaSeparatorFinderTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaSeparatorFinderTest.java
index 9e9f722..ec71a7b 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaSeparatorFinderTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaSeparatorFinderTest.java
@@ -18,7 +18,7 @@
import static com.google.common.truth.Truth.assertThat;
import static com.google.devtools.build.lib.testutil.MoreAsserts.assertThrows;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import com.google.devtools.build.lib.bazel.rules.ninja.file.IncorrectSeparatorException;
import com.google.devtools.build.lib.bazel.rules.ninja.file.NinjaSeparatorFinder;
import java.nio.ByteBuffer;
@@ -61,7 +61,7 @@
// Test for incorrect separators.
byte[] bytes = "a\rb".getBytes(StandardCharsets.ISO_8859_1);
ByteBuffer buffer = ByteBuffer.wrap(bytes);
- ByteBufferFragment fragment = new ByteBufferFragment(buffer, 0, buffer.limit());
+ FileFragment fragment = new FileFragment(buffer, 0, 0, buffer.limit());
assertThrows(
IncorrectSeparatorException.class,
() -> NinjaSeparatorFinder.INSTANCE.findNextSeparator(fragment, 0, -1));
@@ -70,7 +70,7 @@
private static void doTestIsSeparator(String s, int expected) throws IncorrectSeparatorException {
byte[] bytes = s.getBytes(StandardCharsets.ISO_8859_1);
ByteBuffer buffer = ByteBuffer.wrap(bytes);
- ByteBufferFragment fragment = new ByteBufferFragment(buffer, 0, buffer.limit());
+ FileFragment fragment = new FileFragment(buffer, 0, 0, buffer.limit());
int result = NinjaSeparatorFinder.INSTANCE.findNextSeparator(fragment, 0, -1);
assertThat(result).isEqualTo(expected);
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/ParallelFileProcessingTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/ParallelFileProcessingTest.java
index cfe5a49..b1cb73f 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/ParallelFileProcessingTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/ParallelFileProcessingTest.java
@@ -24,8 +24,8 @@
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ByteBufferFragment;
import com.google.devtools.build.lib.bazel.rules.ninja.file.DeclarationConsumer;
+import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
import com.google.devtools.build.lib.bazel.rules.ninja.file.NinjaSeparatorFinder;
import com.google.devtools.build.lib.bazel.rules.ninja.file.ParallelFileProcessing;
@@ -102,12 +102,12 @@
long[] parallel =
nTimesAvg(
() -> {
- List<List<ByteBufferFragment>> list = Lists.newArrayList();
+ List<List<FileFragment>> list = Lists.newArrayList();
Supplier<DeclarationConsumer> factory =
() -> {
- List<ByteBufferFragment> inner = Lists.newArrayList();
+ List<FileFragment> inner = Lists.newArrayList();
list.add(inner);
- return byteFragmentAtOffset -> inner.add(byteFragmentAtOffset.getFragment());
+ return fragment -> inner.add(fragment);
};
parseFile(file, factory, null);
assertThat(list).isNotEmpty();
@@ -186,7 +186,7 @@
List<String> lines = Collections.synchronizedList(Lists.newArrayListWithCapacity(limit));
parseFile(
file,
- () -> (byteFragmentAtOffset) -> lines.add(byteFragmentAtOffset.getFragment().toString()),
+ () -> fragment -> lines.add(fragment.toString()),
new BlockParameters(file.length()).setReadBlockSize(blockSize));
// Copy to non-synchronized list for check
assertNumbers(limit, Lists.newArrayList(lines));
