src/test/java/com/google/devtools/build/lib/remote/ChunkerTest.java

// Copyright 2015 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.remote;

import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;

import com.github.luben.zstd.Zstd;
import com.google.devtools.build.lib.remote.Chunker.Chunk;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
import com.google.protobuf.ByteString;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.NoSuchElementException;
import java.util.Random;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.mockito.Mockito;

/** Tests for {@link Chunker}. */
@RunWith(JUnit4.class)
public class ChunkerTest {

  @Test
  public void chunkingShouldWork() throws IOException {
    Random rand = new Random();
    byte[] expectedData = new byte[21];
    rand.nextBytes(expectedData);

    Chunker chunker = Chunker.builder().setInput(expectedData).setChunkSize(10).build();

    ByteArrayOutputStream actualData = new ByteArrayOutputStream();

    assertThat(chunker.hasNext()).isTrue();
    Chunk next = chunker.next();
    assertThat(next.getOffset()).isEqualTo(0);
    assertThat(next.getData()).hasSize(10);
    next.getData().writeTo(actualData);

    assertThat(chunker.hasNext()).isTrue();
    next = chunker.next();
    assertThat(next.getOffset()).isEqualTo(10);
    assertThat(next.getData()).hasSize(10);
    next.getData().writeTo(actualData);

    assertThat(chunker.hasNext()).isTrue();
    next = chunker.next();
    assertThat(next.getOffset()).isEqualTo(20);
    assertThat(next.getData()).hasSize(1);
    next.getData().writeTo(actualData);

    assertThat(chunker.hasNext()).isFalse();

    assertThat(actualData.toByteArray()).isEqualTo(expectedData);
  }

  @Test
  public void nextShouldThrowIfNoMoreData() throws IOException {
    byte[] data = new byte[10];
    Chunker chunker = Chunker.builder().setInput(data).setChunkSize(10).build();

    assertThat(chunker.hasNext()).isTrue();
    assertThat(chunker.next()).isNotNull();

    assertThat(chunker.hasNext()).isFalse();

    assertThrows(NoSuchElementException.class, () -> chunker.next());
  }

  @Test
  public void emptyData() throws Exception {
    var inp =
        new ByteArrayInputStream(new byte[0]) {
          private boolean closed;

          @Override
          public void close() throws IOException {
            closed = true;
            super.close();
          }
        };
    Chunker chunker = Chunker.builder().setInput(0, () -> inp).build();

    assertThat(chunker.hasNext()).isTrue();

    Chunk next = chunker.next();

    assertThat(next).isNotNull();
    assertThat(next.getData()).isEmpty();
    assertThat(next.getOffset()).isEqualTo(0);

    assertThat(chunker.hasNext()).isFalse();
    assertThat(inp.closed).isTrue();

    assertThrows(NoSuchElementException.class, () -> chunker.next());
  }

  @Test
  public void reset() throws Exception {
    byte[] data = new byte[] {1, 2, 3};
    Chunker chunker = Chunker.builder().setInput(data).setChunkSize(1).build();

    assertNextEquals(chunker, (byte) 1);
    assertNextEquals(chunker, (byte) 2);

    chunker.reset();

    assertNextEquals(chunker, (byte) 1);
    assertNextEquals(chunker, (byte) 2);
    assertNextEquals(chunker, (byte) 3);

    chunker.reset();

    assertNextEquals(chunker, (byte) 1);
  }

  @Test
  public void resourcesShouldBeReleased() throws IOException {
    // Test that after having consumed all data or after reset() is called (whatever happens first)
    // the underlying InputStream should be closed.

    byte[] data = new byte[] {1, 2};
    final AtomicReference<InputStream> in = new AtomicReference<>();
    RemoteCacheClient.Blob supplier =
        () -> {
          in.set(Mockito.spy(new ByteArrayInputStream(data)));
          return in.get();
        };

    Chunker chunker = new Chunker(supplier, data.length, 1, false);
    assertThat(in.get()).isNull();
    assertNextEquals(chunker, (byte) 1);
    Mockito.verify(in.get(), Mockito.never()).close();
    assertNextEquals(chunker, (byte) 2);
    Mockito.verify(in.get()).close();

    chunker.reset();
    chunker.next();
    chunker.reset();
    Mockito.verify(in.get()).close();
  }

  @Test
  public void seekAfterReset() throws IOException {
    // Test that seek() works on an uninitialized chunker

    byte[] data = new byte[10];
    Chunker chunker = Chunker.builder().setInput(data).setChunkSize(10).build();

    chunker.reset();
    chunker.seek(2);

    Chunk next = chunker.next();
    assertThat(next).isNotNull();
    assertThat(next.getOffset()).isEqualTo(2);
    assertThat(next.getData()).hasSize(8);
  }

  @Test
  public void seekBackwards() throws IOException {
    byte[] data = new byte[10];
    Chunker chunker = Chunker.builder().setInput(data).setChunkSize(10).build();

    chunker.seek(4);
    chunker.seek(2);

    Chunk next = chunker.next();
    assertThat(next).isNotNull();
    assertThat(next.getOffset()).isEqualTo(2);
    assertThat(next.getData()).hasSize(8);
  }

  @Test
  public void seekForwards() throws IOException {
    byte[] data = new byte[10];
    for (byte i = 0; i < data.length; i++) {
      data[i] = i;
    }
    Chunker chunker = Chunker.builder().setInput(data).setChunkSize(2).build();

    var chunk = chunker.next();
    assertThat(chunk.getOffset()).isEqualTo(0);
    assertThat(chunk.getData().toByteArray()).isEqualTo(new byte[] {0, 1});
    chunker.seek(8);
    chunk = chunker.next();
    assertThat(chunk.getOffset()).isEqualTo(8);
    assertThat(chunk.getData().toByteArray()).isEqualTo(new byte[] {8, 9});
    assertThat(chunker.hasNext()).isFalse();
  }

  @Test
  public void seekEmptyData() throws IOException {
    for (boolean compressed : new boolean[] {false, true}) {
      try (var chunker =
          Chunker.builder().setInput(new byte[0]).setCompressed(compressed).build()) {
        chunker.seek(0);
        var next = chunker.next();
        assertThat(next).isNotNull();
        assertThat(next.getData()).isEmpty();
        assertThat(next.getOffset()).isEqualTo(0);

        assertThat(chunker.hasNext()).isFalse();
        assertThrows(NoSuchElementException.class, chunker::next);
      }
    }
  }

  @Test
  public void testSingleChunkCompressed() throws IOException {
    byte[] data = {72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33};
    Chunker chunker =
        Chunker.builder().setInput(data).setChunkSize(data.length * 2).setCompressed(true).build();
    Chunk next = chunker.next();
    assertThat(chunker.hasNext()).isFalse();
    assertThat(Zstd.decompress(next.getData().toByteArray(), data.length)).isEqualTo(data);
  }

  @Test
  public void testMultiChunkCompressed() throws IOException {
    byte[] data = {72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33};
    Chunker chunker =
        Chunker.builder().setInput(data).setChunkSize(data.length / 2).setCompressed(true).build();

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    chunker.next().getData().writeTo(baos);
    assertThat(chunker.hasNext()).isTrue();
    while (chunker.hasNext()) {
      chunker.next().getData().writeTo(baos);
    }
    baos.close();

    assertThat(Zstd.decompress(baos.toByteArray(), data.length)).isEqualTo(data);
  }

  @Test
  public void compressedLargeIncompressibleDataHasContiguousChunkOffsets() throws IOException {
    byte[] data = new byte[1024 * 1024 + 17];
    new Random(0).nextBytes(data);
    try (Chunker chunker =
        Chunker.builder().setInput(data).setChunkSize(16 * 1024).setCompressed(true).build()) {
      byte[] compressed = collectChunks(chunker, 0, 16 * 1024);

      assertThat(compressed.length).isGreaterThan(16 * 1024);
      assertThat(Zstd.decompress(compressed, data.length)).isEqualTo(data);
    }
  }

  @Test
  public void compressedChunkDoesNotConsumeEntireHighlyCompressibleSource() throws IOException {
    long size = 64L * 1024 * 1024;
    var source =
        new InputStream() {
          long consumed;

          @Override
          public int read() {
            if (consumed == size) {
              return -1;
            }
            consumed++;
            return 0;
          }

          @Override
          public int read(byte[] b, int off, int len) {
            if (consumed == size) {
              return -1;
            }
            int n = (int) Math.min(len, size - consumed);
            Arrays.fill(b, off, off + n, (byte) 0);
            consumed += n;
            return n;
          }
        };
    try (Chunker chunker =
        Chunker.builder().setInput(size, () -> source).setCompressed(true).build()) {
      Chunk first = chunker.next();
      assertThat(first.getOffset()).isEqualTo(0);
      assertThat(first.getData().size()).isGreaterThan(0);
      assertThat(first.getData().size()).isLessThan(16 * 1024);
      assertThat(source.consumed).isAtMost(2L * 1024 * 1024);
      assertThat(chunker.hasNext()).isTrue();
    }
  }

  @Test
  public void compressedResetReproducesStreamAfterPartialAndCompleteReads() throws IOException {
    byte[] data = new byte[1024 * 1024 + 17];
    new Random(0).nextBytes(data);
    try (Chunker chunker =
        Chunker.builder().setInput(data).setChunkSize(16 * 1024).setCompressed(true).build()) {
      byte[] expected = collectChunks(chunker, 0, 16 * 1024);
      chunker.reset();
      Chunk first = chunker.next();
      assertThat(first.getOffset()).isEqualTo(0);
      assertThat(chunker.hasNext()).isTrue();
      chunker.reset();

      assertThat(collectChunks(chunker, 0, 16 * 1024)).isEqualTo(expected);
      chunker.reset();
      assertThat(collectChunks(chunker, 0, 16 * 1024)).isEqualTo(expected);
    }
  }

  @Test
  public void compressedSeekStartsFreshFrameAtUncompressedOffset() throws IOException {
    byte[] data = new byte[1024 * 1024 + 17];
    new Random(0).nextBytes(data);
    int resumeOffset = 135791;
    try (Chunker chunker =
        Chunker.builder().setInput(data).setChunkSize(16 * 1024).setCompressed(true).build()) {
      chunker.next();
      chunker.seek(resumeOffset);
      byte[] compressedSuffix = collectChunks(chunker, resumeOffset, 16 * 1024);

      assertThat(Zstd.decompress(compressedSuffix, data.length - resumeOffset))
          .isEqualTo(Arrays.copyOfRange(data, resumeOffset, data.length));
    }
  }

  @Test
  public void testActualSizeIsCorrectAfterSeek() throws IOException {
    byte[] data = {72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33};
    for (boolean compressed : new boolean[] {false, true}) {
      try (Chunker chunker =
          Chunker.builder()
              .setInput(data)
              .setChunkSize(data.length * 2)
              .setCompressed(compressed)
              .build()) {
        chunker.seek(5);
        byte[] actual = collectChunks(chunker, 5, data.length * 2);
        byte[] suffix = Arrays.copyOfRange(data, 5, data.length);
        assertThat(compressed ? Zstd.decompress(actual, suffix.length) : actual).isEqualTo(suffix);
      }
    }
  }

  private byte[] collectChunks(Chunker chunker, long initialOffset, int chunkSize)
      throws IOException {
    ByteArrayOutputStream data = new ByteArrayOutputStream();
    long expectedOffset = initialOffset;
    while (chunker.hasNext()) {
      Chunk next = chunker.next();
      assertThat(next.getOffset()).isEqualTo(expectedOffset);
      assertThat(next.getData().size()).isAtMost(chunkSize);
      next.getData().writeTo(data);
      expectedOffset += next.getData().size();
    }
    assertThat(chunker.getOffset()).isEqualTo(expectedOffset);
    return data.toByteArray();
  }

  private void assertNextEquals(Chunker chunker, byte... data) throws IOException {
    assertThat(chunker.hasNext()).isTrue();
    ByteString next = chunker.next().getData();
    assertThat(next.toByteArray()).isEqualTo(data);
  }
}