src/tools/singlejar/transient_bytes.h

// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef SRC_TOOLS_SINGLEJAR_TRANSIENT_BYTES_H_
#define SRC_TOOLS_SINGLEJAR_TRANSIENT_BYTES_H_

#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS 1
#endif

#include <inttypes.h>
#include <algorithm>
#include <ostream>

#include "src/tools/singlejar/diag.h"
#include "src/tools/singlejar/zip_headers.h"
#include "src/tools/singlejar/zlib_interface.h"

/*
 * An instance of this class holds decompressed data in a list of chunks,
 * to be eventually compressed to the output buffer.
 * Use DecompressFile() or ReadFile() (depending on whether an entry is
 * compressed or not) to append the contents of a Zip entry.
 * Use Append() to append a sequence of bytes or a string.
 * Use Write() to write out the contents, it will compress the entry if
 * necessary.
 */
class TransientBytes {
 public:
  TransientBytes()
      : allocated_(0),
        data_size_(0),
        first_block_(nullptr),
        last_block_(nullptr) {}

  ~TransientBytes() {
    while (first_block_) {
      auto block = first_block_;
      first_block_ = first_block_->next_block_;
      delete block;
    }
    last_block_ = nullptr;
  }

  // Appends raw bytes.
  void Append(const uint8_t *data, uint64_t data_size) {
    uint64_t chunk_size;
    auto data_end = data + data_size;
    for (; data < data_end; data += chunk_size) {
      chunk_size =
          std::min(static_cast<uint64_t>(data_end - data), ensure_space());
      copy(data, chunk_size);
    }
  }

  // Same, but for a string.
  void Append(const char *str) {
    Append(reinterpret_cast<const uint8_t *>(str), strlen(str));
  }

  // Appends the contents of the uncompressed Zip entry.
  void ReadEntryContents(const LH *lh) {
    Append(lh->data(), lh->uncompressed_file_size());
  }

  // Appends the contents of the compressed Zip entry. Resets the inflater
  // used to decompress.
  void DecompressEntryContents(const CDH *cdh, const LH *lh,
                               Inflater *inflater) {
    uint64_t old_total_out = inflater->total_out();
    uint64_t in_bytes;
    uint64_t out_bytes;
    const uint8_t *data = lh->data();

    if (cdh->no_size_in_local_header()) {
      in_bytes = cdh->compressed_file_size();
      out_bytes = cdh->uncompressed_file_size();
    } else {
      in_bytes = lh->compressed_file_size();
      out_bytes = lh->uncompressed_file_size();
    }

    while (in_bytes > 0) {
      // A single region to inflate cannot exceed 4GB-1.
      uint32_t in_bytes_chunk = 0xFFFFFFFF;
      if (in_bytes_chunk > in_bytes) {
        in_bytes_chunk = in_bytes;
      }
      inflater->DataToInflate(data, in_bytes_chunk);
      for (;;) {
        uint32_t available_out = ensure_space();
        int ret = inflater->Inflate(append_position(), available_out);
        uint32_t inflated = available_out - inflater->available_out();
        if (Z_STREAM_END == ret) {
          // No more data to decompress. Update write position and we are done
          // for this input chunk.
          advance(inflated);
          break;
        } else if (Z_OK == ret) {
          // No more space in the output buffer. Advance write position, update
          // the number of remaining bytes.
          if (inflater->available_out()) {
            diag_errx(2,
                      "%s:%d: Internal error inflating %.*s: Inflate reported "
                      "Z_OK but there are still %" PRIu32
                      " bytes available in the output buffer",
                      __FILE__, __LINE__, lh->file_name_length(),
                      lh->file_name(), inflater->available_out());
          }
          advance(inflated);
        } else {
          diag_errx(2,
                    "%s:%d: Internal error inflating %.*s: inflate() call "
                    "returned %d (%s)",
                    __FILE__, __LINE__, lh->file_name_length(), lh->file_name(),
                    ret, inflater->error_message());
        }
      }
      data += in_bytes_chunk;
      in_bytes -= in_bytes_chunk;
    }

    // Smog check
    // This check is disabled on Windows because z_stream::total_out is of type
    // of uLong (unsigned long), which is 64-bit for most 64-bit Unix platforms,
    // but it is 32-bit even for Win64. This means even though zlib is capable
    // of compressing data >4GB as long as it is processed by chunks, zlib
    // cannot report the correct total number of processed bytes >4GB through
    // z_stream::total_out on Windows.
#ifndef _WIN32
    if (inflater->total_out() - old_total_out != out_bytes) {
      diag_errx(2,
                "%s:%d: Internal error inflating %.*s: inflater wrote %" PRIu64
                " bytes , but the uncompressed entry should be %" PRIu64
                "bytes long",
                __FILE__, __LINE__, lh->file_name_length(), lh->file_name(),
                inflater->total_out() - old_total_out, out_bytes);
    }
#endif
    inflater->reset();
  }

  // Writes the contents bytes to the given buffer in an optimal way, i.e., the
  // shorter of compressed or uncompressed. Sets the checksum and number of
  // bytes written and returns Z_DEFLATED if compression took place or
  // Z_NO_COMPRESSION otherwise.
  uint16_t CompressOut(uint8_t *buffer, uint32_t *checksum,
                       uint64_t *bytes_written) {
    *checksum = 0;
    uint64_t to_compress = data_size();
    if (to_compress == 0) {
      *bytes_written = 0;
      return Z_NO_COMPRESSION;
    }

    Deflater deflater;
    deflater.next_out = buffer;
    uint16_t compression_method = Z_DEFLATED;

    // Feed data blocks to the deflater one by one, but break if the compressed
    // size exceeds the original size.
    for (auto data_block = first_block_;
         data_block && compression_method != Z_NO_COMPRESSION;
         data_block = data_block->next_block_) {
      // The compressed size should not exceed the original size less the number
      // of bytes already compressed. And, it should not exceed 4GB-1.
      deflater.avail_out = std::min(data_size() - deflater.total_out,
                                    static_cast<uint64_t>(0xFFFFFFFF));
      // Out of the total number of bytes that remain to be compressed, we
      // can compress no more than this block.
      uint32_t chunk_size = static_cast<uint32_t>(std::min(
          static_cast<uint64_t>(sizeof(data_block->data_)), to_compress));
      *checksum = crc32(*checksum, data_block->data_, chunk_size);
      deflater.avail_in = chunk_size;
      to_compress -= chunk_size;
      int ret = deflater.Deflate(data_block->data_, chunk_size,
                                 to_compress ? Z_NO_FLUSH : Z_FINISH);
      if (ret == Z_OK) {
        if (!deflater.avail_out) {
          // We ran out of space in the output buffer, which means
          // that deflated size exceeds original size. Leave the loop
          // and just copy the data.
          compression_method = Z_NO_COMPRESSION;
        }
      } else if (ret == Z_BUF_ERROR && !deflater.avail_in) {
        // We ran out of data block, this is not a error.
      } else if (ret == Z_STREAM_END) {
        if (data_block->next_block_ || to_compress) {
          diag_errx(2,
                    "%s:%d: Internal error: deflate() call at the end, but "
                    "there is more data to compress!",
                    __FILE__, __LINE__);
        }
      } else {
        diag_errx(2, "%s:%d: deflate error %d(%s)", __FILE__, __LINE__, ret,
                  deflater.msg);
      }
    }
    if (compression_method != Z_NO_COMPRESSION) {
      *bytes_written = deflater.total_out;
      return compression_method;
    }

    // Compression does not help, just copy the bytes to the output buffer.
    CopyOut(buffer, checksum);
    *bytes_written = data_size();
    return Z_NO_COMPRESSION;
  }

  // Copies the bytes to the buffer and sets the checksum.
  void CopyOut(uint8_t *buffer, uint32_t *checksum) {
    uint64_t to_copy = data_size();
    uint8_t *buffer_end = buffer + to_copy;
    *checksum = 0;
    for (auto data_block = first_block_; data_block;
         data_block = data_block->next_block_) {
      size_t chunk_size =
          std::min(static_cast<uint64_t>(sizeof(data_block->data_)), to_copy);
      *checksum = crc32(*checksum, data_block->data_, chunk_size);
      memcpy(buffer_end - to_copy, data_block->data_, chunk_size);
      to_copy -= chunk_size;
    }
  }

  // Number of data bytes.
  uint64_t data_size() const { return data_size_; }

  // This is mostly for testing: stream out contents to a Sink instance.
  // The class Sink has to have
  //     void operator()(const void *chunk, uint64_t chunk_size) const;
  //
  template <class Sink>
  void stream_out(const Sink &sink) const {
    uint64_t to_copy = data_size();
    for (auto data_block = first_block_; data_block;
         data_block = data_block->next_block_) {
      uint64_t chunk_size = sizeof(data_block->data_);
      if (chunk_size > to_copy) {
        chunk_size = to_copy;
      }
      sink.operator()(data_block->data_, chunk_size);
      to_copy -= chunk_size;
    }
  }

  uint8_t last_byte() const {
    if (!data_size()) {
      diag_errx(1, "%s:%d: last_char() cannot be called if buffer is empty",
                __FILE__, __LINE__);
    }
    if (free_size() >= sizeof(last_block_->data_)) {
      diag_errx(1, "%s:%d: internal error: the last data block is empty",
                __FILE__, __LINE__);
    }
    return *(last_block_->End() - free_size() - 1);
  }

 private:
  // Ensures there is some space to write to, returns the amount available.
  uint64_t ensure_space() {
    if (!free_size()) {
      auto *data_block = new DataBlock();
      if (last_block_) {
        last_block_->next_block_ = data_block;
      }
      last_block_ = data_block;
      if (!first_block_) {
        first_block_ = data_block;
      }
      allocated_ += sizeof(data_block->data_);
    }
    return free_size();
  }

  // Records that given amount of bytes is to be appended to the buffer.
  // Returns the old write position.
  uint8_t *advance(size_t amount) {
    if (amount > free_size()) {
      diag_errx(
          2, "%s: %d: Cannot advance %zu bytes, only %" PRIu64 " is available",
          __FILE__, __LINE__, amount, free_size());
    }
    uint8_t *pos = append_position();
    data_size_ += amount;
    return pos;
  }

  void copy(const uint8_t *from, size_t count) {
    memcpy(advance(count), from, count);
  }

  uint8_t *append_position() {
    return last_block_ ? last_block_->End() - free_size() : nullptr;
  }

  // Returns the amount of free space.
  uint64_t free_size() const { return allocated_ - data_size_; }

  // The bytes are kept in an linked list of the DataBlock instances.
  // TODO(asmundak): perhaps use mmap to allocate these?
  struct DataBlock {
    struct DataBlock *next_block_;
    uint8_t data_[0x40000 - 8];
    DataBlock() : next_block_(nullptr) {}
    uint8_t *End() { return data_ + sizeof(data_); }
  };

  uint64_t allocated_;
  uint64_t data_size_;
  struct DataBlock *first_block_;
  struct DataBlock *last_block_;
};

#endif  // SRC_TOOLS_SINGLEJAR_TRANSIENT_BYTES_H_