Move the parallel dexing tools to the Bazel tree.
--
MOS_MIGRATED_REVID=95278949
diff --git a/src/main/java/BUILD b/src/main/java/BUILD
index f397173..5d9c44e 100644
--- a/src/main/java/BUILD
+++ b/src/main/java/BUILD
@@ -60,6 +60,7 @@
java_library(
name = "options",
srcs = glob(["com/google/devtools/common/options/*.java"]),
+ visibility = ["//visibility:public"],
deps = [
"//third_party:guava",
"//third_party:jsr305",
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/BUILD b/src/tools/android/java/com/google/devtools/build/android/ziputils/BUILD
new file mode 100644
index 0000000..f99ea5e
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/BUILD
@@ -0,0 +1,65 @@
+# Low level zip archive processing library.
+
+package(
+ default_visibility = ["//visibility:public"],
+)
+
+java_library(
+ name = "ziputils_lib",
+ srcs = glob(
+ ["*.java"],
+ exclude = [
+ "DexMapper.java",
+ "DexReducer.java",
+ "SplitZip.java",
+ "Splitter.java",
+ ],
+ ),
+ visibility = ["//visibility:public"],
+ deps = [
+ "//third_party:guava",
+ "//third_party:jsr305",
+ ],
+)
+
+java_library(
+ name = "splitter_lib",
+ srcs = glob(
+ [
+ "SplitZip.java",
+ "Splitter.java",
+ ],
+ ),
+ visibility = ["//visibility:public"],
+ deps = [
+ ":ziputils_lib",
+ "//third_party:guava",
+ "//third_party:jsr305",
+ ],
+)
+
+java_binary(
+ name = "mapper",
+ srcs = [
+ "DexMapper.java",
+ ],
+ main_class = "com.google.devtools.build.android.ziputils.DexMapper",
+ visibility = ["//visibility:public"],
+ deps = [
+ ":splitter_lib",
+ "//src/main/java:options",
+ ],
+)
+
+java_binary(
+ name = "reducer",
+ srcs = [
+ "DexReducer.java",
+ ],
+ main_class = "com.google.devtools.build.android.ziputils.DexReducer",
+ visibility = ["//visibility:public"],
+ deps = [
+ ":ziputils_lib",
+ "//src/main/java:options",
+ ],
+)
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/BufferedFile.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/BufferedFile.java
new file mode 100644
index 0000000..1977254
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/BufferedFile.java
@@ -0,0 +1,163 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import com.google.common.base.Preconditions;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+
+/**
+ * An API for reading big files through a direct byte buffer spanning a region of the file.
+ * This object maintains an internal buffer, which may store all or some of the file content.
+ * When a request for data is made ({@link #getBuffer(long, int) }, the implementation will
+ * first determine if the requested data range is within the region specified at time of
+ * construction. If it is, it checks to see if the request is within the capacity range of
+ * the current internal buffer. If not, the buffer is reallocated, based at the requested offset.
+ * Then the implementation checks to see if the requested data falls within the current fill limit
+ * of the internal buffer. If not additional data is read from the file. Finally, a slice of
+ * the internal buffer is returned, with the requested data.
+ *
+ * <p>This is optimized for forward scanning of files. Random access is supported, but will likely
+ * be inefficient, especially if the entire file doesn't fit in the internal buffer.
+ *
+ * <p>Clients of this API should take care not to keep references to returned buffers indefinitely,
+ * as this would prevent collection of buffers discarded by the {@code BufferedFile} object.
+ */
+public class BufferedFile {
+
+ private int maxAlloc;
+ private long offset;
+ private long limit;
+ private FileChannel channel;
+ private ByteBuffer current;
+ private long currOff;
+
+ /**
+ * Same as {@code BufferedFile(channel, 0, channel.size(), blockSize)}.
+ *
+ * @param channel file channel opened for reading.
+ * @param blockSize maximum buffer allocation.
+ * @throws NullPointerException if {@code channel} is {@code null}.
+ * @throws IllegalArgumentException if {@code maxAlloc}, {@code off}, or {@code len} are negative
+ * or if {@code off + len > channel.size()}.
+ * @throws IOException
+ */
+ public BufferedFile(FileChannel channel, int blockSize) throws IOException {
+ this(channel, 0, channel.size(), blockSize);
+ }
+
+ /**
+ * Allocates a buffered file.
+ *
+ * @param channel file channel opened for reading.
+ * @param off the first byte that can be read through this object.
+ * @param len the max number of bytes that can be read through this object.
+ * @param blockSize default max buffer allocation size is {@code Math.min(blockSize, len)}.
+ * @throws NullPointerException if {@code channel} is {@code null}.
+ * @throws IllegalArgumentException if {@code blockSize}, {@code off}, or {@code len} are negative
+ * or if {@code off + len > channel.size()}.
+ * @throws IOException if thrown by the underlying file channel.
+ */
+ public BufferedFile(FileChannel channel, long off, long len, int blockSize) throws IOException {
+ Preconditions.checkNotNull(channel);
+ Preconditions.checkArgument(blockSize >= 0);
+ Preconditions.checkArgument(off >= 0);
+ Preconditions.checkArgument(len >= 0);
+ Preconditions.checkArgument(off + len <= channel.size());
+ this.maxAlloc = (int) Math.min(blockSize, len);
+ this.offset = off;
+ this.limit = off + len;
+ this.channel = channel;
+ this.current = null;
+ currOff = -1;
+ }
+
+ /**
+ * Returns the offset of the first byte beyond the readable region.
+ * @return the file offset just beyond the readable region.
+ */
+ public long limit() {
+ return limit;
+ }
+
+ /**
+ * Returns a byte buffer for reading {@code len} bytes from the {@code off} position
+ * in the file. If the requested bytes are already loaded in the internal buffer, a slice is
+ * returned, with position 0 and limit set to {@code len}. The slice may have a capacity greater
+ * than its limit, if more bytes are already available in the internal buffer. If the requested
+ * bytes are not available, but can fit in the current internal buffer, then more data is read,
+ * before a slice is created as described above. If the requested data falls outside the range
+ * that can be fitted into the current internal buffer, then a new internal buffer is allocated.
+ * The prior internal buffer (if any), is no longer referenced by this object (but it may still
+ * be referenced by the client, holding references to byte buffers returned from prior call to
+ * this method). The new internal buffer will be based at {@code off} file position, and have a
+ * capacity equal to the maximum of the {@code blockSize} of this buffer and {@code len}, except
+ * that it will never exceed the the number of bytes from {@code off} to the end of the readable
+ * region of the file (min-max rule).
+ *
+ * @param off
+ * @param len
+ * @return a slice of the internal byte buffer containing the requested data. Except, if the
+ * client request data beyond the readable region of the file, the {@code len} value is reduced
+ * to the maximum number of bytes available from the given {@code off}.
+ * @throws IllegalArgumentException if {@code len} is less than 0, or {@code off} is outside the
+ * readable region specified when constructing this object.
+ * @throws IOException if thrown by the underlying file channel.
+ */
+ public synchronized ByteBuffer getBuffer(long off, int len) throws IOException {
+ Preconditions.checkArgument(off >= offset);
+ Preconditions.checkArgument(len >= 0);
+ Preconditions.checkArgument(off < limit || (off == limit && len == 0));
+ if (limit - off < len) { // never return data beyond limit
+ len = (int) (limit - off);
+ }
+ Preconditions.checkState(off + len <= limit);
+ if (current == null || off < currOff || off + len > currOff + current.capacity()) {
+ allocate(off, len);
+ Preconditions.checkState(current != null && off == currOff
+ && off + len <= currOff + current.capacity());
+ }
+ Preconditions.checkState(current != null && off >= currOff
+ && off + len <= currOff + current.capacity());
+ if (off - currOff + len > current.limit()) {
+ readMore((int) (off - currOff) + len);
+ }
+ Preconditions.checkState(current != null && off >= currOff
+ && off + len <= currOff + current.limit());
+ current.position((int) (off - currOff));
+ return (ByteBuffer) current.slice().limit(len);
+ }
+
+ private void readMore(int newMin) throws IOException {
+ channel.position(currOff + current.limit());
+ current.position(current.limit());
+ current.limit(current.capacity());
+ do {
+ channel.read(current);
+ } while(current.position() < newMin);
+ current.limit(current.position()).position(0);
+ }
+
+ private void allocate(long off, int len) {
+ current = ByteBuffer.allocateDirect(bufferSize(off, len));
+ current.limit(0);
+ currOff = off;
+ }
+
+ private int bufferSize(long off, int len) {
+ return (int) Math.min(Math.max(len, maxAlloc), limit - off);
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/CentralDirectory.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/CentralDirectory.java
new file mode 100644
index 0000000..c7c2ff6
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/CentralDirectory.java
@@ -0,0 +1,165 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENOFF;
+
+import com.google.common.base.Preconditions;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+/**
+ * Provides a view of a zip file's central directory. For reading, a single memory mapped view is
+ * used. For writing, the central directory is stored as one or more views, each backed by a direct
+ * byte buffer.
+ */
+public class CentralDirectory extends View<CentralDirectory> {
+
+ // Cached map from entry name to directory entry.
+ private NavigableMap<String, DirectoryEntry> mapByNameSorted;
+ // Cached map from entry file offset to directory entry.
+ private NavigableMap<Integer, DirectoryEntry> mapByOffsetSorted;
+ // Number of directory entries in this view.
+ private int count;
+ // Parsed or added entries
+ private List<DirectoryEntry> entries;
+
+ /**
+ * Gets the number of directory entries in this view.
+ */
+ public int getCount() {
+ return count;
+ }
+
+ /**
+ * Returns a list of directory entries, in the order they occur in the central directory.
+ * This will typically also be the order of entries in the zip file, although that's not
+ * guaranteed.
+ */
+ public List<DirectoryEntry> list() {
+ return entries;
+ }
+
+ /**
+ * Returns a navigable map of directory entries, by zip entry file offset.
+ */
+ public NavigableMap<Integer, DirectoryEntry> mapByOffset() {
+ if (entries == null) {
+ return null;
+ }
+ return mapEntriesByOffset();
+ }
+
+ /**
+ * Returns a navigable map of directory entries, by entry filename.
+ */
+ public NavigableMap<String, DirectoryEntry> mapByFilename() {
+ if (entries == null) {
+ return null;
+ }
+ return mapEntriesByName();
+ }
+
+ /**
+ * Returns a {@code CentralDirectory} of the given buffer. This may be a full or a partial
+ * central directory. This method assumes ownership of the underlying buffer. Unlike most
+ * "view-of" methods, this method doesn't slice the argument buffer, and rather than advancing
+ * the buffer position, it sets it to 0.
+ *
+ * @param buffer containing data of a central directory.
+ * @return a {@code CentralDirectory} of the data at the current position of the given byte
+ * buffer.
+ */
+ public static CentralDirectory viewOf(ByteBuffer buffer) {
+ buffer.position(0);
+ return new CentralDirectory(buffer);
+ }
+
+ private CentralDirectory(ByteBuffer buffer) {
+ super(buffer);
+ count = -1;
+ }
+
+ /**
+ * Parses this central directory, and maps the contained entries with {@link DirectoryEntry}s.
+ *
+ * @return this central directory view
+ * @throws IllegalStateException if the file offset is not set prior to parsing
+ */
+ public CentralDirectory parse() throws IllegalStateException {
+ Preconditions.checkState(fileOffset != -1, "File offset not set prior to parsing");
+ count = 0;
+ clearMaps();
+ int relPos = 0;
+ buffer.position(0);
+ while (buffer.hasRemaining() && buffer.getInt(buffer.position()) == DirectoryEntry.SIGNATURE) {
+ count++;
+ DirectoryEntry entry = DirectoryEntry.viewOf(buffer).at(fileOffset + relPos);
+ entries.add(entry);
+ relPos += entry.getSize();
+ buffer.position(relPos);
+ }
+ return this;
+ }
+
+ /**
+ * Creates a new directory entry for output. The given entry is copied into the buffer of this
+ * central directory, and a view of the copied data is returned.
+ *
+ * @param entry prototype directory entry, typically an entry read from another zip file, for
+ * an entry being copied.
+ * @return a directory entry view of the copied entry.
+ */
+ public DirectoryEntry nextEntry(DirectoryEntry entry) {
+ DirectoryEntry clone = entry.copy(buffer);
+ if (count == -1) {
+ clearMaps();
+ count = 1;
+ } else {
+ count++;
+ }
+ entries.add(clone);
+ return clone;
+ }
+
+ private NavigableMap<String, DirectoryEntry> mapEntriesByName() {
+ if (mapByNameSorted == null) {
+ mapByNameSorted = new TreeMap<>();
+ for (DirectoryEntry entry : entries) {
+ mapByNameSorted.put(entry.getFilename(), entry);
+ }
+ }
+ return mapByNameSorted;
+ }
+
+ private NavigableMap<Integer, DirectoryEntry> mapEntriesByOffset() {
+ if (mapByOffsetSorted == null) {
+ mapByOffsetSorted = new TreeMap<>();
+ for (DirectoryEntry entry : entries) {
+ mapByOffsetSorted.put(entry.get(CENOFF), entry);
+ }
+ }
+ return mapByOffsetSorted;
+ }
+
+ private void clearMaps() {
+ entries = new ArrayList<>();
+ mapByOffsetSorted = null;
+ mapByNameSorted = null;
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/DataDescriptor.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/DataDescriptor.java
new file mode 100644
index 0000000..61f3907
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/DataDescriptor.java
@@ -0,0 +1,162 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import static java.nio.ByteOrder.LITTLE_ENDIAN;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Provides a view of a data descriptor record, for a zip file entry.
+ */
+public class DataDescriptor extends View<DataDescriptor> {
+ private boolean hasMarker;
+
+ /**
+ * Returns a {@code DataDescriptor} view of the given buffer. The buffer is assumed to contain a
+ * valid "data descriptor" record beginning at the buffers current position.
+ *
+ * @param buffer containing the data of a data descriptor record.
+ * @return a {@code DataDescriptor} of the data at the current position of the given byte
+ * buffer.
+ */
+ public static DataDescriptor viewOf(ByteBuffer buffer) {
+ DataDescriptor view = new DataDescriptor(buffer.slice());
+ int size = view.getSize();
+ view.buffer.position(0).limit(size);
+ buffer.position(buffer.position() + size);
+ return view;
+ }
+
+ private DataDescriptor(ByteBuffer buffer) {
+ super(buffer);
+ hasMarker = buffer.getInt(0) == SIGNATURE;
+ }
+
+ /**
+ * Creates a {@code DataDescriptor} with a heap allocated buffer.
+ *
+ * @return a {@code DataDescriptor} with a heap allocated buffer.
+ */
+ public static DataDescriptor allocate() {
+ ByteBuffer buffer = ByteBuffer.allocate(SIZE).order(LITTLE_ENDIAN);
+ return new DataDescriptor(buffer).init();
+ }
+
+ /**
+ * Creates a {@code DataDescriptor} view over a writable buffer. The given buffers position is
+ * advanced by the number of bytes consumed by the view.
+ *
+ * @param buffer buffer to hold data for the "data descriptor" record.
+ * @return a {@code DataDescriptor} with a heap allocated buffer.
+ */
+ public static DataDescriptor view(ByteBuffer buffer) {
+ DataDescriptor view = new DataDescriptor(buffer.slice()).init();
+ buffer.position(buffer.position() + SIZE);
+ return view;
+ }
+
+ /**
+ * Copies this {@code DataDescriptor} into a writable buffer. The copy is made at the
+ * buffer's current position, and the position is advanced, by the size of the copy.
+ *
+ * @param buffer buffer to hold data for the copy.
+ * @return a {@code DataDescriptor} backed by the given buffer.
+ */
+ public DataDescriptor copy(ByteBuffer buffer) {
+ int size = getSize();
+ DataDescriptor view = new DataDescriptor(buffer.slice());
+ this.buffer.rewind();
+ view.buffer.put(this.buffer).flip();
+ buffer.position(buffer.position() + size);
+ this.buffer.rewind();
+ return view;
+ }
+
+ private DataDescriptor init() {
+ buffer.putInt(0, SIGNATURE);
+ hasMarker = true;
+ buffer.limit(SIZE);
+ return this;
+ }
+
+ /**
+ * Data descriptor signature.
+ */
+ public static final int SIGNATURE = 0x08074b50; // 134695760L
+
+ /**
+ * Data descriptor size (including optional signature).
+ */
+ public static final int SIZE = 16;
+
+ /**
+ * For accessing the data descriptor signature, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<DataDescriptor> EXTSIG = new IntFieldId<>(0);
+
+ /**
+ * For accessing the "crc" data descriptor field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<DataDescriptor> EXTCRC = new IntFieldId<>(4);
+
+ /**
+ * For accessing the "compressed size" data descriptor field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<DataDescriptor> EXTSIZ = new IntFieldId<>(8);
+
+ /**
+ * For accessing the "uncompressed size" data descriptor field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<DataDescriptor> EXTLEN = new IntFieldId<>(12);
+
+
+ /**
+ * Overrides the generic field getter, to handle optionality of signature.
+ * @see View#get(com.google.devtools.build.android.ziputils.View.IntFieldId).
+ */
+ @Override @SuppressWarnings("unchecked") // safe by specification (FieldId.type()).
+ public int get(IntFieldId<? extends DataDescriptor> item) {
+ int address = hasMarker ? item.address() : (item.address() - 4);
+ return address < 0 ? -1 : buffer.getInt(address);
+ }
+
+ /**
+ * Returns whether this data descriptor has the optional signature.
+ * @return {@code true} if this data descriptor has a signature, {@code false} otherwise.
+ */
+ public final boolean hasMarker() {
+ return hasMarker;
+ }
+
+ /**
+ * Returns the size of this data descriptor.
+ * @return 12, or 16, depending on whether or not this data descriptor has a signature.
+ */
+ public final int getSize() {
+ return hasMarker ? SIZE : SIZE - 4;
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/DexMapper.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/DexMapper.java
new file mode 100644
index 0000000..3e0ec47
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/DexMapper.java
@@ -0,0 +1,97 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.android.ziputils;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+
+import java.util.List;
+
+/**
+ * Command-line entry point for dex mapper utility, that maps an applications classes (given in
+ * one or more input jar files), to one or more output jars, that may then be compiled separately.
+ * This class performs command line parsing and delegates to an {@link SplitZip} instance.
+ *
+ * <p>The result of compiling each output archive can be consolidated using the dex reducer utility.
+ */
+public class DexMapper {
+
+ /**
+ * @param args the command line arguments
+ */
+ public static void main(String[] args) {
+ OptionsParser optionsParser = OptionsParser.newOptionsParser(Options.class);
+ optionsParser.parseAndExitUponError(args);
+ Options options = optionsParser.getOptions(Options.class);
+ List<String> inputs = options.inputJars;
+ List<String> outputs = options.outputJars;
+ String filterFile = options.mainDexFilter;
+ String resourceFile = options.outputResources;
+
+ try {
+ new SplitZip()
+ .setVerbose(false)
+ .useDefaultEntryDate()
+ .addInputs(inputs)
+ .addOutputs(outputs)
+ .setMainClassListFile(filterFile)
+ .setResourceFile(resourceFile)
+ .run()
+ .close();
+ } catch (Exception ex) {
+ System.err.println("Caught exception" + ex.getMessage());
+ ex.printStackTrace(System.out);
+ System.exit(1);
+ }
+ }
+
+ /**
+ * Commandline options.
+ */
+ public static class Options extends OptionsBase {
+ @Option(name = "input_jar",
+ defaultValue = "null",
+ category = "input",
+ allowMultiple = true,
+ abbrev = 'i',
+ help = "Input file to read classes and jars from. Classes in "
+ + " earlier files override those in later ones.")
+ public List<String> inputJars;
+
+ @Option(name = "output_jar",
+ defaultValue = "null",
+ category = "output",
+ allowMultiple = true,
+ abbrev = 'o',
+ help = "Output file to write. Each argument is one shard. "
+ + "Output files are filled in the order specified.")
+ public List<String> outputJars;
+
+ @Option(name = "main_dex_filter",
+ defaultValue = "null",
+ category = "input",
+ abbrev = 'f',
+ help = "List of classes to include in the first output file.")
+ public String mainDexFilter;
+
+ @Option(name = "output_resources",
+ defaultValue = "null",
+ category = "output",
+ abbrev = 'r',
+ help = "File to write the Java resources to.")
+ public String outputResources;
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/DexReducer.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/DexReducer.java
new file mode 100644
index 0000000..e814fdf
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/DexReducer.java
@@ -0,0 +1,133 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.android.ziputils;
+
+import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTCRC;
+import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTLEN;
+import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTSIZ;
+import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENCRC;
+import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENLEN;
+import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENSIZ;
+import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENTIM;
+import static com.google.devtools.build.android.ziputils.LocalFileHeader.LOCFLG;
+import static com.google.devtools.build.android.ziputils.LocalFileHeader.LOCTIM;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Command-line entry point for the dex reducer utility. This utility extracts .dex files
+ * from one or more archives, and packaging them in a single output archive, renaming entries
+ * to: classes.dex, classes2.dex, ...
+ *
+ * <p>This utility is intended used to consolidate the result of compiling the output produced
+ * by the dex mapper utility.</p>
+ */
+public class DexReducer implements EntryHandler {
+ private static final String SUFFIX = ".dex";
+ private static final String BASENAME = "classes";
+ private ZipOut out;
+ private int count = 0;
+ private String outFile;
+ private List<String> paths;
+
+ DexReducer() {
+ outFile = null;
+ paths = new ArrayList<>();
+ }
+
+ /**
+ * Command-line entry point.
+ * @param args
+ */
+ public static void main(String[] args) {
+ try {
+ DexReducer dexDexReducer = new DexReducer();
+ dexDexReducer.parseArguments(args);
+ dexDexReducer.run();
+ } catch (Exception ex) {
+ System.err.println("DexReducer threw exception: " + ex.getMessage());
+ System.exit(1);
+ }
+ }
+
+ private void parseArguments(String[] args) {
+ OptionsParser optionsParser = OptionsParser.newOptionsParser(Options.class);
+ optionsParser.parseAndExitUponError(args);
+ Options options = optionsParser.getOptions(Options.class);
+ paths = options.inputZips;
+ outFile = options.outputZip;
+ }
+
+ private void run() throws IOException {
+ out = new ZipOut(new FileOutputStream(outFile, false).getChannel(), outFile);
+ for (String filename : paths) {
+ ZipIn in = new ZipIn(new FileInputStream(filename).getChannel(), filename);
+ in.scanEntries(this);
+ }
+ out.close();
+ }
+
+ @Override
+ public void handle(ZipIn in, LocalFileHeader header, DirectoryEntry dirEntry, ByteBuffer data)
+ throws IOException {
+ String path = header.getFilename();
+ if (!path.endsWith(".dex")) {
+ return;
+ }
+ count++;
+ String filename = BASENAME + (count == 1 ? "" : Integer.toString(count)) + SUFFIX;
+ String comment = dirEntry.getComment();
+ byte[] extra = dirEntry.getExtraData();
+ out.nextEntry(dirEntry.clone(filename, extra, comment).set(CENTIM, DosTime.EPOCH.time));
+ out.write(header.clone(filename, extra).set(LOCTIM, DosTime.EPOCH.time));
+ out.write(data);
+ if ((header.get(LOCFLG) & LocalFileHeader.SIZE_MASKED_FLAG) != 0) {
+ DataDescriptor desc = DataDescriptor.allocate()
+ .set(EXTCRC, dirEntry.get(CENCRC))
+ .set(EXTSIZ, dirEntry.get(CENSIZ))
+ .set(EXTLEN, dirEntry.get(CENLEN));
+ out.write(desc);
+ }
+ }
+
+ /**
+ * Commandline options.
+ */
+ public static class Options extends OptionsBase {
+ @Option(name = "input_zip",
+ defaultValue = "null",
+ category = "input",
+ allowMultiple = true,
+ abbrev = 'i',
+ help = "Input zip file containing entries to collect and enumerate.")
+ public List<String> inputZips;
+
+ @Option(name = "output_zip",
+ defaultValue = "null",
+ category = "output",
+ abbrev = 'o',
+ help = "Output zip file, containing enumerated entries.")
+ public String outputZip;
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/DirectoryEntry.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/DirectoryEntry.java
new file mode 100644
index 0000000..2033540
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/DirectoryEntry.java
@@ -0,0 +1,316 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import static java.nio.ByteOrder.LITTLE_ENDIAN;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Provides a view of a central directory entry.
+ */
+public class DirectoryEntry extends View<DirectoryEntry> {
+
+ /**
+ * Returns a {@code DirectoryEntry of the given buffer. The buffer is assumed to contain a
+ * valid "central directory entry" record beginning at the buffers current position.
+ *
+ * @param buffer containing the data of a "central directory entry" record.
+ * @return a {@code DirectoryEntry} of the data at the current position of the given byte
+ * buffer.
+ */
+ public static DirectoryEntry viewOf(ByteBuffer buffer) {
+ DirectoryEntry view = new DirectoryEntry(buffer.slice());
+ int size = view.getSize();
+ buffer.position(buffer.position() + size);
+ view.buffer.position(0).limit(size);
+ return view;
+ }
+
+ private DirectoryEntry(ByteBuffer buffer) {
+ super(buffer);
+ }
+
+ /**
+ * Creates a {@code DirectoryEntry} with a heap allocated buffer. Apart from the signature,
+ * and provided extra data and comment data, the returned object is uninitialized.
+ *
+ * @param name entry file name. Cannot be {@code null}.
+ * @param extraData extra data, or {@code null}
+ * @param comment zip file comment, or {@code null}.
+ * @return a {@code DirectoryEntry} with a heap allocated buffer.
+ */
+ public static DirectoryEntry allocate(String name, byte[] extraData, String comment) {
+ byte[] nameData = name.getBytes(UTF_8);
+ byte[] commentData = comment != null ? comment.getBytes(UTF_8) : EMPTY;
+ if (extraData == null) {
+ extraData = EMPTY;
+ }
+ int size = SIZE + nameData.length + extraData.length + commentData.length;
+ ByteBuffer buffer = ByteBuffer.allocate(size).order(LITTLE_ENDIAN);
+ return new DirectoryEntry(buffer).init(nameData, extraData, commentData, size);
+ }
+
+ /**
+ * Creates a {@code DirectoryEntry} over a writable buffer. The given buffers position is
+ * advanced by the number of bytes consumed by the view. Apart from the signature, and
+ * provided extra data and comment data, the returned view is uninitialized.
+ *
+ * @param buffer buffer to hold data for the "central directory entry" record.
+ * @param name entry file name. Cannot be {@code null}.
+ * @param extraData extra data, or {@code null}
+ * @param comment zip file global comment, or {@code null}.
+ * @return a {@code DirectoryEntry} with a heap allocated buffer.
+ */
+ public static DirectoryEntry view(ByteBuffer buffer, String name, byte[] extraData,
+ String comment) {
+ byte[] nameData = name.getBytes(UTF_8);
+ byte[] commentData = comment != null ? comment.getBytes(UTF_8) : EMPTY;
+ if (extraData == null) {
+ extraData = EMPTY;
+ }
+ int size = SIZE + nameData.length + extraData.length + commentData.length;
+ DirectoryEntry view = new DirectoryEntry(buffer.slice()).init(nameData, extraData,
+ commentData, size);
+ buffer.position(buffer.position() + size);
+ return view;
+ }
+
+ /**
+ * Copies this {@code DirectoryEntry} into a heap allocated buffer, overwriting path name,
+ * extra data and comment with the given values.
+ *
+ * @param name entry file name. Cannot be {@code null}.
+ * @param extraData extra data, or {@code null}
+ * @param comment zip file global comment, or {@code null}.
+ * @return a {@code DirectoryEntry} with a heap allocated buffer, initialized with the
+ * given values, and otherwise as a copy of this object.
+ */
+ public DirectoryEntry clone(String name, byte[] extraData, String comment) {
+ return DirectoryEntry.allocate(name, extraData, comment).copy(this, CENOFF, CENCRC, CENSIZ,
+ CENLEN, CENFLG, CENHOW, CENTIM, CENVER, CENVER, CENDSK, CENATX, CENATT);
+ }
+
+ /**
+ * Copies this {@code DirectoryEntry} into a writable buffer. The copy is made at the
+ * buffer's current position, and the position is advanced, by the size of the copy.
+ *
+ * @param buffer buffer to hold data for the copy.
+ * @return a {@code DirectoryEntry} backed by the given buffer.
+ */
+ public DirectoryEntry copy(ByteBuffer buffer) {
+ int size = getSize();
+ DirectoryEntry view = new DirectoryEntry(buffer.slice());
+ this.buffer.rewind();
+ view.buffer.put(this.buffer).flip();
+ buffer.position(buffer.position() + size);
+ this.buffer.rewind().limit(size);
+ return view;
+ }
+
+ private DirectoryEntry init(byte[] name, byte[] extra, byte[] comment, int size) {
+ buffer.putInt(0, SIGNATURE);
+ set(CENNAM, (short) name.length);
+ set(CENEXT, (short) extra.length);
+ set(CENCOM, (short) comment.length);
+ buffer.position(SIZE);
+ buffer.put(name);
+ if (extra.length > 0) {
+ buffer.put(extra);
+ }
+ if (comment.length > 0) {
+ buffer.put(comment);
+ }
+ buffer.position(0).limit(size);
+ return this;
+ }
+
+ public final int getSize() {
+ return SIZE + get(CENNAM) + get(CENEXT) + get(CENCOM);
+ }
+
+ /**
+ * Directory entry signature.
+ */
+ public static final int SIGNATURE = 0x02014b50; // 33639248L
+
+ /**
+ * Size of directory entry, not including variable data.
+ * Also the offset of the entry filename.
+ */
+ public static final int SIZE = 46;
+
+ /**
+ * For accessing the directory entry signature, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<DirectoryEntry> CENSIG = new IntFieldId<>(0);
+
+ /**
+ * For accessing the "made by version" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<DirectoryEntry> CENVEM = new ShortFieldId<>(4);
+
+ /**
+ * For accessing the "version needed" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<DirectoryEntry> CENVER = new ShortFieldId<>(6);
+
+ /**
+ * For accessing the "flags" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<DirectoryEntry> CENFLG = new ShortFieldId<>(8);
+
+ /**
+ * For accessing the "method" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<DirectoryEntry> CENHOW = new ShortFieldId<>(10);
+
+ /**
+ * For accessing the "modified time" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<DirectoryEntry> CENTIM = new IntFieldId<>(12);
+
+ /**
+ * For accessing the "crc" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<DirectoryEntry> CENCRC = new IntFieldId<>(16);
+
+ /**
+ * For accessing the "compressed size" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<DirectoryEntry> CENSIZ = new IntFieldId<>(20);
+
+ /**
+ * For accessing the "uncompressed size" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<DirectoryEntry> CENLEN = new IntFieldId<>(24);
+
+ /**
+ * For accessing the "filename length" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<DirectoryEntry> CENNAM = new ShortFieldId<>(28);
+
+ /**
+ * For accessing the "extra data length" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<DirectoryEntry> CENEXT = new ShortFieldId<>(30);
+
+ /**
+ * For accessing the "file comment length" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<DirectoryEntry> CENCOM = new ShortFieldId<>(32);
+
+ /**
+ * For accessing the "disk number" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<DirectoryEntry> CENDSK = new ShortFieldId<>(34);
+
+ /**
+ * For accessing the "internal attributes" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<DirectoryEntry> CENATT = new ShortFieldId<>(36);
+
+ /**
+ * For accessing the "external attributes" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<DirectoryEntry> CENATX = new IntFieldId<>(38);
+
+ /**
+ * For accessing the "local file header offset" directory entry field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<DirectoryEntry> CENOFF = new IntFieldId<>(42);
+
+ /**
+ * Returns the filename of this entry.
+ */
+ public final String getFilename() {
+ return getString(SIZE, get(CENNAM));
+ }
+
+ /**
+ * Returns the extra data of this entry.
+ * @return a byte array with 0 or more bytes.
+ */
+ public final byte[] getExtraData() {
+ return getBytes(SIZE + get(CENNAM), get(CENEXT));
+ }
+
+ /**
+ * Return the comment of this entry.
+ * @return a string with 0 or more characters.
+ */
+ public final String getComment() {
+ return getString(SIZE + get(CENNAM) + get(CENEXT), get(CENCOM));
+ }
+
+ /**
+ * Returns entry data size, based on directory entry information. For a valid zip file, this will
+ * be the correct size of of the entry data.
+ * @return if the {@link #CENHOW} field is 0, returns the value of the
+ * {@link #CENLEN} field (uncompressed size), otherwise returns the value of
+ * the {@link #CENSIZ} field (compressed size).
+ */
+ public int dataSize() {
+ return get(CENHOW) == 0 ? get(CENLEN) : get(CENSIZ);
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/DosTime.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/DosTime.java
new file mode 100644
index 0000000..0d228aa
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/DosTime.java
@@ -0,0 +1,69 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
+
+/**
+ * This class supports conversion from a {@code java.util.Date} object, to a
+ * 4 bytes DOS date and time representation.
+ */
+public final class DosTime {
+
+ /** DOS representation of DOS epoch (midnight, jan 1, 1980) */
+ public static final DosTime EPOCH;
+ /** {@code java.util.Date} for DOS epoch */
+ public static final Date DOS_EPOCH;
+ private static final Calendar calendar;
+
+ /**
+ * DOS representation of date passed to constructor.
+ * Time is lower the 16 bit, date the upper 16 bit.
+ */
+ public final int time;
+
+ /**
+ * Creates a DOS representation of the given date.
+ * @param date date to represent in DOS format.
+ */
+ public DosTime(Date date) {
+ this.time = dateToDosTime(date);
+ }
+
+ static {
+ calendar = new GregorianCalendar(1980, 0, 1, 0, 0, 0);
+ DOS_EPOCH = calendar.getTime();
+ EPOCH = new DosTime(DOS_EPOCH);
+ }
+
+ private static synchronized int dateToDosTime(Date date) {
+ calendar.setTime(date);
+ int year = calendar.get(Calendar.YEAR);
+ if (year < 1980) {
+ throw new IllegalArgumentException("date must be in or after 1980");
+ }
+ if (year > 2107) {
+ throw new IllegalArgumentException("date must be before 2107");
+ }
+ int month = calendar.get(Calendar.MONTH) + 1;
+ int day = calendar.get(Calendar.DAY_OF_MONTH);
+ int hour = calendar.get(Calendar.HOUR_OF_DAY);
+ int minute = calendar.get(Calendar.MINUTE);
+ int second = calendar.get(Calendar.SECOND);
+ return ((year - 1980) << 25) | (month << 21) | (day << 16)
+ | (hour << 11) | (minute << 5) | (second >> 1);
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/EndOfCentralDirectory.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/EndOfCentralDirectory.java
new file mode 100644
index 0000000..efb6bdf
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/EndOfCentralDirectory.java
@@ -0,0 +1,188 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import static java.nio.ByteOrder.LITTLE_ENDIAN;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Provides a view of a zip files "end of central directory" record.
+ */
+public class EndOfCentralDirectory extends View<EndOfCentralDirectory> {
+
+ /**
+ * Returns a {@code EndOfCentralDirectory} of the given buffer.
+ *
+ * @param buffer containing the data of a "end of central directory" record.
+ * @return a {@code EndOfCentralDirectory} of the data at the current position of the given
+ * byte buffer.
+ */
+ public static EndOfCentralDirectory viewOf(ByteBuffer buffer) {
+ EndOfCentralDirectory view = new EndOfCentralDirectory(buffer.slice());
+ view.buffer.position(0).limit(view.getSize());
+ buffer.position(buffer.position() + view.buffer.remaining());
+ return view;
+ }
+
+ private EndOfCentralDirectory(ByteBuffer buffer) {
+ super(buffer);
+ }
+
+ /**
+ * Creates a {@code EndOfCentralDirectory} with a heap allocated buffer.
+ *
+ * @param comment zip file comment, or {@code null}.
+ * @return a {@code EndOfCentralDirectory} with a heap allocated buffer.
+ */
+ public static EndOfCentralDirectory allocate(String comment) {
+ byte[] commentData = comment != null ? comment.getBytes(UTF_8) : EMPTY;
+ int size = SIZE + commentData.length;
+ ByteBuffer buffer = ByteBuffer.allocate(size).order(LITTLE_ENDIAN);
+ return new EndOfCentralDirectory(buffer).init(commentData);
+ }
+
+ /**
+ * Creates a {@code EndOfCentralDirectory} over a writable buffer.
+ *
+ * @param buffer buffer to hold data for the "end of central directory" record.
+ * @param comment zip file global comment, or {@code null}.
+ * @return a {@code EndOfCentralDirectory} with a heap allocated buffer.
+ */
+ public static EndOfCentralDirectory view(ByteBuffer buffer, String comment) {
+ byte[] commentData = comment != null ? comment.getBytes(UTF_8) : EMPTY;
+ int size = SIZE + commentData.length;
+ EndOfCentralDirectory view = new EndOfCentralDirectory(buffer.slice()).init(commentData);
+ buffer.position(buffer.position() + size);
+ return view;
+ }
+
+ /**
+ * Copies this {@code EndOfCentralDirectory} over a writable buffer.
+ *
+ * @param buffer writable byte buffer to hold the data of the copy.
+ */
+ public EndOfCentralDirectory copy(ByteBuffer buffer) {
+ EndOfCentralDirectory view = new EndOfCentralDirectory(buffer.slice());
+ this.buffer.rewind();
+ view.buffer.put(this.buffer).flip();
+ this.buffer.rewind();
+ buffer.position(buffer.position() + this.buffer.remaining());
+ return view;
+ }
+
+ private EndOfCentralDirectory init(byte[] comment) {
+ buffer.putInt(0, SIGNATURE);
+ set(ENDCOM, (short) comment.length);
+ if (comment.length > 0) {
+ buffer.position(SIZE);
+ buffer.put(comment);
+ }
+ buffer.position(0).limit(SIZE + comment.length);
+ return this;
+ }
+
+ /**
+ * Signature of end of directory record.
+ */
+ public static final int SIGNATURE = 0x06054b50; //101010256L
+
+ /**
+ * Size of end of directory record, not including variable data.
+ * Also the offset of the file comment, if any.
+ */
+ public static final int SIZE = 22;
+
+ /**
+ * For accessing the end of central directory signature, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<EndOfCentralDirectory> ENDSIG = new IntFieldId<>(0);
+
+ /**
+ * For accessing the "this disk number" end of central directory field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<EndOfCentralDirectory> ENDDSK = new ShortFieldId<>(4);
+
+
+ /**
+ * For accessing the "central directory start disk" end of central directory field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<EndOfCentralDirectory> ENDDCD = new ShortFieldId<>(6);
+
+ /**
+ * For accessing the "central directory local records" end of central directory field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<EndOfCentralDirectory> ENDSUB = new ShortFieldId<>(8);
+
+ /**
+ * For accessing the "central directory total records" end of central directory field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<EndOfCentralDirectory> ENDTOT = new ShortFieldId<>(10);
+
+ /**
+ * For accessing the "central directory size" end of central directory field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<EndOfCentralDirectory> ENDSIZ = new IntFieldId<>(12);
+
+ /**
+ * For accessing the "central directory offset" end of central directory field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<EndOfCentralDirectory> ENDOFF = new IntFieldId<>(16);
+
+ /**
+ * For accessing the "file comment length" end of central directory field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<EndOfCentralDirectory> ENDCOM = new ShortFieldId<>(20);
+
+ /**
+ * Returns the file comment.
+ * @return the file comment, or an empty string.
+ */
+ public final String getComment() {
+ return getString(SIZE, get(ENDCOM));
+ }
+
+ /**
+ * Returns the total size of the end of directory record, including file comment, if any.
+ * @return the total size of the end of directory record.
+ */
+ public int getSize() {
+ return SIZE + get(ENDCOM);
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/EntryHandler.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/EntryHandler.java
new file mode 100644
index 0000000..9f55dfb
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/EntryHandler.java
@@ -0,0 +1,41 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Entry handler to pass to {@link ZipIn#scanEntries(EntryHandler)}. Implementations,
+ * typically perform actions such as selecting, copying, renaming, merging, entries,
+ * and writing entries to one or more {@link ZipOut}.
+ */
+public interface EntryHandler {
+
+ /**
+ * Handles a zip file entry. Called by the {@code ZipIn} scanner, for each entry.
+ *
+ * @param in The {@code ZipIn} from which we're called.
+ * @param header The header for the entry to handle.
+ * @param dirEntry The directory entry corresponding to this entry.
+ * @param data byte buffer containing the data of the entry. If the data size cannot be
+ * determine from the header or directory entry (zip error), then the provided buffer
+ * may not contain all of the data.
+ * @throws IOException implementations may thrown an IOException to signal that an error occurred
+ * handling an entry. An IOException may also be generated by certain methods that the
+ * may invoke on the arguments passed to this method.
+ */
+ void handle(ZipIn in, LocalFileHeader header, DirectoryEntry dirEntry, ByteBuffer data)
+ throws IOException;
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/LocalFileHeader.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/LocalFileHeader.java
new file mode 100644
index 0000000..be7ce3a
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/LocalFileHeader.java
@@ -0,0 +1,262 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import static java.nio.ByteOrder.LITTLE_ENDIAN;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Provides a view of a local file header for a zip file entry.
+ */
+public class LocalFileHeader extends View<LocalFileHeader> {
+
+ /**
+ * Returns a {@code LocalFileHeader} view of the given buffer. The buffer is assumed to contain a
+ * valid "central directory entry" record beginning at the buffers current position.
+ *
+ * @param buffer containing the data of a "central directory entry" record.
+ * @return a {@code LocalFileHeader} of the data at the current position of the given byte
+ * buffer.
+ */
+ public static LocalFileHeader viewOf(ByteBuffer buffer) {
+ LocalFileHeader view = new LocalFileHeader(buffer.slice());
+ view.buffer.limit(view.getSize());
+ buffer.position(buffer.position() + view.buffer.remaining());
+ return view;
+ }
+
+ private LocalFileHeader(ByteBuffer buffer) {
+ super(buffer);
+ }
+
+ /**
+ * Creates a {@code LocalFileHeader} with a heap allocated buffer. Apart from the signature
+ * and extra data data (if any), the returned object is uninitialized.
+ *
+ * @param name entry file name. Cannot be {@code null}.
+ * @param extraData extra data, or {@code null}
+ * @return a {@code LocalFileHeader} with a heap allocated buffer.
+ */
+ public static LocalFileHeader allocate(String name, byte[] extraData) {
+ byte[] nameData = name.getBytes(UTF_8);
+ if (extraData == null) {
+ extraData = EMPTY;
+ }
+ int size = SIZE + nameData.length + extraData.length;
+ ByteBuffer buffer = ByteBuffer.allocate(size).order(LITTLE_ENDIAN);
+ return new LocalFileHeader(buffer).init(nameData, extraData, size);
+ }
+
+ /**
+ * Creates a {@code LocalFileHeader} over a writable buffer. The given buffer's position is
+ * advanced by the number of bytes consumed by the view. Apart from the signature and extra data
+ * (if any), the returned view is uninitialized.
+ *
+ * @param buffer buffer to hold data for the "central directory entry" record.
+ * @param name entry file name. Cannot be {@code null}.
+ * @param extraData extra data, or {@code null}
+ * @return a {@code DirectoryEntry} with a heap allocated buffer.
+ */
+ public static LocalFileHeader view(ByteBuffer buffer, String name, byte[] extraData) {
+ byte[] nameData = name.getBytes(UTF_8);
+ if (extraData == null) {
+ extraData = EMPTY;
+ }
+ int size = SIZE + nameData.length + extraData.length;
+ LocalFileHeader view = new LocalFileHeader(buffer.slice()).init(nameData, extraData, size);
+ buffer.position(buffer.position() + size);
+ return view;
+ }
+
+ /**
+ * Copies this {@code LocalFileHeader} into a heap allocated buffer, overwriting the current
+ * path name and extra data with the given values.
+ *
+ * @param name entry file name. Cannot be {@code null}.
+ * @param extraData extra data, or {@code null}
+ * @return a {@code LocalFileHeader} with a heap allocated buffer, initialized with the given
+ * name and extra data, and otherwise a copy of this object.
+ */
+ public LocalFileHeader clone(String name, byte[] extraData) {
+ return LocalFileHeader.allocate(name, extraData).copy(this, LOCCRC, LOCSIZ, LOCLEN, LOCFLG,
+ LOCHOW, LOCTIM, LOCVER);
+ }
+
+ /**
+ * Copies this {@code LocalFileHeader} into a writable buffer. The copy is made at the
+ * buffer's current position, and the position is advanced, by the size of the copy.
+ *
+ * @param buffer buffer to hold data for the copy.
+ * @return a {@code LocalFileHeader} backed by the given buffer.
+ */
+ public LocalFileHeader copy(ByteBuffer buffer) {
+ int size = getSize();
+ LocalFileHeader view = new LocalFileHeader(buffer.slice());
+ this.buffer.rewind();
+ view.buffer.put(this.buffer).flip();
+ buffer.position(buffer.position() + size);
+ this.buffer.rewind();
+ return view;
+ }
+
+ private LocalFileHeader init(byte[] name, byte[] extra, int size) {
+ buffer.putInt(0, SIGNATURE);
+ set(LOCNAM, (short) name.length);
+ set(LOCEXT, (short) extra.length);
+ buffer.position(SIZE);
+ buffer.put(name);
+ if (extra.length > 0) {
+ buffer.put(extra);
+ }
+ buffer.position(0).limit(size);
+ return this;
+ }
+
+ /**
+ * Flag used to mark a compressed entry, for which the size is unknown at the time
+ * of writing the header.
+ */
+ public static final short SIZE_MASKED_FLAG = 0x8;
+
+ /**
+ * Signature of local file header.
+ */
+ public static final int SIGNATURE = 0x04034b50; // 67324752L
+
+ /**
+ * Size of local file header, not including variable data.
+ * Also the offset of the filename.
+ */
+ public static final int SIZE = 30;
+
+ /**
+ * For accessing the local header signature, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<LocalFileHeader> LOCSIG = new IntFieldId<>(0);
+
+ /**
+ * For accessing the "needed version" local header field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<LocalFileHeader> LOCVER = new ShortFieldId<>(4);
+
+ /**
+ * For accessing the "flags" local header field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<LocalFileHeader> LOCFLG = new ShortFieldId<>(6);
+
+ /**
+ * For accessing the "method" local header field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<LocalFileHeader> LOCHOW = new ShortFieldId<>(8);
+
+ /**
+ * For accessing the "modified time" local header field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<LocalFileHeader> LOCTIM = new IntFieldId<>(10);
+
+ /**
+ * For accessing the "crc" local header field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<LocalFileHeader> LOCCRC = new IntFieldId<>(14);
+
+ /**
+ * For accessing the "compressed size" local header field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<LocalFileHeader> LOCSIZ = new IntFieldId<>(18);
+
+ /**
+ * For accessing the "uncompressed size" local header field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)}
+ * methods.
+ */
+ public static final IntFieldId<LocalFileHeader> LOCLEN = new IntFieldId<>(22);
+
+ /**
+ * For accessing the "filename length" local header field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<LocalFileHeader> LOCNAM = new ShortFieldId<>(26);
+
+ /**
+ * For accessing the "extra data length" local header field, with the
+ * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)}
+ * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)}
+ * methods.
+ */
+ public static final ShortFieldId<LocalFileHeader> LOCEXT = new ShortFieldId<>(28);
+
+ /**
+ * Returns the filename for this entry.
+ */
+ public final String getFilename() {
+ return getString(SIZE, get(LOCNAM));
+ }
+
+ /**
+ * Returns the extra data for this entry.
+ * @return an array of 0 or more bytes.
+ */
+ public final byte[] getExtraData() {
+ return getBytes(SIZE + get(LOCNAM), get(LOCEXT));
+ }
+
+ /**
+ * Returns the size of this header, including filename, and extra data (if any).
+ */
+ public final int getSize() {
+ return SIZE + get(LOCNAM) + get(LOCEXT);
+ }
+
+ /**
+ * Returns entry data size, based on directory entry information. For a valid zip file, this will
+ * be the correct size of the entry data, or -1, if the size cannot be determined from the
+ * header. Notice, if ths method returns 0, it may be because the writer of the zip file forgot
+ * to set the {@link #SIZE_MASKED_FLAG}.
+ *
+ * @return if the {@link #LOCHOW} field is 0, returns the value of the
+ * {@link #LOCLEN} field (uncompressed size). If {@link #LOCHOW} is not 0, and the
+ * {@link #SIZE_MASKED_FLAG} is not set, returns the value of the {@link #LOCSIZ} field
+ * (compressed size). Otherwise return -1 (size unknown).
+ */
+ public int dataSize() {
+ return get(LOCHOW) == 0 ? get(LOCLEN)
+ : (get(LOCFLG) & SIZE_MASKED_FLAG) == 0 ? get(LOCSIZ) : -1;
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/README b/src/tools/android/java/com/google/devtools/build/android/ziputils/README
new file mode 100644
index 0000000..25891bd
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/README
@@ -0,0 +1,26 @@
+This directory contains utilities needed to support parallel dex compilation
+of Android applications.
+
+The DexMapper utility maps an applications classes (given in one or more input jar files),
+to one or more output jars, that may then be compiled separately.
+
+The DexReducer utility extracts .dex files from one or more archives, and package them in a
+single output archive, renaming entries to: classes.dex, classes2.dex, ...
+
+These utilities uses a low-level zip-file manipulation library contained in this package.
+The library is optimized for serial parsing of zip archives and tasks such as copying entries
+to one or more output archives.It employs direct byte buffers, to avoid copying file data to
+the java heap, and allowing fast copies from input to output. Output is asynchronous, for optimal
+performance when running on non-memory-based file systems.
+
+WARNING: This library was designed for creating build tools needed to support parallel
+dex compilation of Android applications, related functions. While the library provides
+fairly general facilities for processing zip files, the API is still immature, and
+subject to change.
+
+Defined targets
+---------------------------------------------------
+blaze build java/com/google/devtools/build/android/ziputils:ziputils_lib
+blaze build java/com/google/devtools/build/android/ziputils:mapper
+blaze build java/com/google/devtools/build/android/ziputils:reducer
+blaze test javatests/com/google/devtools/build/android/ziputils:all_tests
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/ScanUtil.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/ScanUtil.java
new file mode 100644
index 0000000..185838f
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/ScanUtil.java
@@ -0,0 +1,104 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import com.google.common.base.Preconditions;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Helper class for finding byte patterns in a byte buffer, scanning forwards or backwards.
+ * This is used to located the "end of central directory" marker, and in other instances
+ * where zip file elements must be located by scanning for known signatures.
+ * For instance, when it's necessary to scan for zip entries, without
+ * relying on the central directory to provide the exact location, or when scanning for
+ * data descriptor at the end of an entry of unknown size.
+ */
+public class ScanUtil {
+
+ /**
+ * Finds the next previous position of a given byte sequence in a given byte buffer,
+ * starting from the buffer's current position (excluded).
+ *
+ * @param target byte sequence to search for.
+ * @param buffer byte buffer in which to search.
+ * @return position of the last location of the target sequence, at a position
+ * strictly smaller than the current position, or -1 if not found.
+ * @throws IllegalArgumentException if either {@code target} or {@code buffer} is {@code null}.
+ */
+ static int scanBackwardsTo(byte[] target, ByteBuffer buffer) {
+ Preconditions.checkNotNull(target);
+ Preconditions.checkNotNull(buffer);
+ if (target.length == 0) {
+ return buffer.position() - 1;
+ }
+ int pos = buffer.position() - target.length;
+ if (pos < 0) {
+ return -1;
+ }
+ scan:
+ while (true) {
+ while (pos >= 0 && buffer.get(pos) != target[0]) {
+ pos--;
+ }
+ if (pos < 0) {
+ return -1;
+ }
+ for (int i = 1; i < target.length; i++) {
+ if (buffer.get(pos + i) != target[i]) {
+ pos--;
+ continue scan;
+ }
+ }
+ return pos;
+ }
+ }
+
+ /**
+ * Finds the next position of a given byte sequence in a given byte buffer, starting from the
+ * buffer's current position (included).
+ *
+ * @param target byte sequence to search for.
+ * @param buffer byte buffer in which to search.
+ * @return position of the first location of the target sequence, or -1 if not found.
+ * @throws IllegalArgumentException if either {@code target} or {@code buffer} is {@code null}.
+ */
+ static int scanTo(byte[] target, ByteBuffer buffer) {
+ Preconditions.checkNotNull(target);
+ Preconditions.checkNotNull(buffer);
+ if (!buffer.hasRemaining()) {
+ return -1;
+ }
+ int pos = buffer.position();
+ if (target.length == 0) {
+ return pos;
+ }
+ scan:
+ while (true) {
+ while (pos <= buffer.limit() - target.length && buffer.get(pos) != target[0]) {
+ pos++;
+ }
+ if (pos > buffer.limit() - target.length + 1) {
+ return -1;
+ }
+ for (int i = 1; i < target.length; i++) {
+ if (buffer.get(pos + i) != target[i]) {
+ pos++;
+ continue scan;
+ }
+ }
+ return pos;
+ }
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/SplitZip.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/SplitZip.java
new file mode 100644
index 0000000..3d39b47
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/SplitZip.java
@@ -0,0 +1,452 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTCRC;
+import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTLEN;
+import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTSIZ;
+import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENCRC;
+import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENLEN;
+import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENSIZ;
+import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENTIM;
+import static com.google.devtools.build.android.ziputils.LocalFileHeader.LOCFLG;
+import static com.google.devtools.build.android.ziputils.LocalFileHeader.LOCTIM;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.base.Preconditions;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Extracts entries from a set of input archives, and copies them to N output archive of
+ * approximately equal size, while attempting to split archives on package (directory) boundaries.
+ * Optionally, accept a list of entries to be added to the first output archive, splitting
+ * remaining entries by package boundaries.
+ */
+public class SplitZip implements EntryHandler {
+ private boolean verbose = false;
+ private final List<ZipIn> inputs;
+ private final List<ZipOut> outputs;
+ private String filterFile;
+ private InputStream filterInputStream;
+ private String resourceFile;
+ private Date date;
+ private DosTime dosTime;
+ // Internal state variables:
+ private boolean finished = false;
+ private Set<String> filter;
+ private ZipOut[] zipOuts;
+ private ZipOut resourceOut;
+ private final Map<String, ZipOut> assignments = new HashMap<>();
+ private final Map<String, CentralDirectory> centralDirectories;
+ private final Set<String> classes = new TreeSet<>();
+
+ /**
+ * Creates an un-configured {@code SplitZip} instance.
+ */
+ public SplitZip() {
+ inputs = new ArrayList<>();
+ outputs = new ArrayList<>();
+ centralDirectories = new HashMap<>();
+ }
+
+ /**
+ * Configures a resource file. By default, resources are output in the initial shard.
+ * If a resource file is specified, resources are written to this instead.
+ * @param resourceFile in not {@code null}, the name of a file in which to output resources.
+ * @return this object.
+ */
+ public SplitZip setResourceFile(String resourceFile) {
+ this.resourceFile = resourceFile;
+ return this;
+ }
+
+ // Package private for testing with mock file
+ SplitZip setResourceFile(ZipOut resOut) {
+ resourceOut = resOut;
+ return this;
+ }
+
+ /**
+ * Gets the name of the resource output file. If no resource output file is configured, resources
+ * are output in the initial shard.
+ * @return the name of the resource output file, or {@code null} if no file has been configured.
+ */
+ public String getResourceFile() {
+ return resourceFile;
+ }
+
+ /**
+ * Configures a file containing a list of files to be included in the first output archive.
+ *
+ * @param clFile path of class file list.
+ * @return this object
+ */
+ public SplitZip setMainClassListFile(String clFile) {
+ filterFile = clFile;
+ return this;
+ }
+
+ // Package private for testing with mock file
+ SplitZip setMainClassListFile(InputStream clInputStream) {
+ filterInputStream = clInputStream;
+ return this;
+ }
+
+ /**
+ * Gets the path of the file listing the content of the initial shard.
+ * @return return path of file list file, or {@code null} if not set.
+ */
+ public String getMainClassListFile() {
+ return filterFile;
+ }
+
+ /**
+ * Configures verbose mode.
+ *
+ * @param flag set to {@code true} to turn on verbose mode.
+ * @return this object
+ */
+ public SplitZip setVerbose(boolean flag) {
+ verbose = flag;
+ return this;
+ }
+
+ /**
+ * Gets the verbosity mode..
+ * @return {@code true} iff verbose mode is enabled
+ */
+ public boolean isVerbose() {
+ return verbose;
+ }
+
+ /**
+ * Sets date to overwrite timestamp of copied entries. Setting the date to {@code null} means
+ * using the date and time information in the input file. Set an explicit date to override.
+ *
+ * @param date modified date and time to set for entries in output.
+ * @return this object.
+ */
+ public SplitZip setEntryDate(Date date) {
+ this.date = date;
+ this.dosTime = date == null ? null : new DosTime(date);
+ return this;
+ }
+
+ /**
+ * Sets date to {@link DosTime#DOS_EPOCH}.
+ * @return this object.
+ */
+ public SplitZip useDefaultEntryDate() {
+ this.date = DosTime.DOS_EPOCH;
+ this.dosTime = DosTime.EPOCH;
+ return this;
+ }
+
+ /**
+ * Gets the entry modified date.
+ */
+ public Date getEntryDate() {
+ return date;
+ }
+
+ /**
+ * Configures multiple input file locations.
+ *
+ * @param inputs list of input locations.
+ * @return this object
+ * @throws java.io.IOException
+ */
+ public SplitZip addInputs(Iterable<String> inputs) throws IOException {
+ for (String i : inputs) {
+ addInput(i);
+ }
+ return this;
+ }
+
+ /**
+ * Configures an input location. An input file must be a zip archive.
+ *
+ * @param filename path for an input location.
+ * @return this object
+ * @throws java.io.IOException
+ */
+ public SplitZip addInput(String filename) throws IOException {
+ if (filename != null) {
+ inputs.add(new ZipIn(new FileInputStream(filename).getChannel(), filename));
+ }
+ return this;
+ }
+
+ // Package private, for testing using mock file system.
+ SplitZip addInput(ZipIn in) throws IOException {
+ Preconditions.checkNotNull(in);
+ inputs.add(in);
+ return this;
+ }
+
+ /**
+ * Configures multiple output file locations.
+ *
+ * @param outputs list of output files.
+ * @return this object
+ * @throws java.io.IOException
+ */
+ public SplitZip addOutputs(Iterable<String> outputs) throws IOException {
+ for (String o : outputs) {
+ addOutput(o);
+ }
+ return this;
+ }
+
+ /**
+ * Configures an output location.
+ *
+ * @param output path for an output location.
+ * @return this object
+ * @throws java.io.IOException
+ */
+ public SplitZip addOutput(String output) throws IOException {
+ Preconditions.checkNotNull(output);
+ outputs.add(new ZipOut(new FileOutputStream(output, false).getChannel(), output));
+ return this;
+ }
+
+ // Package private for testing with mock file
+ SplitZip addOutput(ZipOut output) throws IOException {
+ Preconditions.checkNotNull(output);
+ outputs.add(output);
+ return this;
+ }
+
+ /**
+ * Executes this {@code SplitZip}, reading content from the configured input locations, creating
+ * the specified number of archives, in the configured output directory.
+ *
+ * @return this object
+ * @throws java.io.IOException
+ */
+ public SplitZip run() throws IOException {
+ verbose("SplitZip: Splitting in: " + outputs.size());
+ verbose("SplitZip: with filter: " + filterFile);
+ checkConfig();
+ // Prepare output files
+ zipOuts = outputs.toArray(new ZipOut[outputs.size()]);
+ if (resourceFile != null) {
+ resourceOut = new ZipOut(new FileOutputStream(resourceFile, false).getChannel(),
+ resourceFile);
+ } else if (resourceOut == null) { // may have been set for testing
+ resourceOut = zipOuts[0];
+ }
+
+ // Read directories of input files
+ for (ZipIn zip : inputs) {
+ zip.endOfCentralDirectory();
+ centralDirectories.put(zip.getFilename(), zip.centralDirectory());
+ zip.centralDirectory();
+ }
+ // Assign input entries to output files
+ split();
+ // Copy entries to the assigned output files
+ for (ZipIn zip : inputs) {
+ zip.scanEntries(this);
+ }
+ return this;
+ }
+
+ /**
+ * Copies an entry to the assigned output files. Called for each entry in the input files.
+ * @param in
+ * @param header
+ * @param dirEntry
+ * @param data
+ * @throws IOException
+ */
+ @Override
+ public void handle(ZipIn in, LocalFileHeader header, DirectoryEntry dirEntry,
+ ByteBuffer data) throws IOException {
+ String localFilename = header.getFilename();
+ ZipOut out = assignments.remove(localFilename);
+ if (out == null) {
+ // Skip unassigned file;
+ return;
+ }
+ if (dirEntry == null) {
+ // Shouldn't get here, as there should be no assignment.
+ System.out.println("Warning: no directory entry");
+ return;
+ }
+ // Clone directory entry
+ DirectoryEntry entryOut = out.nextEntry(dirEntry);
+ if (dosTime != null) {
+ // Overwrite time stamp
+ header.set(LOCTIM, dosTime.time);
+ entryOut.set(CENTIM, dosTime.time);
+ }
+ out.write(header);
+ out.write(data);
+ if ((header.get(LOCFLG) & LocalFileHeader.SIZE_MASKED_FLAG) != 0) {
+ // Instead of this, we could fix the header with the size information
+ // from the directory entry. For now, keep the entry encoded as-is.
+ DataDescriptor desc = DataDescriptor.allocate()
+ .set(EXTCRC, dirEntry.get(CENCRC))
+ .set(EXTSIZ, dirEntry.get(CENSIZ))
+ .set(EXTLEN, dirEntry.get(CENLEN));
+ out.write(desc);
+ }
+ }
+
+ /**
+ * Writes any remaining output data to the output stream.
+ *
+ * @throws IOException if the output stream or the filter throws an IOException
+ * @throws IllegalStateException if this method was already called earlier
+ */
+ public void finish() throws IOException {
+ checkNotFinished();
+ finished = true;
+ if (resourceOut != null) {
+ resourceOut.finish();
+ }
+ for (ZipOut zo : zipOuts) {
+ zo.finish();
+ }
+ }
+
+ /**
+ * Writes any remaining output data to the output stream and closes it.
+ *
+ * @throws IOException if the output stream or the filter throws an IOException
+ */
+ public void close() throws IOException {
+ if (!finished) {
+ finish();
+ }
+ if (resourceOut != null) {
+ resourceOut.close();
+ }
+ for (ZipOut zo : zipOuts) {
+ zo.close();
+ }
+ }
+
+ private void checkNotFinished() {
+ if (finished) {
+ throw new IllegalStateException();
+ }
+ }
+
+ /**
+ * Validates configuration before execution.
+ */
+ private void checkConfig() throws IOException {
+ if (outputs.size() < 1) {
+ throw new IllegalStateException("Require at least one output file");
+ }
+ filter = filterFile == null && filterInputStream == null ? null : readPaths(filterFile);
+ }
+
+ /**
+ * Parses the entries and assign each entry to an output file.
+ */
+ private void split() {
+ for (ZipIn in : inputs) {
+ CentralDirectory cdir = centralDirectories.get(in.getFilename());
+ for (DirectoryEntry entry : cdir.list()) {
+ String filename = entry.getFilename();
+ if (filename.endsWith(".class")) {
+ // Only pass classes to the splitter, so that it can do the best job
+ // possible distributing them across output files.
+ classes.add(filename);
+ } else if (!filename.endsWith("/")) {
+ // Non class files (resources) are either assigned to the first
+ // output file, or to a specified resource output file.
+ assignments.put(filename, resourceOut);
+ }
+ }
+ }
+ Splitter entryFilter = new Splitter(outputs.size(), classes.size());
+ if (filter != null) {
+ // Assign files in the filter to the first output file.
+ entryFilter.assign(filter);
+ entryFilter.nextShard(); // minimal initial shard
+ }
+ for (String path : classes) {
+ int assignment = entryFilter.assign(path);
+ Preconditions.checkState(assignment >= 0 && assignment < zipOuts.length);
+ assignments.put(path, zipOuts[assignment]);
+ }
+ }
+
+ /**
+ * Reads paths of classes required in first shard. For testing purposes, this relies
+ * on the file system configured for the {@code Zip} library class.
+ */
+ private Set<String> readPaths(String fileName) throws IOException {
+ Set<String> paths = new HashSet<>();
+ BufferedReader reader = null;
+ try {
+ if (filterInputStream == null) {
+ filterInputStream = new FileInputStream(fileName);
+ }
+ reader = new BufferedReader(new InputStreamReader(filterInputStream, UTF_8));
+ String line;
+ while (null != (line = reader.readLine())) {
+ paths.add(fixPath(line));
+ }
+ return paths;
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ }
+ }
+
+ // TODO(bazel-team): Got this from 'dx'. I'm not sure we need this part. Keep it for now,
+ // to make sure we read the main dex list the exact same way that dx would.
+ private String fixPath(String path) {
+ if (File.separatorChar == '\\') {
+ path = path.replace('\\', '/');
+ }
+ int index = path.lastIndexOf("/./");
+ if (index != -1) {
+ return path.substring(index + 3);
+ }
+ if (path.startsWith("./")) {
+ return path.substring(2);
+ }
+ return path;
+ }
+
+ private void verbose(String msg) {
+ if (verbose) {
+ System.out.println(msg);
+ }
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/Splitter.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/Splitter.java
new file mode 100644
index 0000000..f701bb1
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/Splitter.java
@@ -0,0 +1,136 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import com.google.common.base.Preconditions;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+class Splitter {
+
+ private static final String ARCHIVE_FILE_SEPARATOR = "/";
+
+ private final int numberOfShards;
+ private final Map<String, Integer> assigned;
+ private int size = 0;
+ private int shard = 0;
+ private String currPath = null;
+ private int remaining;
+ private int idealSize;
+ private int almostFull;
+
+ /**
+ * Creates a splitter for splitting an expected number of entries into
+ * a given number of shards. The current shard is shard 0.
+ */
+ public Splitter(int numberOfShards, int expectedSize) {
+ this.numberOfShards = numberOfShards;
+ this.remaining = expectedSize;
+ this.assigned = new HashMap<>();
+ idealSize = remaining / (numberOfShards - shard);
+ // Before you change this, please do the math.
+ // It's not always perfect, but designed to keep shards reasonably balanced in most cases.
+ int limit = Math.min(Math.min(10, (idealSize + 3) / 4), (int) Math.log(numberOfShards));
+ almostFull = idealSize - limit;
+ }
+
+ /**
+ * Forces mapping of the given entries to be that of the current shard.
+ * The estimated number of remaining entries to process is adjusted,
+ * by subtracting the number of as-of-yet unassigned entries from the
+ * filter.
+ */
+ public void assign(Collection<String> filter) {
+ if (filter != null) {
+ for (String s : filter) {
+ if (!assigned.containsKey(s)) {
+ remaining--;
+ }
+ assigned.put(s, shard);
+ }
+ size = filter.size();
+ }
+ }
+
+ /**
+ * Forces increment of the current shard. May be called externally.
+ * Typically right after calling {@link #assign(java.util.Collection)}.
+ */
+ public void nextShard() {
+ if (shard < numberOfShards - 1) {
+ shard++;
+ size = 0;
+ addEntries(0);
+ }
+ }
+
+ /**
+ * Adjusts the number of estimated entries to be process by the given count.
+ */
+ public void addEntries(int count) {
+ this.remaining += count;
+ idealSize = numberOfShards > shard ? remaining / (numberOfShards - shard) : remaining;
+ // Before you change this, please do the math.
+ // It's not always perfect, but designed to keep shards reasonably balanced in most cases.
+ int limit = Math.min(Math.min(10, (idealSize + 3) / 4), (int) Math.log(numberOfShards));
+ almostFull = idealSize - limit;
+ }
+
+ /**
+ * Assigns the given entry to an output file.
+ */
+ public int assign(String path) {
+ Preconditions.checkState(shard < numberOfShards, "Too many shards!");
+ Integer assignment = assigned.get(path);
+ if (assignment != null) {
+ return assignment;
+ }
+ remaining--;
+
+ // last shard, no choice
+ if (shard == numberOfShards - 1) {
+ size++;
+ assigned.put(currPath, shard);
+ return shard;
+ }
+
+ // Forced split to try to avoid empty shards
+ if (remaining < numberOfShards - shard - 1) {
+ if (size > 0) {
+ nextShard();
+ }
+ size++;
+ assigned.put(currPath, shard);
+ return shard;
+ }
+
+ String prevPath = currPath;
+ currPath = path;
+ // If current shard is at least "almost full", check for package boundary?
+ if (prevPath != null && size >= almostFull) {
+ int i = currPath.lastIndexOf(ARCHIVE_FILE_SEPARATOR);
+ String dir = i > 0 ? currPath.substring(0, i) : ".";
+ i = prevPath.lastIndexOf(ARCHIVE_FILE_SEPARATOR);
+ String prevDir = i > 0 ? prevPath.substring(0, i) : ".";
+ if (!dir.equals(prevDir)) {
+ nextShard();
+ }
+ }
+ assigned.put(currPath, shard);
+ size++;
+ return shard;
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/View.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/View.java
new file mode 100644
index 0000000..cc3535f
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/View.java
@@ -0,0 +1,271 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import static java.nio.ByteOrder.LITTLE_ENDIAN;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A {@code View} represents a range of a larger sequence of bytes (e.g. part of a file).
+ * It consist of an internal byte buffer providing access to the data range, and an
+ * offset of the first byte in the buffer within the larger sequence.
+ * Subclasses will typically assign a specific interpretations of the data in a view
+ * (e.g. records of a specific type).
+ *
+ * <p>Instances of subclasses may generally be "allocated", or created "of" or "over" a
+ * byte buffer provided at creation time.
+ *
+ * <p>An "allocated" view gets a new heap allocated byte buffer. Allocation methods
+ * typically defines parameters for variable sized part of the object they create a
+ * view of. Once allocated, the variable size parts (e.g. filename) cannot be changed.
+ *
+ * <p>A view created "of" an existing byte buffer, expects the buffer to contain an
+ * appropriate record at its current position. The buffers limit is set at the
+ * end of the object being viewed.
+ *
+ * <p>A view created "over" an existing byte buffer, reserves space in the buffer for the
+ * object being viewed. The variable sized parts of the object are initialized, but
+ * otherwise the existing buffer data are left as-is, and can be manipulated through the
+ * view.
+ *
+ * <p> An view can also be copied into an existing byte buffer. This is like creating a
+ * view "over" the buffer, but initializing the new view as an exact copy of the one
+ * being copied.
+ *
+ * @param <SUB> to be type-safe, a subclass {@code S} must extend {@code View<S>}. It must not
+ * extend {@code View<S2>}, where {@code S2} is another subclass, which is not also a superclass
+ * of {@code S}. To maintain this guarantee, this class is declared abstract and package private.
+ * Unchecked warnings are suppressed as per this specification constraint.
+ */
+abstract class View<SUB extends View<?>> {
+
+ /** Zero length byte array */
+ protected static final byte[] EMPTY = {};
+
+ /** {@code ByteBuffer} backing this view. */
+ protected final ByteBuffer buffer;
+
+ /**
+ * Offset of first byte covered by this view. For input views, this is the file offset where the
+ * item occur. For output, it's the offset at which we expect to write the item (may be -1, for
+ * unknown).
+ */
+ protected long fileOffset;
+
+ /**
+ * Creates a view backed by the given {@code ByteBuffer}. Sets the buffer's byte order to
+ * little endian, and sets the file offset to -1 (unknown).
+ *
+ * @param buffer backing byte buffer.
+ */
+ protected View(ByteBuffer buffer) {
+ buffer.order(LITTLE_ENDIAN);
+ this.buffer = buffer;
+ this.fileOffset = -1;
+ }
+
+ /**
+ * Sets the file offset of the data item covered by this view,
+ *
+ * @param fileOffset
+ * @return this object.
+ */
+ @SuppressWarnings("unchecked") // safe by specification
+ public SUB at(long fileOffset) {
+ this.fileOffset = fileOffset;
+ return (SUB) this;
+ }
+
+ /**
+ * Gets the fileOffset of this view.
+ *
+ * @return the location of the viewed object within the underlying file.
+ */
+ public long fileOffset() {
+ return fileOffset;
+ }
+
+ /**
+ * Returns an array with data copied from the backing byte buffer, at the given offset, relative
+ * to the beginning of this view. This method does not perform boundary checks of offset or
+ * length. This method may temporarily changes the position of the backing buffer. However, after
+ * the call, the position will be unchanged.
+ *
+ * @param off offset relative to this view.
+ * @param len number of bytes to return.
+ * @return Newly allocated array with copy of requested data.
+ * @throws IndexOutOfBoundsException in case of illegal arguments.
+ */
+ protected byte[] getBytes(int off, int len) {
+ if (len == 0) {
+ return EMPTY;
+ }
+ byte[] bytes;
+ try {
+ bytes = new byte[len];
+ int currPos = buffer.position();
+ buffer.position(off);
+ buffer.get(bytes);
+ buffer.position(currPos);
+ } catch (Exception ex) {
+ throw new IndexOutOfBoundsException();
+ }
+ return bytes;
+ }
+
+ /**
+ * Returns a String representation of {@code len} bytes starting at offset {@code off} in this
+ * view. This method may temporarily changes the position of the backing buffer. However, after
+ * the call, the position will be unchanged.
+ *
+ * @param off offset relative to backing buffer.
+ * @param len number of bytes to return. This method may throw an
+ * @return Newly allocated String created by interpreting the specified bytes as UTF-8 data.
+ * @throws IndexOutOfBoundsException in case of illegal arguments.
+ */
+ protected String getString(int off, int len) {
+ if (len == 0) {
+ return "";
+ }
+ if (buffer.hasArray()) {
+ return new String(buffer.array(), buffer.arrayOffset() + off, len, UTF_8);
+ } else {
+ return new String(getBytes(off, len), UTF_8);
+ }
+ }
+
+ /**
+ * Gets the value of an identified integer field.
+ *
+ * @param id field identifier
+ * @return the value of the field identified by {@code id}.
+ */
+ public int get(IntFieldId<? extends SUB> id) {
+ return buffer.getInt(id.address());
+ }
+
+ /**
+ * Gets the value of an identified short field.
+ *
+ * @param id field identifier
+ * @return the value of the field identified by {@code id}.
+ */
+ public short get(ShortFieldId<? extends SUB> id) {
+ return buffer.getShort(id.address());
+ }
+
+ /**
+ * Sets the value of an identified integer field.
+ *
+ * @param id field identifier
+ * @param value value to set for the field identified by {@code id}.
+ * @return this object.
+ */
+ @SuppressWarnings("unchecked") // safe by specification
+ public SUB set(IntFieldId<? extends SUB> id, int value) {
+ buffer.putInt(id.address(), value);
+ return (SUB) this;
+ }
+
+ /**
+ * Sets the value of an identified short field.
+ *
+ * @param id field identifier
+ * @param value value to set for the field identified by {@code id}.
+ * @return this object.
+ */
+ @SuppressWarnings("unchecked") // safe by specification
+ public SUB set(ShortFieldId<? extends SUB> id, short value) {
+ buffer.putShort(id.address(), value);
+ return (SUB) this;
+ }
+
+ /**
+ * Copies the values of one or more identified fields from another view to this view.
+ *
+ * @param from The view from which to copy field values.
+ * @param ids field identifiers for fields to copy.
+ * @return this object.
+ */
+ @SuppressWarnings("unchecked") // safe by specification
+ public SUB copy(View<SUB> from, FieldId<? extends SUB, ?>... ids) {
+ for (FieldId<? extends SUB, ?> id : ids) {
+ int address = id.address;
+ buffer.put(address, from.buffer.get(address++));
+ buffer.put(address, from.buffer.get(address++));
+ if (id.type() == Integer.TYPE) {
+ buffer.put(address, from.buffer.get(address++));
+ buffer.put(address, from.buffer.get(address));
+ }
+ }
+ return (SUB) this;
+ }
+
+ /**
+ * Base class for data field descriptors. Describes a data field's type and address in a view.
+ * This base class allows the
+ * {@link #copy(View, com.google.devtools.build.android.ziputils.View.FieldId[])} method
+ * to operate of fields of mixed types.
+ *
+ * @param <T> {@code Integer.TYPE} or {@code Short.TYPE}.
+ * @param <V> subclass of {@code View} for which a field id is defined.
+ */
+ protected abstract static class FieldId<V extends View<?>, T> {
+ private final int address;
+ private final Class<T> type;
+
+ protected FieldId(int address, Class<T> type) {
+ this.address = address;
+ this.type = type;
+ }
+
+ /**
+ * Returns the class of the field type, {@code Class<T>}.
+ */
+ public Class<T> type() {
+ return type;
+ }
+
+ /**
+ * Returns the field address, within a record of type {@code V}
+ */
+ public int address() {
+ return address;
+ }
+ }
+
+ /**
+ * Describes an integer fields for a view.
+ *
+ * @param <V> subclass of {@code View} for which a field id is defined.
+ */
+ protected static class IntFieldId<V extends View<?>> extends FieldId<V, Integer> {
+ protected IntFieldId(int address) {
+ super(address, Integer.TYPE);
+ }
+ }
+
+ /**
+ * Describes a short field for a view.
+ *
+ * @param <V> subclass of {@code View} for which a field id is defined.
+ */
+ protected static class ShortFieldId<V extends View<?>> extends FieldId<V, Short> {
+ protected ShortFieldId(int address) {
+ super(address, Short.TYPE);
+ }
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/ZipIn.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/ZipIn.java
new file mode 100644
index 0000000..afda213
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/ZipIn.java
@@ -0,0 +1,695 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTLEN;
+import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTSIZ;
+import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENLEN;
+import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENOFF;
+import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENSIZ;
+import static com.google.devtools.build.android.ziputils.EndOfCentralDirectory.ENDOFF;
+import static com.google.devtools.build.android.ziputils.EndOfCentralDirectory.ENDSUB;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+import java.util.Map.Entry;
+
+/**
+ * API for reading a zip file. This does not perform decompression of entry data, but provides
+ * a raw view of the content of a zip archive.
+ */
+public class ZipIn {
+
+ private static final byte[] EOCD_SIG = {0x50, 0x4b, 0x05, 0x06};
+ private static final byte[] HEADER_SIG = {0x50, 0x4b, 0x03, 0x04};
+ private static final byte[] DATA_DESC_SIG = {0x50, 0x4b, 0x07, 0x08};
+
+
+ /**
+ * Max end-of-central-directory size, including variable length file comment..
+ */
+ private static final int MAX_EOCD_SIZE = 1024;
+
+ /**
+ * Max local file header size, including long filename.
+ */
+ private static final int MAX_HEADER_SIZE = 64 * 1024;
+
+ /**
+ * Default size of direct byte buffer used for reading content. Actual allocation will not
+ * exceed the archive content size, and may be at least as big as the largest entry.
+ */
+ private static final int READ_BLOCK_SIZE = 20 * 1024 * 1024;
+
+ private final String filename; // filename or nickname.
+ private final FileChannel fileChannel; // input file.
+ private BufferedFile bufferedFile;
+ private CentralDirectory cdir = null;
+ private EndOfCentralDirectory eocd = null;
+ private final boolean useDirectory;
+ private final boolean ignoreDeleted;
+ private final boolean verbose = false;
+
+ /**
+ * Creates a {@code ZipIn} view of a file, with a (nick)name.
+ *
+ * @param channel File channel open for reading.
+ * @param filename filename or nickname.
+ */
+ public ZipIn(FileChannel channel, String filename) {
+ this.fileChannel = channel;
+ this.filename = filename;
+ this.useDirectory = true;
+ this.ignoreDeleted = useDirectory;
+ }
+
+ /**
+ * Gets the file name for this zip input file.
+ * @return the filename set at time of construction.
+ */
+ public String getFilename() {
+ return filename;
+ }
+
+ /**
+ * Returns a view of the "end of central directory" record expected at (or towards) the end of a
+ * zip file.
+ *
+ * @return A read-only, {@link EndOfCentralDirectory}.
+ * @throws IOException
+ */
+ public EndOfCentralDirectory endOfCentralDirectory() throws IOException {
+ if (eocd == null) {
+ loadEndOfCentralDirectory();
+ }
+ return eocd;
+ }
+
+ /**
+ * Returns a memory mapped view of the central directory.
+ *
+ * @return A read-only, {@link CentralDirectory} of the central directory.
+ * @throws IOException
+ */
+ public CentralDirectory centralDirectory() throws IOException {
+ if (cdir == null) {
+ loadCentralDirectory();
+ }
+ return cdir;
+ }
+
+ /**
+ * Scans all entries in the zip file and invokes the given {@link EntryHandler} on each.
+ *
+ * @param handler handler to invoke for each file entry.
+ * @throws IOException
+ */
+ public void scanEntries(EntryHandler handler) throws IOException {
+ centralDirectory();
+ ZipEntry zipEntry = nextFrom(null);
+ while (zipEntry.getCode() != ZipEntry.Status.ENTRY_NOT_FOUND) {
+ if (zipEntry.getCode() != ZipEntry.Status.ENTRY_OK) {
+ throw new IOException(zipEntry.getCode().toString());
+ }
+ handler.handle(this, zipEntry.getHeader(), zipEntry.getDirEntry(), zipEntry.getContent());
+ if (useDirectory && ignoreDeleted) {
+ zipEntry = ZipIn.this.nextFrom(zipEntry.getDirEntry());
+ } else {
+ zipEntry = nextFrom(zipEntry.limit());
+ }
+ }
+ }
+
+ /**
+ * Finds the next header, by scanning for a local header signature starting
+ * at {@code offset}. This method will find headers for deleted or updated entries that
+ * are not listed in the central directory, and may pickup false positive (e.g. entries
+ * of an embedded zip file stored without compression). This method is primarily intended
+ * for applications trying to recover data from corrupt archives.
+ *
+ * @param offset offset where to start the search.
+ * @return the next local header at or beyond {@code offset}, or {@code null} if no
+ * header is found.
+ * @throws IOException
+ */
+ public LocalFileHeader nextHeaderFrom(long offset) throws IOException {
+ int skipped = 0;
+ for (ByteBuffer buffer = getData(offset + skipped, MAX_HEADER_SIZE);
+ buffer.limit() >= LocalFileHeader.SIZE;
+ buffer = getData(offset + skipped, MAX_HEADER_SIZE)) {
+ int markerOffset = ScanUtil.scanTo(HEADER_SIG, buffer);
+ if (markerOffset < 0) {
+ skipped += buffer.limit() - 3;
+ } else {
+ skipped += markerOffset;
+ LocalFileHeader header = markerOffset == 0 ? localHeaderIn(buffer, offset + skipped)
+ : localHeaderAt(offset + skipped);
+ if (header != null) {
+ if (skipped > 0) {
+ System.out.println("Warning: local header search: skipped " + skipped + " bytes");
+ }
+ return header;
+ }
+ // If localHeaderIn or localHeaderAt decided it is not a header location,
+ // we continue the search.
+ skipped += 4;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Finds the header at the next higher offset listed in the central directory as containing
+ * a local file header, starting from the offset of the given {@code dirEntry}. This method will
+ * bypass any deleted or updated entries not listed in the directory, and also any entries from
+ * embedded zip files, or random instance of the header signature. This is the preferred method
+ * for sequentially reading the entries of a valid zip file.
+ *
+ * @param dirEntry directory entry for the "current entry", providing the start point
+ * for searching the central directory for the entry with the next higher offset.
+ * @return the next header according to the central directory, or {@code null} if there are no
+ * more headers.
+ * @throws IOException
+ */
+ public LocalFileHeader nextHeaderFrom(DirectoryEntry dirEntry) throws IOException {
+ Integer nextOffset = dirEntry == null ? -1 : dirEntry.get(CENOFF);
+ while ((nextOffset = cdir.mapByOffset().higherKey(nextOffset)) != null) {
+ LocalFileHeader header = localHeaderAt(nextOffset);
+ if (header != null) {
+ return header;
+ }
+ System.out.println("Warning: no header for file listed in directory "
+ + dirEntry.getFilename());
+ // The file is corrupt! Continue to see how bad it is.
+ }
+ return null;
+ }
+
+ /**
+ * Provides a {@code LocalFileHeader} view of a local header located at the offset indicated
+ * by the given {@code dirEntry}.
+ *
+ * @param dirEntry the directory entry referring to the headers location.
+ * @return the requested header, or {@code null} if the given location can't possibly contain a
+ * valid file header (e.g. missing header signature), or if {@code dirEntry} is {@code null}.
+ * @throws IOException
+ */
+ public LocalFileHeader localHeaderFor(DirectoryEntry dirEntry) throws IOException {
+ return dirEntry == null ? null : localHeaderAt(dirEntry.get(CENOFF));
+ }
+
+ /**
+ * Provides a {@code LocalFileHeader} view of a local header located at the offset indicated
+ * by the given {@code dirEntry}.
+ *
+ * @param offset offset a which the a header is presumed to exist.
+ * @return the requested header, or {@code null} if the given location can't possibly contain a
+ * valid file header (e.g. missing header signature).
+ * @throws IOException
+ */
+ public LocalFileHeader localHeaderAt(long offset) throws IOException {
+ return localHeaderIn(getData(offset, MAX_HEADER_SIZE), offset);
+ }
+
+ /**
+ * Finds the next zip file entry, by scanning for a local header using the
+ * {@link #nextHeaderFrom(long) }method.
+ *
+ * @param offset offset where to start the search.
+ * @return a {@code ZipEntry} object with the result of the search.
+ * @throws IOException
+ */
+ public ZipEntry nextFrom(long offset) throws IOException {
+ LocalFileHeader header = ZipIn.this.nextHeaderFrom(offset);
+ return entryWith(header);
+ }
+
+ /**
+ * Finds the next zip file entry, by first invoking
+ * {@link #nextHeaderFrom(com.google.devtools.build.android.ziputils.DirectoryEntry) }
+ * to find its header.
+ *
+ * @param entry the directory entry for the "current" zip entry, or {@code null} to get
+ * the first entry.
+ * @return a {@code ZipEntry} object with the result of the search.
+ * @throws IOException
+ */
+ public ZipEntry nextFrom(DirectoryEntry entry) throws IOException {
+ int offset = entry == null ? -1 : entry.get(CENOFF);
+ Entry<Integer, DirectoryEntry> mapEntry = cdir.mapByOffset().higherEntry(offset);
+ if (mapEntry == null) {
+ return entryWith(null);
+ }
+ LocalFileHeader header = localHeaderAt(mapEntry.getKey());
+ return entryWith(header, mapEntry.getValue());
+ }
+
+ /**
+ * Finds the zip file entry, for a given directory entry.
+ *
+ * @param entry the directory entry for which a zip entry is requested.
+ * @return a {@code ZipEntry} object with the result of the search.
+ * @throws IOException
+ */
+ public ZipEntry entryFor(DirectoryEntry entry) throws IOException {
+ return entryWith(localHeaderFor(entry), entry);
+ }
+
+ /**
+ * Returns the zip file entry at the given offset.
+ *
+ * @param offset presumed location of local file header.
+ * @return a {@link ZipEntry} for the given location.
+ * @throws IOException
+ */
+ public ZipEntry entryAt(long offset) throws IOException {
+ LocalFileHeader header = localHeaderAt(offset);
+ return entryWith(header);
+ }
+
+ /**
+ * Constructs a {@link ZipEntry} view of the entry at the location of the given header.
+ *
+ * @param header a previously located header. If (@code useDirectory} is set, this will
+ * attempt to lookup a corresponding directory entry. If there is none, and {@code ignoreDeleted}
+ * is also set, the return value will flag this entry with a
+ * {@code ZipEntry.Status.ENTRY_NOT_FOUND} status code.
+ *
+ * @return {@link ZipEntry} for the given location.
+ * @throws IOException
+ */
+ public ZipEntry entryWith(LocalFileHeader header) throws IOException {
+ if (header == null) {
+ return new ZipEntry().withCode(ZipEntry.Status.ENTRY_NOT_FOUND);
+ }
+ // header != null
+ long offset = header.fileOffset();
+ DirectoryEntry dirEntry = null;
+ if (useDirectory) {
+ dirEntry = cdir.mapByOffset().get((int) offset);
+ if (dirEntry == null && ignoreDeleted) {
+ return new ZipEntry().withCode(ZipEntry.Status.ENTRY_DELETED);
+ }
+ }
+ return entryWith(header, dirEntry);
+ }
+
+ /**
+ * Scans for a data descriptor from a given offset.
+ *
+ * @param offset position where to start the search.
+ * @param dirEntry directory entry for validation, or {@code null}.
+ * @return A data descriptor view for the next position containing the data descriptor signature.
+ * @throws IOException
+ */
+ public DataDescriptor descriptorFrom(final long offset, final DirectoryEntry dirEntry)
+ throws IOException {
+ int skipped = 0;
+ for (ByteBuffer buffer = getData(offset + skipped, MAX_HEADER_SIZE);
+ buffer.limit() >= 16; buffer = getData(offset + skipped, MAX_HEADER_SIZE)) {
+ int markerOffset = ScanUtil.scanTo(DATA_DESC_SIG, buffer);
+ if (markerOffset < 0) {
+ skipped += buffer.limit() - 3;
+ } else {
+ skipped += markerOffset;
+ return markerOffset == 0 ? descriptorIn(buffer, offset + skipped, dirEntry)
+ : descriptorAt(offset + skipped, dirEntry);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Creates a data descriptor view at a given offset.
+ *
+ * @param offset presumed location of data descriptor.
+ * @param dirEntry directory entry to use for validation, or {@code null}.
+ * @return a data descriptor view over the given file offset.
+ * @throws IOException
+ */
+ public DataDescriptor descriptorAt(long offset, DirectoryEntry dirEntry) throws IOException {
+ return descriptorIn(getData(offset, 16), offset, dirEntry);
+ }
+
+ /**
+ * Constructs a zip entry object for the location of the given header, with the corresponding
+ * directory entry.
+ *
+ * @param header local file header for the entry.
+ * @param dirEntry corresponding directory entry, or {@code null} if not available.
+ * @return a zip entry with the given header and directory entry.
+ * @throws IOException
+ */
+ private ZipEntry entryWith(LocalFileHeader header, DirectoryEntry dirEntry) throws IOException {
+ ZipEntry zipEntry = new ZipEntry().withHeader(header).withEntry(dirEntry);
+ int offset = (int) (header.fileOffset() + header.getSize());
+ // !useDirectory || dirEntry != null || !ignoreDeleted
+ String entryName = header.getFilename();
+ if (dirEntry != null && !entryName.equals(dirEntry.getFilename())) {
+ return zipEntry.withEntry(dirEntry).withCode(ZipEntry.Status.FILENAME_ERROR);
+ }
+ int sizeByHeader = header.dataSize();
+ int sizeByDir = dirEntry != null ? dirEntry.dataSize() : -1;
+ ByteBuffer content;
+ if (sizeByDir == sizeByHeader && sizeByDir >= 0) {
+ // Ideal case, header and directory in agreement
+ content = getData(offset, sizeByHeader);
+ if (content.limit() == sizeByHeader) {
+ return zipEntry.withContent(content).withCode(ZipEntry.Status.ENTRY_OK);
+ } else {
+ return zipEntry.withContent(content).withCode(ZipEntry.Status.NOT_ENOUGH_DATA);
+ }
+ }
+ if (sizeByDir >= 0) {
+ // If file is correct, we get here because of a 0x8 flag, and we expect
+ // data to be followed by a data descriptor.
+ content = getData(offset, sizeByDir);
+ DataDescriptor dataDesc = descriptorAt(offset + sizeByDir, dirEntry);
+ if (dataDesc != null) {
+ return zipEntry.withContent(content).withDescriptor(dataDesc).withCode(
+ ZipEntry.Status.ENTRY_OK);
+ }
+ return zipEntry.withContent(content).withCode(ZipEntry.Status.NO_DATA_DESC);
+ }
+ if (!ignoreDeleted) {
+ if (sizeByHeader >= 0) {
+ content = getData(offset, sizeByHeader);
+ if (content.limit() == sizeByHeader) {
+ return zipEntry.withContent(content).withCode(ZipEntry.Status.ENTRY_OK);
+ }
+ return zipEntry.withContent(content).withCode(ZipEntry.Status.NOT_ENOUGH_DATA);
+ } else {
+
+ DataDescriptor dataDesc = descriptorFrom(offset, dirEntry);
+ if (dataDesc == null) {
+ // Only way now would be to decompress
+ return zipEntry.withCode(ZipEntry.Status.UNKNOWN_SIZE);
+ }
+ int sizeByDesc = dataDesc.get(EXTSIZ);
+ if (sizeByDesc != dataDesc.fileOffset() - offset) {
+ // That just can't be the right
+ return zipEntry.withDescriptor(dataDesc).withCode(ZipEntry.Status.UNKNOWN_SIZE);
+ }
+ content = getData(offset, sizeByDesc);
+ return zipEntry.withContent(content).withDescriptor(dataDesc).withCode(
+ ZipEntry.Status.ENTRY_OK);
+ }
+ }
+ return zipEntry.withCode(ZipEntry.Status.UNKNOWN_SIZE);
+ }
+
+ /**
+ * Constructs a local header view over a give byte buffer.
+ *
+ * @param buffer byte buffer with local header data.
+ * @param offset file offset at which the buffer is based.
+ * @return a local header view.
+ */
+ private LocalFileHeader localHeaderIn(ByteBuffer buffer, long offset) {
+ return buffer.limit() < LocalFileHeader.SIZE
+ || buffer.getInt(0) != LocalFileHeader.SIGNATURE
+ ? null : LocalFileHeader.viewOf(buffer).at(offset);
+ }
+
+ /**
+ * Constructs a data descriptor view over a given byte buffer.
+ *
+ * @param buf byte buffer with data descriptor data.
+ * @param offset file offset at which the buffer is based.
+ * @param dirEntry directory entry with presumed reliable content size information.
+ * @return a data descriptor
+ */
+ private DataDescriptor descriptorIn(ByteBuffer buf, long offset, DirectoryEntry dirEntry) {
+ if (buf.limit() < 12) {
+ return null;
+ }
+ DataDescriptor desc = DataDescriptor.viewOf(buf).at(offset);
+ if (desc.hasMarker() || (dirEntry != null
+ && desc.get(EXTSIZ) == dirEntry.get(CENSIZ)
+ && desc.get(EXTLEN) == dirEntry.get(CENLEN))) {
+ return desc;
+ }
+ return null;
+ }
+
+ /**
+ * Obtains a byte buffer at a given offset.
+ */
+ private ByteBuffer getData(long offset, int size) throws IOException {
+ return bufferedFile.getBuffer(offset, size).order(ByteOrder.LITTLE_ENDIAN);
+ }
+
+ /**
+ * Locates the "end of central directory" record, expected located at the end of the file, and
+ * reads it into a byte buffer. Called on the first invocation of
+ * {@link #endOfCentralDirectory() }.
+ *
+ * @throws IOException
+ */
+ protected void loadEndOfCentralDirectory() throws IOException {
+ cdir = null;
+ long size = fileChannel.size();
+ verbose("Loading ZipIn: " + filename);
+ verbose("-- size: " + size);
+ int cap = (int) Math.min(size, MAX_EOCD_SIZE);
+ ByteBuffer buffer = ByteBuffer.allocate(cap).order(ByteOrder.LITTLE_ENDIAN);
+ long offset = size - cap;
+ while (true) {
+ fileChannel.position(offset);
+ while (buffer.hasRemaining()) {
+ fileChannel.read(buffer, offset);
+ }
+ // scan to find it...
+ int endOfDirOffset = ScanUtil.scanBackwardsTo(EOCD_SIG, buffer);
+ if (endOfDirOffset < 0) {
+ if (offset == 0) {
+ if (useDirectory) {
+ throw new IllegalStateException("No end of central directory marker");
+ } else {
+ break;
+ }
+ }
+ offset = Math.max(offset - 1000, 0);
+ buffer.clear();
+ continue;
+ }
+ long eocdFileOffset = offset + endOfDirOffset;
+ verbose("-- EOCD: " + eocdFileOffset + " size: " + (size - eocdFileOffset));
+ buffer.position(endOfDirOffset);
+ eocd = EndOfCentralDirectory.viewOf(buffer).at(offset + endOfDirOffset);
+ // TODO (bazel-team): check that the end of central directory, points to a valid
+ // first directory entry. If not, assume we happened to find the signature inside
+ // a file comment, and resume the search.
+ break;
+ }
+
+ if (eocd != null) {
+ bufferedFile = new BufferedFile(fileChannel, 0, eocd.get(ENDOFF),
+ READ_BLOCK_SIZE);
+ } else {
+ bufferedFile = new BufferedFile(fileChannel, READ_BLOCK_SIZE);
+ }
+ }
+
+ /**
+ * Maps the central directory to memory. Called on the first invocation of
+ * {@link #centralDirectory() }.
+ *
+ * @throws IOException
+ */
+ protected void loadCentralDirectory() throws IOException {
+ if (eocd == null) {
+ loadEndOfCentralDirectory();
+ }
+ if (eocd == null) {
+ return;
+ }
+ long cdOffset = eocd.get(ENDOFF);
+ long len = eocd.fileOffset() - cdOffset;
+ verbose("-- CDIR: " + cdOffset + " size: " + len + " count: " + eocd.get(ENDSUB));
+ // Read directory to buffer.
+ // TODO(bazel-team): we currently assume the directory fits in memory (and int).
+ ByteBuffer buffer = ByteBuffer.allocateDirect((int) len);
+ while (len > 0) {
+ int read = fileChannel.read(buffer, cdOffset);
+ len -= read;
+ cdOffset += read;
+ }
+ buffer.rewind();
+ cdir = CentralDirectory.viewOf(buffer).at(cdOffset).parse();
+ cdir.buffer.flip();
+ }
+
+ /**
+ * Zip file entry container class, for use with the low-level scanning operations of this
+ * API, supporting zip file scanner construction.
+ */
+ public static class ZipEntry {
+
+ private LocalFileHeader header;
+ private DataDescriptor descriptor;
+ private ByteBuffer content;
+ private DirectoryEntry entry;
+ private Status code;
+
+ /**
+ * Creates a zip entry, setting the initial status to not found.
+ */
+ public ZipEntry() {
+ code = Status.ENTRY_NOT_FOUND;
+ }
+
+ /**
+ * Gets the header of this zip entry.
+ */
+ public LocalFileHeader getHeader() {
+ return header;
+ }
+
+ /**
+ * Sets the header of this zip entry.
+ * @return this object.
+ */
+ public ZipEntry withHeader(LocalFileHeader header) {
+ this.header = header;
+ return this;
+ }
+
+ /**
+ * Gets the data descriptor of this zip entry, if any.
+ */
+ public DataDescriptor getDescriptor() {
+ return descriptor;
+ }
+
+ /**
+ * Sets the data descriptor of this zip entry.
+ * @return this object.
+ */
+ public ZipEntry withDescriptor(DataDescriptor descriptor) {
+ this.descriptor = descriptor;
+ return this;
+ }
+
+ /**
+ * Gets a byte buffer for accessing the raw content of this zip entry.
+ */
+ public ByteBuffer getContent() {
+ return content;
+ }
+
+ /**
+ * Sets the byte buffer providing access to the raw content of this zip entry.
+ * @return this object
+ */
+ public ZipEntry withContent(ByteBuffer content) {
+ this.content = content;
+ return this;
+ }
+
+ /**
+ * Gets the central directory entry for this zip entry, if any.
+ */
+ public DirectoryEntry getDirEntry() {
+ return entry;
+ }
+
+ /**
+ * Sets the central directory entry for this zip entry.
+ * @return this object.
+ */
+ public ZipEntry withEntry(DirectoryEntry entry) {
+ this.entry = entry;
+ return this;
+ }
+
+ /**
+ * Gets the status code for parsing this zip entry.
+ */
+ public Status getCode() {
+ return code;
+ }
+
+ /**
+ * Sets the status code for this zip entry.
+ * @return this object.
+ */
+ public ZipEntry withCode(Status code) {
+ this.code = code;
+ return this;
+ }
+
+ /**
+ * Calculates, best-effort, the file offset just past this zip entry.
+ */
+ public long limit() {
+ if (header == null) {
+ return 0;
+ }
+ if (descriptor != null) {
+ return descriptor.fileOffset() + descriptor.getSize();
+ }
+ long offset = header.fileOffset() + header.dataSize();
+ if (content != null) {
+ offset += content.limit();
+ }
+ return offset;
+ }
+
+ /**
+ * Zip entry parsing status codes.
+ */
+ public enum Status {
+ /**
+ * This zip entry contains valid header and data
+ */
+ ENTRY_OK,
+ /**
+ * No header at the given location
+ */
+ ENTRY_NOT_FOUND,
+ /**
+ * The given location contains a header that is not listed in the central directory
+ */
+ ENTRY_DELETED,
+ /**
+ * The header in the given location has a different filename than the
+ * directory entry for this location.
+ */
+ FILENAME_ERROR,
+ /**
+ * The given location has the header signature, but the remaining data is insufficient
+ * to constitute a complete entry.
+ */
+ NOT_ENOUGH_DATA,
+ /**
+ * The entry appears to be missing an expected data descriptor.
+ */
+ NO_DATA_DESC,
+ /**
+ * The implementation was unable to determine the size of the content of the entry.
+ * The client will have to either parse using the central directory, or if all else
+ * fails, attempt to decompress the entry.
+ */
+ UNKNOWN_SIZE,
+ }
+ }
+
+ private void verbose(String msg) {
+ if (verbose) {
+ System.out.println(msg);
+ }
+ }
+}
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/ZipOut.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/ZipOut.java
new file mode 100644
index 0000000..c1954d4
--- /dev/null
+++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/ZipOut.java
@@ -0,0 +1,227 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.android.ziputils;
+
+import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENOFF;
+import static com.google.devtools.build.android.ziputils.EndOfCentralDirectory.ENDOFF;
+import static com.google.devtools.build.android.ziputils.EndOfCentralDirectory.ENDSIZ;
+import static com.google.devtools.build.android.ziputils.EndOfCentralDirectory.ENDSUB;
+import static com.google.devtools.build.android.ziputils.EndOfCentralDirectory.ENDTOT;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * API for writing to a zip archive. This does not currently perform compression,
+ * but merely provides the facilities for creating a zip archive. The client must
+ * ensure that file content is written conforming to the created headers.
+ */
+public class ZipOut {
+ /**
+ * Central directory output buffer block size
+ */
+ private static final int DIR_BLOCK_SIZE = 1024 * 1024;
+
+ private final String filename;
+ // To ensure good performance when writing to a remote file system
+ // we need to use asynchronous output. However, because we may want this
+ // to run on and off devices, we can't depend on java.nio.AsynchronousFileChannel (JDK 1.7).
+ // Instead, we use a FileChannel (JDK 1.4), and use a single-thread pool,
+ // to execute writes serially, but non-blocking from our client's
+ // point-of-view. All data written, are assumed to remain unchanged until the write is complete.
+ // ZipIn is designed to not reuse internal buffers, to make direct data transfer safe.
+ // This optimizes the common cases where input is processed serially.
+ private final FileChannel fileChannel;
+ private final ExecutorService executor;
+ private final List<Future<?>> futures;
+ private final List<CentralDirectory> centralDirectory;
+ private CentralDirectory current;
+ private int fileOffset = 0;
+ private int entryCount = 0;
+ private boolean finished = false;
+ private final boolean verbose = false;
+
+ /**
+ * Creates a {@code ZipOut} for writing to file, with the given (nick)name.
+ *
+ * @param channel File channel open for output.
+ * @param filename File name or nickname.
+ * @throws java.io.IOException
+ */
+ public ZipOut(FileChannel channel, String filename) throws IOException {
+ this.executor = Executors.newSingleThreadExecutor();
+ this.futures = new ArrayList<>();
+ this.fileChannel = channel;
+ this.filename = filename;
+ centralDirectory = new ArrayList<>();
+ fileOffset = (int) fileChannel.position();
+ }
+
+ /**
+ * Returns a writable copy of the given
+ * {@link com.google.devtools.build.android.ziputils.DirectoryEntry}, backed by an internal
+ * direct byte buffer, allocated over storage for this files central directory. The file offset
+ * is set, and must not be changed. The variable entry data (filename, extra data and comment),
+ * must not be changed (in a way that changes the total size of the directory entry).
+ * @param entry directory entry to copy.
+ * @return a writable directory entry view, over the provided byte buffer.
+ */
+ public DirectoryEntry nextEntry(DirectoryEntry entry) {
+ entryCount++;
+ int size = entry.getSize();
+ if (current == null || current.buffer.remaining() < size) {
+ ByteBuffer buffer = ByteBuffer.allocateDirect(DIR_BLOCK_SIZE);
+ current = CentralDirectory.viewOf(buffer);
+ centralDirectory.add(current);
+ }
+ return current.nextEntry(entry).set(CENOFF, fileOffset);
+ }
+
+ /**
+ * Writes content to the current entry. Content is written as-is, and the client is responsible
+ * for compression, consistent with the storage method set in the current directory entry. The
+ * client must first write an appropriate local file header, and if necessary, complete the entry
+ * with a data descriptor. If the header indicates that the content is compressed, the client is
+ * responsible for compressing the data before writing.
+ *
+ * <p>Data is written serially, but asynchronously, to the output file. The client must not change
+ * the underlying data after it has been scheduled for writing. Usually, a client will release
+ * any references to the data, so that storage may be eligible for GC, once the write operation
+ * has completed. It's safe to pass references to views of data obtained from a {#link ZipIn},
+ * object, because {@code ZipIn} doesn't reuse internal buffers.
+ *
+ * @param content
+ */
+ public synchronized void write(ByteBuffer content) {
+ fileOffset += content.remaining();
+ futures.add(executor.submit(new OutputTask(content)));
+ }
+
+ /**
+ * Writes a {@link com.google.devtools.build.android.ziputils.View} to the current entry.
+ * Used to write a {@link com.google.devtools.build.android.ziputils.LocalFileHeader}
+ * before the content, and if needed, a
+ * {@link com.google.devtools.build.android.ziputils.DataDescriptor} after the content.
+ * <P>
+ * See also {@link #write(java.nio.ByteBuffer)}.
+ * </P>
+ *
+ * @param view the view to write as part of the current entry.
+ * @throws java.io.IOException
+ */
+ public void write(View<?> view) throws IOException {
+ view.at(fileOffset).buffer.rewind();
+ write(view.buffer);
+ }
+
+ /**
+ * Returns the file position for the next write operation. Because writes are asynchronous, this
+ * may not be the actual position of the underlying file channel.
+ * @return the file position position for the next write operation.
+ */
+ public int fileOffset() {
+ return fileOffset;
+ }
+
+ /**
+ * Writes out the central directory. This doesn't close the output file.
+ */
+ public void finish() throws IOException {
+ if (finished) {
+ return;
+ }
+ finished = true;
+ int cdOffset = fileOffset;
+ for (CentralDirectory cd : centralDirectory) {
+ //cd.finish().buffer.rewind();
+ cd.buffer.flip();
+ write(cd.buffer);
+ }
+ int size = fileOffset - cdOffset;
+ verbose("ZipOut finishing: " + filename);
+ verbose("-- CDIR: " + cdOffset + " count: " + entryCount);
+ verbose("-- EOCD: " + fileOffset + " size: " + size);
+ EndOfCentralDirectory eocd = EndOfCentralDirectory.allocate(null)
+ .set(ENDSUB, (short) entryCount)
+ .set(ENDTOT, (short) entryCount)
+ .set(ENDSIZ, size)
+ .set(ENDOFF, cdOffset)
+ .at(fileOffset);
+ eocd.buffer.rewind();
+ write(eocd.buffer);
+ verbose("-- size: " + fileOffset);
+ }
+
+ /**
+ * Closes the output file. If this object has not been finished yet, this method will call
+ * {@link #finish()} before closing the output channel.
+ *
+ * @throws java.io.IOException
+ */
+ public void close() throws IOException {
+ if (!finished) {
+ finish();
+ }
+ try {
+ executor.shutdown();
+ executor.awaitTermination(30, TimeUnit.SECONDS);
+ for (Future<?> f : futures) {
+ try {
+ f.get();
+ } catch (ExecutionException ex) {
+ throw new IOException(ex.getCause().getMessage());
+ }
+ }
+ } catch (InterruptedException ex) {
+ executor.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
+ fileChannel.close();
+ }
+
+ /**
+ * Helper class to write asynchronously to the output channel.
+ */
+ private class OutputTask implements Runnable {
+
+ final ByteBuffer buffer;
+
+ public OutputTask(ByteBuffer buffer) {
+ this.buffer = buffer;
+ }
+
+ @Override
+ public void run() {
+ try {
+ fileChannel.write(buffer);
+ } catch (IOException ex) {
+ throw new IllegalStateException("Unexpected IOException writing to output channel");
+ }
+ }
+ }
+
+ private void verbose(String msg) {
+ if (verbose) {
+ System.out.println(msg);
+ }
+ }
+}
