src/tools/android/java/com/google/devtools/build/android/dexer/DexFileMerger.java

// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.android.dexer;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.concurrent.TimeUnit.SECONDS;

import com.android.dex.Dex;
import com.android.dex.DexFormat;
import com.android.dx.command.dexer.DxContext;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.io.ByteStreams;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.devtools.build.android.Converters.ExistingPathConverter;
import com.google.devtools.build.android.Converters.PathConverter;
import com.google.devtools.common.options.EnumConverter;
import com.google.devtools.common.options.Option;
import com.google.devtools.common.options.OptionDocumentationCategory;
import com.google.devtools.common.options.OptionEffectTag;
import com.google.devtools.common.options.OptionsBase;
import com.google.devtools.common.options.OptionsParser;
import com.google.devtools.common.options.ShellQuotedParamsFilePreProcessor;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;

/**
 * Tool used by Bazel as a replacement for Android's {@code dx} tool that assembles a single or, if
 * allowed and necessary, multiple {@code .dex} files from a given archive of {@code .dex} and
 * {@code .class} files.  The tool merges the {@code .dex} files it encounters into a single file
 * and additionally encodes any {@code .class} files it encounters.  If multidex is allowed then the
 * tool will generate multiple files subject to the {@code .dex} file format's limits on the number
 * of methods and fields.
 */
class DexFileMerger {

  /** File name prefix of a {@code .dex} file automatically loaded in an archive. */
  private static final String DEX_PREFIX = "classes";

  /**
   * Commandline options.
   */
  public static class Options extends OptionsBase {
    @Option(
        name = "input",
        allowMultiple = true,
        defaultValue = "null",
        category = "input",
        documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
        effectTags = {OptionEffectTag.UNKNOWN},
        converter = ExistingPathConverter.class,
        abbrev = 'i',
        help =
            "Input archives with .dex files to merge.  Inputs are processed in given order, so"
                + " classes from later inputs will be added after earlier inputs.  Duplicate"
                + " classes are dropped.")
    public List<Path> inputArchives;

    @Option(
      name = "output",
      defaultValue = "classes.dex.jar",
      category = "output",
      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
      effectTags = {OptionEffectTag.UNKNOWN},
      converter = PathConverter.class,
      abbrev = 'o',
      help = "Output archive to write."
    )
    public Path outputArchive;

    @Option(
      name = "multidex",
      defaultValue = "off",
      category = "multidex",
      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
      effectTags = {OptionEffectTag.UNKNOWN},
      converter = MultidexStrategyConverter.class,
      help = "Allow more than one .dex file in the output."
    )
    public MultidexStrategy multidexMode;

    @Option(
      name = "main-dex-list",
      defaultValue = "null",
      category = "multidex",
      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
      effectTags = {OptionEffectTag.UNKNOWN},
      converter = ExistingPathConverter.class,
      help = "List of classes to be placed into \"main\" classes.dex file."
    )
    public Path mainDexListFile;

    @Option(
      name = "minimal-main-dex",
      defaultValue = "false",
      category = "multidex",
      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
      effectTags = {OptionEffectTag.UNKNOWN},
      help =
          "If true, *only* classes listed in --main_dex_list file are placed into \"main\" "
              + "classes.dex file."
    )
    public boolean minimalMainDex;

    @Option(
      name = "verbose",
      defaultValue = "false",
      category = "misc",
      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
      effectTags = {OptionEffectTag.UNKNOWN},
      help = "If true, print information about the merged files and resulting files to stdout."
    )
    public boolean verbose;

    @Option(
      name = "max-bytes-wasted-per-file",
      defaultValue = "0",
      category = "misc",
      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
      effectTags = {OptionEffectTag.UNKNOWN},
      help =
          "Limit on conservatively allocated but unused bytes per dex file, which can enable "
              + "faster merging."
    )
    public int wasteThresholdPerDex;

    // Undocumented dx option for testing multidex logic
    @Option(
      name = "set-max-idx-number",
      defaultValue = "" + DexFormat.MAX_MEMBER_IDX,
      documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
      effectTags = {OptionEffectTag.UNKNOWN},
      help = "Limit on fields and methods in a single dex file."
    )
    public int maxNumberOfIdxPerDex;

    @Option(
      name = "forceJumbo",
      defaultValue = "false", // dx's default
      documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
      effectTags = {OptionEffectTag.UNKNOWN},
      help = "Typically not needed flag intended to imitate dx's --forceJumbo."
    )
    public boolean forceJumbo;

    @Option(
      name = "dex_prefix",
      defaultValue = DEX_PREFIX, // dx's default
      category = "misc",
      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
      effectTags = {OptionEffectTag.UNKNOWN},
      help = "Dex file output prefix."
    )
    public String dexPrefix;
  }

  public static class MultidexStrategyConverter extends EnumConverter<MultidexStrategy> {
    public MultidexStrategyConverter() {
      super(MultidexStrategy.class, "multidex strategy");
    }
  }

  public static void main(String[] args) throws Exception {
    OptionsParser optionsParser =
        OptionsParser.builder()
            .optionsClasses(Options.class)
            .argsPreProcessor(new ShellQuotedParamsFilePreProcessor(FileSystems.getDefault()))
            .build();
    optionsParser.parseAndExitUponError(args);

    buildMergedDexFiles(optionsParser.getOptions(Options.class));
  }

  @VisibleForTesting
  static void buildMergedDexFiles(Options options) throws IOException {
    ListeningExecutorService executor;
    checkArgument(!options.inputArchives.isEmpty(), "Need at least one --input");
    checkArgument(
        options.mainDexListFile == null || options.inputArchives.size() == 1,
        "--main-dex-list only supported with exactly one --input, use DexFileSplitter for more");
    if (options.multidexMode.isMultidexAllowed()) {
      executor = createThreadPool();
    } else {
      checkArgument(
          options.mainDexListFile == null,
          "--main-dex-list is only supported with multidex enabled, but mode is: %s",
          options.multidexMode);
      checkArgument(
          !options.minimalMainDex,
          "--minimal-main-dex is only supported with multidex enabled, but mode is: %s",
          options.multidexMode);
      // We'll only ever merge and write one dex file, so multi-threading is pointless.
      executor = MoreExecutors.newDirectExecutorService();
    }

    ImmutableSet<String> classesInMainDex = options.mainDexListFile != null
        ? ImmutableSet.copyOf(Files.readAllLines(options.mainDexListFile, UTF_8))
        : null;
    PrintStream originalStdOut = System.out;
    try (DexFileAggregator out = createDexFileAggregator(options, executor)) {
      if (!options.verbose) {
        // com.android.dx.merge.DexMerger prints status information to System.out that we silence
        // here unless it was explicitly requested.  (It also prints debug info to DxContext.out,
        // which we populate accordingly below.)
        System.setOut(Dexing.nullout);
      }

      LinkedHashSet<String> seen = new LinkedHashSet<>();
      for (Path inputArchive : options.inputArchives) {
        // Simply merge files from inputs in order.  Doing that with a main dex list doesn't work,
        // but we rule out more than one input with a main dex list above.
        try (ZipFile zip = new ZipFile(inputArchive.toFile())) {
          ArrayList<ZipEntry> dexFiles = filesToProcess(zip);
          if (classesInMainDex == null) {
            processDexFiles(zip, dexFiles, seen, out);
          } else {
            // To honor --main_dex_list make two passes:
            // 1. process only the classes listed in the given file
            // 2. process the remaining files
            Predicate<ZipEntry> mainDexFilter =
                ZipEntryPredicates.classFileFilter(classesInMainDex);
            processDexFiles(zip, Iterables.filter(dexFiles, mainDexFilter), seen, out);
            // Fail if main_dex_list is too big, following dx's example
            checkState(out.getDexFilesWritten() == 0, "Too many classes listed in main dex list "
                + "file %s, main dex capacity exceeded", options.mainDexListFile);
            if (options.minimalMainDex) {
              out.flush(); // Start new .dex file if requested
            }
            processDexFiles(
                zip, Iterables.filter(dexFiles, Predicates.not(mainDexFilter)), seen, out);
          }
        }
      }
    } finally {
      // Kill threads in the pool so we don't hang
      MoreExecutors.shutdownAndAwaitTermination(executor, 1, SECONDS);
      System.setOut(originalStdOut);
    }
  }

  /**
   * Returns all .dex and .class files in the given zip.  .class files are unexpected but we'll
   * deal with them later.
   */
  private static ArrayList<ZipEntry> filesToProcess(ZipFile zip) {
    ArrayList<ZipEntry> result = Lists.newArrayList(
        Iterators.filter(
            Iterators.forEnumeration(zip.entries()),
            Predicates.and(
                Predicates.not(ZipEntryPredicates.isDirectory()),
                ZipEntryPredicates.suffixes(".dex", ".class"))));
    Collections.sort(result, ZipEntryComparator.LIKE_DX);
    return result;
  }

  private static void processDexFiles(
      ZipFile zip,
      Iterable<ZipEntry> filesToProcess,
      LinkedHashSet<String> seen,
      DexFileAggregator out)
      throws IOException {
    for (ZipEntry entry : filesToProcess) {
      String filename = entry.getName();
      checkState(filename.endsWith(".dex"), "Input shouldn't contain .class files: %s", filename);
      if (!seen.add(filename)) {
        continue;  // pick first occurrence of each file to match how JVM treats dupes on classpath
      }
      try (InputStream content = zip.getInputStream(entry)) {
        // We don't want to use the Dex(InputStream) constructor because it closes the stream,
        // which will break the for loop, and it has its own bespoke way of reading the file into
        // a byte buffer before effectively calling Dex(byte[]) anyway.
        out.add(new Dex(ByteStreams.toByteArray(content)));
      }
    }
  }

  private static DexFileAggregator createDexFileAggregator(
      Options options, ListeningExecutorService executor) throws IOException {
    String filePrefix = options.dexPrefix;
    if (options.multidexMode == MultidexStrategy.GIVEN_SHARD) {
      checkArgument(options.inputArchives.size() == 1,
          "--multidex=given_shard requires exactly one --input");
      Pattern namingPattern = Pattern.compile("([0-9]+)\\..*");
      Matcher matcher = namingPattern.matcher(options.inputArchives.get(0).toFile().getName());
      checkArgument(matcher.matches(),
          "expect input named <N>.xxx.zip for --multidex=given_shard but got %s",
          options.inputArchives.get(0).toFile().getName());
      int shard = Integer.parseInt(matcher.group(1));
      checkArgument(shard > 0, "expect positive N in input named <N>.xxx.zip but got %s", shard);
      if (shard > 1) {  // first shard conventionally isn't numbered
        filePrefix += shard;
      }
    }
    return new DexFileAggregator(
        new DxContext(options.verbose ? System.out : ByteStreams.nullOutputStream(), System.err),
        new DexFileArchive(
            new ZipOutputStream(
                new BufferedOutputStream(Files.newOutputStream(options.outputArchive)))),
        executor,
        options.multidexMode,
        options.forceJumbo,
        options.maxNumberOfIdxPerDex,
        options.wasteThresholdPerDex,
        filePrefix);
  }

  /**
   * Creates an unbounded thread pool executor, which is appropriate here since the number of tasks
   * we will add to the thread pool is at most dozens and some of them perform I/O (ie, may block).
   */
  private static ListeningExecutorService createThreadPool() {
    return MoreExecutors.listeningDecorator(Executors.newCachedThreadPool());
  }

  private DexFileMerger() {
  }
}