src/tools/android/java/com/google/devtools/build/android/AndroidDataMerger.java

// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.android;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.devtools.build.android.AndroidResourceMerger.MergingException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;

/** Handles the Merging of ParsedAndroidData. */
class AndroidDataMerger {

  public static class MergeConflictException extends UserException {

    private MergeConflictException(String message) {
      super(message);
    }

    static MergeConflictException withMessage(String message) {
      return new MergeConflictException(message);
    }
  }

  private static final Logger logger = Logger.getLogger(AndroidDataMerger.class.getCanonicalName());

  /** Interface for comparing paths. */
  interface SourceChecker {
    boolean checkEquality(DataSource one, DataSource two) throws IOException;
  }

  /** Compares two paths by the contents of the files. */
  static class ContentComparingChecker implements SourceChecker {

    static SourceChecker create() {
      return new ContentComparingChecker();
    }

    @Override
    public boolean checkEquality(DataSource one, DataSource two) throws IOException {
      if (one.getPath().equals(two.getPath())) {
        return true;
      }
      // TODO(corysmith): Is there a filesystem hash we can use?
      if (one.getFileSize() != two.getFileSize()) {
        return false;
      }
      try (final InputStream oneStream = one.newBufferedInputStream();
          final InputStream twoStream = two.newBufferedInputStream()) {
        int bytesRead = 0;
        while (true) {
          int oneByte = oneStream.read();
          int twoByte = twoStream.read();
          bytesRead++;
          if (oneByte == -1 || twoByte == -1) {
            if (oneByte == twoByte) {
              return true;
            } else {
              // getFileSize did not return correct size.
              logger.severe(
                  String.format(
                      "Filesystem size of %s (%s) or %s (%s) is inconsistent with bytes read %s.",
                      one, one.getFileSize(), two, two.getFileSize(), bytesRead));
              return false;
            }
          }
          if (oneByte != twoByte) {
            return false;
          }
        }
      }
    }
  }

  static class NoopSourceChecker implements SourceChecker {
    static SourceChecker create() {
      return new NoopSourceChecker();
    }

    @Override
    public boolean checkEquality(DataSource one, DataSource two) {
      return false;
    }
  }

  private final SourceChecker deDuplicator;
  private final ListeningExecutorService executorService;
  private final AndroidDataDeserializer deserializer;

  /** Creates a merger with no path deduplication and a default {@link ExecutorService}. */
  @VisibleForTesting
  static AndroidDataMerger createWithDefaults() {
    return createWithDefaultThreadPool(NoopSourceChecker.create());
  }

  /** Creates a merger with a custom deduplicator and a default {@link ExecutorService}. */
  @VisibleForTesting
  static AndroidDataMerger createWithDefaultThreadPool(SourceChecker deDuplicator) {
    return new AndroidDataMerger(
        deDuplicator,
        MoreExecutors.newDirectExecutorService(),
        AndroidParsedDataDeserializer.create());
  }

  /** Creates a merger with a file contents hashing deduplicator. */
  static AndroidDataMerger createWithPathDeduplictor(
      ListeningExecutorService executorService,
      AndroidDataDeserializer deserializer,
      SourceChecker checker) {
    return new AndroidDataMerger(checker, executorService, deserializer);
  }

  private AndroidDataMerger(
      SourceChecker deDuplicator,
      ListeningExecutorService executorService,
      AndroidDataDeserializer deserializer) {
    this.deDuplicator = deDuplicator;
    this.executorService = executorService;
    this.deserializer = deserializer;
  }

  /**
   * Loads a list of dependency {@link SerializedAndroidData} and merge with the primary {@link
   * ParsedAndroidData}.
   *
   * @see AndroidDataMerger#merge(ParsedAndroidData, ParsedAndroidData, UnvalidatedAndroidData,
   *     boolean, boolean) for details.
   */
  UnwrittenMergedAndroidData loadAndMerge(
      List<? extends SerializedAndroidData> transitive,
      List<? extends SerializedAndroidData> direct,
      ParsedAndroidData primary,
      Path primaryManifest,
      boolean allowPrimaryOverrideAll,
      boolean throwOnResourceConflict) {
    Stopwatch timer = Stopwatch.createStarted();
    try {
      logger.fine(
          String.format("Merged dependencies read in %sms", timer.elapsed(TimeUnit.MILLISECONDS)));
      timer.reset().start();
      return doMerge(
          ParsedAndroidData.loadedFrom(
              DependencyInfo.DependencyType.TRANSITIVE, transitive, executorService, deserializer),
          ParsedAndroidData.loadedFrom(
              DependencyInfo.DependencyType.DIRECT, direct, executorService, deserializer),
          primary,
          primaryManifest,
          allowPrimaryOverrideAll,
          throwOnResourceConflict);
    } finally {
      logger.fine(String.format("Resources merged in %sms", timer.elapsed(TimeUnit.MILLISECONDS)));
    }
  }

  /**
   * Merges DataResources into an UnwrittenMergedAndroidData.
   *
   * <p>This method has two basic states, library and binary. These are distinguished by
   * allowPrimaryOverrideAll, which allows the primary data to overwrite any value in the closure, a
   * trait associated with binaries, as a binary is a leaf node. The other semantics are slightly
   * more complicated: a given resource can be overwritten only if it resides in the direct
   * dependencies of primary data. This forces an explicit simple priority for each resource,
   * instead of the more subtle semantics of multiple layers of libraries with potential overwrites.
   *
   * <p>The UnwrittenMergedAndroidData contains only one of each DataKey in both the direct and
   * transitive closure.
   *
   * <p>The merge semantics for overwriting resources (non id and styleable) are as follows:
   *
   * <pre>
   *   Key:
   *     A(): package A
   *     A(foo): package A with resource symbol foo
   *     A() -> B(): a dependency relationship of B.deps = [:A]
   *     A(),B() -> C(): a dependency relationship of C.deps = [:A,:B]
   *
   *   For android library (allowPrimaryOverrideAll = False)
   *
   *     A() -> B(foo) -> C(foo) == Valid
   *     A() -> B() -> C(foo) == Valid
   *     A() -> B() -> C(foo),D(foo) == Conflict
   *     A(foo) -> B(foo) -> C() == Conflict
   *     A(foo) -> B() -> C(foo) == Conflict
   *     A(foo),B(foo) -> C() -> D() == Conflict
   *     A() -> B(foo),C(foo) -> D() == Conflict
   *     A(foo),B(foo) -> C() -> D(foo) == Conflict
   *     A() -> B(foo),C(foo) -> D(foo) == Conflict
   *
   *   For android binary (allowPrimaryOverrideAll = True)
   *
   *     A() -> B(foo) -> C(foo) == Valid
   *     A() -> B() -> C(foo) == Valid
   *     A() -> B() -> C(foo),D(foo) == Conflict
   *     A(foo) -> B(foo) -> C() == Conflict
   *     A(foo) -> B() -> C(foo) == Valid
   *     A(foo),B(foo) -> C() -> D() == Conflict
   *     A() -> B(foo),C(foo) -> D() == Conflict
   *     A(foo),B(foo) -> C() -> D(foo) == Valid
   *     A() -> B(foo),C(foo) -> D(foo) == Valid
   * </pre>
   *
   * <p>Combining resources are much simpler -- since a combining (id and styleable) resource does
   * not get replaced when redefined, they are simply combined:
   *
   * <pre>
   *     A(foo) -> B(foo) -> C(foo) == Valid
   *
   * </pre>
   *
   * @param transitive The transitive dependencies to merge.
   * @param direct The direct dependencies to merge.
   * @param primaryData The primary data to merge against.
   * @param allowPrimaryOverrideAll Boolean that indicates if the primary data will be considered
   *     the ultimate source of truth, provided it doesn't conflict with itself.
   * @return An UnwrittenMergedAndroidData, containing DataResource objects that can be written to
   *     disk for aapt processing or serialized for future merge passes.
   * @throws MergingException if there are issues with parsing resources from primaryData.
   * @throws MergeConflictException if there are merge conflicts
   */
  UnwrittenMergedAndroidData merge(
      ParsedAndroidData transitive,
      ParsedAndroidData direct,
      UnvalidatedAndroidData primaryData,
      boolean allowPrimaryOverrideAll,
      boolean throwOnResourceConflict) {
    try {
      // Extract the primary resources.
      ParsedAndroidData parsedPrimary = ParsedAndroidData.from(primaryData);
      return doMerge(
          transitive,
          direct,
          parsedPrimary,
          primaryData.getManifest(),
          allowPrimaryOverrideAll,
          throwOnResourceConflict);
    } catch (IOException e) {
      throw MergingException.wrapException(e);
    }
  }

  UnwrittenMergedAndroidData doMerge(
      ParsedAndroidData transitive,
      ParsedAndroidData direct,
      ParsedAndroidData parsedPrimary,
      Path primaryManifest,
      boolean allowPrimaryOverrideAll,
      boolean throwOnResourceConflict) {

    // Create the builders for the final parsed data.
    final ParsedAndroidData.Builder primaryBuilder = ParsedAndroidData.Builder.newBuilder();
    final ParsedAndroidData.Builder transitiveBuilder = ParsedAndroidData.Builder.newBuilder();
    final KeyValueConsumers transitiveConsumers = transitiveBuilder.consumers();
    final KeyValueConsumers primaryConsumers = primaryBuilder.consumers();
    final Set<MergeConflict> conflicts = new LinkedHashSet<>();

    // Find all internal conflicts.
    conflicts.addAll(parsedPrimary.conflicts());
    for (MergeConflict conflict : Iterables.concat(direct.conflicts(), transitive.conflicts())) {
      if (allowPrimaryOverrideAll
          && (parsedPrimary.containsOverwritable(conflict.dataKey())
              || parsedPrimary.containsAsset(conflict.dataKey()))) {
        continue;
      }
      conflicts.add(conflict);
    }

    // overwriting resources
    for (Map.Entry<DataKey, DataResource> entry : parsedPrimary.iterateOverwritableEntries()) {
      if (direct.containsOverwritable(entry.getKey())) {
        primaryConsumers.overwritingConsumer.accept(
            entry.getKey(), entry.getValue().overwrite(direct.getOverwritable(entry.getKey())));
      } else {
        primaryConsumers.overwritingConsumer.accept(entry.getKey(), entry.getValue());
      }
    }

    for (Map.Entry<DataKey, DataResource> entry : direct.iterateOverwritableEntries()) {
      // Direct dependencies are simply overwritten, no conflict.
      if (!parsedPrimary.containsOverwritable(entry.getKey())) {
        transitiveConsumers.overwritingConsumer.accept(entry.getKey(), entry.getValue());
      }
    }
    for (Map.Entry<DataKey, DataResource> entry : transitive.iterateOverwritableEntries()) {
      // If the primary is considered to be intentional (usually at the binary level),
      // skip.
      if (allowPrimaryOverrideAll && parsedPrimary.containsOverwritable(entry.getKey())) {
        continue;
      }
      // If a transitive value is in the direct map, report a conflict, as it is commonly
      // unintentional.
      if (direct.containsOverwritable(entry.getKey())) {
        conflicts.add(direct.foundResourceConflict(entry.getKey(), entry.getValue()));
      } else if (parsedPrimary.containsOverwritable(entry.getKey())) {
        // If overwriting a transitive value with a primary map, assume it's an unintentional
        // override, unless allowPrimaryOverrideAll is set. At which point, this code path
        // should not be reached.
        conflicts.add(parsedPrimary.foundResourceConflict(entry.getKey(), entry.getValue()));
      } else {
        // If it's in none of the of sources, add it.
        transitiveConsumers.overwritingConsumer.accept(entry.getKey(), entry.getValue());
      }
    }

    // combining resources
    for (Map.Entry<DataKey, DataResource> entry : parsedPrimary.iterateCombiningEntries()) {
      primaryConsumers.combiningConsumer.accept(entry.getKey(), entry.getValue());
    }
    for (Map.Entry<DataKey, DataResource> entry : direct.iterateCombiningEntries()) {
      if (parsedPrimary.containsCombineable(entry.getKey())) {
        // If it is in the primary, add it to the primary to be combined.
        primaryConsumers.combiningConsumer.accept(entry.getKey(), entry.getValue());
      } else {
        // If the combining asset is not in the primary, put it into the transitive.
        transitiveConsumers.combiningConsumer.accept(entry.getKey(), entry.getValue());
      }
    }
    for (Map.Entry<DataKey, DataResource> entry : transitive.iterateCombiningEntries()) {
      if (parsedPrimary.containsCombineable(entry.getKey())) {
        primaryConsumers.combiningConsumer.accept(entry.getKey(), entry.getValue());
      } else {
        transitiveConsumers.combiningConsumer.accept(entry.getKey(), entry.getValue());
      }
    }

    // assets
    for (Map.Entry<DataKey, DataAsset> entry : parsedPrimary.iterateAssetEntries()) {
      if (direct.containsAsset(entry.getKey())) {
        primaryConsumers.assetConsumer.accept(
            entry.getKey(), entry.getValue().overwrite(direct.getAsset(entry.getKey())));
      } else {
        primaryConsumers.assetConsumer.accept(entry.getKey(), entry.getValue());
      }
    }

    for (Map.Entry<DataKey, DataAsset> entry : direct.iterateAssetEntries()) {
      // Direct dependencies are simply overwritten, no conflict.
      if (!parsedPrimary.containsAsset(entry.getKey())) {
        transitiveConsumers.assetConsumer.accept(entry.getKey(), entry.getValue());
      }
    }
    for (Map.Entry<DataKey, DataAsset> entry : transitive.iterateAssetEntries()) {
      // If the primary is considered to be intentional (usually at the binary level),
      // skip.
      if (allowPrimaryOverrideAll && parsedPrimary.containsAsset(entry.getKey())) {
        continue;
      }
      // If a transitive value is in the direct map report a conflict, as it is commonly
      // unintentional.
      if (direct.containsAsset(entry.getKey())) {
        conflicts.add(direct.foundAssetConflict(entry.getKey(), entry.getValue()));
      } else if (parsedPrimary.containsAsset(entry.getKey())) {
        // If overwriting a transitive value with a primary map, assume it's an unintentional
        // override, unless allowPrimaryOverrideAll is set. At which point, this code path
        // should not be reached.
        conflicts.add(parsedPrimary.foundAssetConflict(entry.getKey(), entry.getValue()));
      } else {
        // If it's in none of the of sources, add it.
        transitiveConsumers.assetConsumer.accept(entry.getKey(), entry.getValue());
      }
    }
    final UnwrittenMergedAndroidData unwrittenMergedAndroidData = UnwrittenMergedAndroidData.of(
        primaryManifest, primaryBuilder.build(), transitiveBuilder.build(), conflicts);

    try {
      List<String> messages = unwrittenMergedAndroidData
          .asConflictMessagesIfValidWith(deDuplicator);

      if (!messages.isEmpty()) {
        if (throwOnResourceConflict) {
          throw MergeConflictException.withMessage(Joiner.on("\n").join(messages));
        } else {
          logger.warning(Joiner.on("\n").join(messages));
        }
      }
    } catch (IOException e) {
      throw MergingException.wrapException(e);
    }

    return unwrittenMergedAndroidData;
  }
}