src/tools/android/java/com/google/devtools/build/android/aapt2/ProtoResourceUsageAnalyzer.java

// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.android.aapt2;

import static com.android.SdkConstants.ATTR_DISCARD;
import static com.android.SdkConstants.ATTR_KEEP;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toList;

import com.android.build.gradle.tasks.ResourceUsageAnalyzer;
import com.android.resources.ResourceFolderType;
import com.android.resources.ResourceType;
import com.android.tools.lint.checks.ResourceUsageModel;
import com.android.tools.lint.checks.ResourceUsageModel.Resource;
import com.android.tools.lint.detector.api.LintUtils;
import com.android.utils.XmlUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Multimap;
import com.google.devtools.build.android.aapt2.ProtoApk.ManifestVisitor;
import com.google.devtools.build.android.aapt2.ProtoApk.ReferenceVisitor;
import com.google.devtools.build.android.aapt2.ProtoApk.ResourcePackageVisitor;
import com.google.devtools.build.android.aapt2.ProtoApk.ResourceValueVisitor;
import com.google.devtools.build.android.aapt2.ProtoApk.ResourceVisitor;
import com.sun.org.apache.xerces.internal.dom.AttrImpl;
import java.io.IOException;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.logging.Logger;
import javax.annotation.CheckReturnValue;
import javax.annotation.Nullable;
import javax.xml.parsers.ParserConfigurationException;
import org.w3c.dom.Attr;
import org.w3c.dom.DOMException;

/** A resource usage analyzer tha functions on apks in protocol buffer format. */
public class ProtoResourceUsageAnalyzer extends ResourceUsageAnalyzer {

  private static final Logger logger = Logger.getLogger(ProtoResourceUsageAnalyzer.class.getName());
  private final Set<String> resourcePackages;
  private final Path rTxt;
  private final Path mapping;
  private final Path resourcesConfigFile;

  public ProtoResourceUsageAnalyzer(
      Set<String> resourcePackages,
      Path rTxt,
      Path mapping,
      Path resourcesConfigFile,
      Path logFile)
      throws DOMException, ParserConfigurationException {
    super(resourcePackages, null, null, null, null, null, logFile);
    this.resourcePackages = resourcePackages;
    this.rTxt = rTxt;
    this.mapping = mapping;
    this.resourcesConfigFile = resourcesConfigFile;
  }

  private static Resource parse(ResourceUsageModel model, String resourceTypeAndName) {
    final Iterator<String> iterator = Splitter.on('/').split(resourceTypeAndName).iterator();
    Preconditions.checkArgument(
        iterator.hasNext(), "%s invalid resource name", resourceTypeAndName);
    ResourceType resourceType = ResourceType.getEnum(iterator.next());
    Preconditions.checkArgument(
        iterator.hasNext(), "%s invalid resource name", resourceTypeAndName);
    return model.getResource(resourceType, iterator.next());
  }

  /**
   * Calculate and removes unused resource from the {@link ProtoApk}.
   *
   * @param apk An apk in the aapt2 proto format.
   * @param classes The associated classes for the apk.
   * @param destination Where to write the reduced resources.
   * @param toolAttributes A map of the tool attributes designating resources to keep or discard.
   */
  @CheckReturnValue
  public ProtoApk shrink(
      ProtoApk apk, Path classes, Path destination, ListMultimap<String, String> toolAttributes)
      throws IOException {

    // Set the usage analyzer as parent to make sure that the usage log contains the subclass data.
    logger.setParent(Logger.getLogger(ResourceUsageAnalyzer.class.getName()));
    // record resources and manifest
    apk.visitResources(
        // First, collect all declarations using the declaration visitor.
        // This allows the model to start with a defined set of resources to build the reference
        // graph on.
        apk.visitResources(new ResourceDeclarationVisitor(model())).toUsageVisitor());

    try {
      // TODO(b/112810967): Remove reflection hack.
      final Method parseResourceTxtFile =
          ResourceUsageAnalyzer.class.getDeclaredMethod(
              "parseResourceTxtFile", Path.class, Set.class);
      parseResourceTxtFile.setAccessible(true);
      parseResourceTxtFile.invoke(this, rTxt, resourcePackages);
      final Method recordMapping =
          ResourceUsageAnalyzer.class.getDeclaredMethod("recordMapping", Path.class);
      recordMapping.setAccessible(true);
      recordMapping.invoke(this, mapping);
    } catch (ReflectiveOperationException e) {
      throw new RuntimeException(e);
    }
    recordClassUsages(classes);

    toolAttributes.entries().stream()
        .filter(entry -> entry.getKey().equals(ATTR_KEEP) || entry.getKey().equals(ATTR_DISCARD))
        .map(entry -> createSimpleAttr(entry.getKey(), entry.getValue()))
        .forEach(attr -> model().recordToolsAttributes(attr));
    model().processToolsAttributes();

    keepPossiblyReferencedResources();

    final List<Resource> resources = model().getResources();

    ImmutableList<String> resourceConfigs =
        resources.stream()
            .filter(Resource::isKeep)
            .map(r -> String.format("%s/%s#no_collapse", r.type.getName(), r.name))
            .collect(toImmutableList());
    Files.write(resourcesConfigFile, resourceConfigs, StandardCharsets.UTF_8);

    List<Resource> roots =
        resources.stream().filter(r -> r.isKeep() || r.isReachable()).collect(toList());

    final Set<Resource> reachable = findReachableResources(roots);
    return apk.copy(
        destination,
        (resourceType, name) -> reachable.contains(model().getResource(resourceType, name)));
  }

  private Set<Resource> findReachableResources(List<Resource> roots) {
    final Multimap<Resource, Resource> referenceLog = LinkedHashMultimap.create();
    Deque<Resource> queue = new ArrayDeque<>(roots);
    final Set<Resource> reachable = new LinkedHashSet<>();
    while (!queue.isEmpty()) {
      Resource resource = queue.pop();
      if (resource.references != null) {
        resource.references.forEach(
            r -> {
              referenceLog.put(r, resource);
              // add if it has not been marked reachable, therefore processed.
              if (!reachable.contains(r)) {
                queue.add(r);
              }
            });
      }
      // if we see it, it is reachable.
      reachable.add(resource);
    }

    // dump resource reference map:
    final StringBuilder keptResourceLog = new StringBuilder();
    referenceLog
        .asMap()
        .forEach(
            (resource, referencesTo) ->
                keptResourceLog
                    .append(printResource(resource))
                    .append(" => [")
                    .append(
                        referencesTo.stream()
                            .map(ProtoResourceUsageAnalyzer::printResource)
                            .collect(joining(", ")))
                    .append("]\n"));

    logger.fine("Kept resource references:\n" + keptResourceLog);

    return reachable;
  }

  private static String printResource(Resource res) {
    return String.format(
        "{%s[isRoot: %s] = %s}",
        res.getUrl(), res.isReachable() || res.isKeep(), "0x" + Integer.toHexString(res.value));
  }

  private static final class ResourceDeclarationVisitor implements ResourceVisitor {

    private final ResourceShrinkerUsageModel model;
    private final Set<Integer> packageIds = new LinkedHashSet<>();

    private ResourceDeclarationVisitor(ResourceShrinkerUsageModel model) {
      this.model = model;
    }

    @Nullable
    @Override
    public ManifestVisitor enteringManifest() {
      return null;
    }

    @Override
    public ResourcePackageVisitor enteringPackage(int pkgId, String packageName) {
      packageIds.add(pkgId);
      return (typeId, resourceType) ->
          (name, resourceId) -> {
            String hexId =
                String.format(
                    "0x%s", Integer.toHexString(((pkgId << 24) | (typeId << 16) | resourceId)));
            model.addDeclaredResource(resourceType, LintUtils.getFieldName(name), hexId, true);
            // Skip visiting the definition when collecting declarations.
            return null;
          };
    }

    ResourceUsageVisitor toUsageVisitor() {
      return new ResourceUsageVisitor(model, ImmutableSet.copyOf(packageIds));
    }
  }

  private static final class ResourceUsageVisitor implements ResourceVisitor {

    private final ResourceShrinkerUsageModel model;
    private final ImmutableSet<Integer> packageIds;

    private ResourceUsageVisitor(
        ResourceShrinkerUsageModel model, ImmutableSet<Integer> packageIds) {
      this.model = model;
      this.packageIds = packageIds;
    }

    @Override
    public ManifestVisitor enteringManifest() {
      return new ManifestVisitor() {
        @Override
        public void accept(String name) {
          ResourceUsageModel.markReachable(model.getResourceFromUrl(name));
        }

        @Override
        public void accept(int value) {
          ResourceUsageModel.markReachable(model.getResource(value));
        }
      };
    }

    @Override
    public ResourcePackageVisitor enteringPackage(int pkgId, String packageName) {
      return (typeId, resourceType) ->
          (name, resourceId) ->
              new ResourceUsageValueVisitor(
                  model, model.getResource(resourceType, name), packageIds);
    }
  }

  private static final class ResourceUsageValueVisitor implements ResourceValueVisitor {

    private final ResourceUsageModel model;
    private final Resource declaredResource;
    private final ImmutableSet<Integer> packageIds;

    private ResourceUsageValueVisitor(
        ResourceUsageModel model, Resource declaredResource, ImmutableSet<Integer> packageIds) {
      this.model = model;
      this.declaredResource = declaredResource;
      this.packageIds = packageIds;
    }

    @Override
    public ReferenceVisitor entering(Path path) {
      return this;
    }

    @Override
    public void acceptOpaqueFileType(Path path) {
      try {
        String pathString = path.toString();
        if (pathString.endsWith(".js")) {
          model.tokenizeJs(
              declaredResource, new String(Files.readAllBytes(path), StandardCharsets.UTF_8));
        } else if (pathString.endsWith(".css")) {
          model.tokenizeCss(
              declaredResource, new String(Files.readAllBytes(path), StandardCharsets.UTF_8));
        } else if (pathString.endsWith(".html")) {
          model.tokenizeHtml(
              declaredResource, new String(Files.readAllBytes(path), StandardCharsets.UTF_8));
        } else if (pathString.endsWith(".xml")) {
          // Force parsing of raw xml files to get any missing keep attributes.
          // The tool keep and discard attributes are held in raw files.
          // There is already processing to handle this, but there has been flakiness.
          // This step is to ensure as much stability as possible until the flakiness can be
          // diagnosed.
          model.recordResourceReferences(
              ResourceFolderType.getTypeByName(declaredResource.type.getName()),
              XmlUtils.parseDocumentSilently(
                  new String(Files.readAllBytes(path), StandardCharsets.UTF_8), true),
              declaredResource);

        } else {
          // Path is a reference to the apk zip -- unpack it before getting a file reference.
          model.tokenizeUnknownBinary(
              declaredResource,
              Files.copy(
                      path,
                      Files.createTempFile("binary-resource", null),
                      StandardCopyOption.REPLACE_EXISTING)
                  .toFile());
        }
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }

    @Override
    public void accept(String name) {
      declaredResource.addReference(parse(model, name));
    }

    @Override
    public void accept(int value) {
      if (isInDeclaredPackages(value)) { // ignore references outside of scanned packages.
        declaredResource.addReference(model.getResource(value));
      }
    }

    /** Tests if the id is in any of the scanned packages. */
    private boolean isInDeclaredPackages(int value) {
      return packageIds.contains(value >> 24);
    }
  }

  @VisibleForTesting
  public static Attr createSimpleAttr(String simpleName, String simpleValue) {
    return new AttrImpl() {
      @Override
      public String getLocalName() {
        return simpleName;
      }

      @Override
      public String getValue() {
        return simpleValue;
      }
    };
  }
}